protondrive: don't list a folder again after each file or folder made in it
Making a file or folder, or a server-side copy, forgot the cached listing of its folder, so the next upload into that folder - which checks the name is free - listed the whole folder again. Copying M new files into a folder of N cost on the order of M*N/100 extra requests. The new node is now added to the cached listing. A failed call still forgets it, as the call may have taken effect. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
751a3ea357
commit
3fac25de75
5 files changed
+93
-8
No files matched your search
@@ -1,6 +1,9 @@
|
||||
package protondrive
|
||||
|
||||
import "sync"
|
||||
import (
|
||||
"slices"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// nodeCache holds decrypted nodes and folder listings in memory.
|
||||
//
|
||||
@@ -88,6 +91,24 @@ func (c *nodeCache) setChildren(parentID string, nodes []*node) {
|
||||
c.listings[parentID] = ids
|
||||
}
|
||||
|
||||
// add caches n, a node new to its parent, and adds it to the cached
|
||||
// listing of its parent, if there is one.
|
||||
func (c *nodeCache) add(n *node) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if old := c.nodes[n.ID]; old != nil && old.ParentID != n.ParentID {
|
||||
delete(c.listings, old.ParentID)
|
||||
delete(c.listings, n.ParentID)
|
||||
}
|
||||
c.nodes[n.ID] = n
|
||||
if ids, ok := c.listings[n.ParentID]; ok && !slices.Contains(ids, n.ID) {
|
||||
c.listings[n.ParentID] = append(ids, n.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// invalidate forgets the node id, its listing if it is a folder, and
|
||||
// the listing of its parent.
|
||||
func (c *nodeCache) invalidate(id string) {
|
||||
|
||||
@@ -93,3 +93,30 @@ func TestNodeCache(t *testing.T) {
|
||||
_, ok = c.children("dir")
|
||||
assert.False(t, ok)
|
||||
}
|
||||
|
||||
func TestNodeCacheAdd(t *testing.T) {
|
||||
c := newNodeCache(true)
|
||||
a := &node{ID: "a", ParentID: "dir"}
|
||||
b := &node{ID: "b", ParentID: "dir"}
|
||||
|
||||
// Without a listing of its parent, only the node is cached.
|
||||
c.add(a)
|
||||
assert.Same(t, a, c.get("a"))
|
||||
_, ok := c.children("dir")
|
||||
assert.False(t, ok)
|
||||
|
||||
c.setChildren("dir", []*node{a})
|
||||
c.add(b)
|
||||
got, ok := c.children("dir")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, []*node{a, b}, got)
|
||||
|
||||
// A node already listed is replaced, not listed twice.
|
||||
b2 := &node{ID: "b", ParentID: "dir", Name: "new"}
|
||||
c.add(b2)
|
||||
got, ok = c.children("dir")
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, []*node{a, b2}, got)
|
||||
|
||||
newNodeCache(false).add(a)
|
||||
}
|
||||
@@ -956,9 +956,9 @@ func (c *client) createFolder(ctx context.Context, parentID, name string, modTim
|
||||
}
|
||||
var resp api.CreateFolderResponse
|
||||
err = c.apiCall(ctx, &rest.Opts{Method: "POST", Path: c.volumePath("/drive/v2/volumes") + "/folders"}, req, &resp)
|
||||
// A failed call may still have created the folder.
|
||||
c.cache.invalidateChildren(parentID)
|
||||
if err != nil {
|
||||
// A failed call may still have created the folder.
|
||||
c.cache.invalidateChildren(parentID)
|
||||
return nil, mutationError(what, err)
|
||||
}
|
||||
|
||||
@@ -999,7 +999,7 @@ func (c *client) createFolder(ctx context.Context, parentID, name string, modTim
|
||||
},
|
||||
}
|
||||
c.rememberKeys(n)
|
||||
c.cache.put(n)
|
||||
c.cache.add(n)
|
||||
return n, nil
|
||||
}
|
||||
|
||||
@@ -1165,15 +1165,17 @@ func (c *client) copyNode(ctx context.Context, n *node, newParentID, newName str
|
||||
}
|
||||
var resp api.CopyResponse
|
||||
err = c.apiCall(ctx, &rest.Opts{Method: "POST", Path: c.volumePath("/drive/volumes", "links", n.ID, "copy")}, req, &resp)
|
||||
// A failed call may still have made the copy.
|
||||
c.cache.invalidateChildren(newParentID)
|
||||
if err != nil {
|
||||
// A failed call may still have made the copy.
|
||||
c.cache.invalidateChildren(newParentID)
|
||||
return nil, mutationError(what, err)
|
||||
}
|
||||
cp, err := c.getNode(ctx, resp.LinkID)
|
||||
if err != nil {
|
||||
c.cache.invalidateChildren(newParentID)
|
||||
return nil, fmt.Errorf("%s: %w", what, err)
|
||||
}
|
||||
c.cache.add(cp)
|
||||
return cp, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1079,3 +1079,34 @@ func TestActiveNode(t *testing.T) {
|
||||
_, err = c.activeNode(ctx, "missing")
|
||||
assert.ErrorIs(t, err, errNotFound)
|
||||
}
|
||||
|
||||
// TestNewNodesKeepListing checks making a node in a listed folder adds
|
||||
// it to the cached listing rather than forgetting the listing, so
|
||||
// making many nodes in a folder doesn't list it again each time.
|
||||
func TestNewNodesKeepListing(t *testing.T) {
|
||||
t.Parallel()
|
||||
c, srv, u := newTestClient(t)
|
||||
ctx := context.Background()
|
||||
dirID := u.SeedFolder(u.RootID(), "dir")
|
||||
fileID := u.SeedFile(u.RootID(), "f.txt", []byte("content"), time.Now())
|
||||
require.Empty(t, listAll(t, c, dirID))
|
||||
|
||||
srv.ResetRequests()
|
||||
_, err := c.createFolder(ctx, dirID, "sub", time.Time{})
|
||||
require.NoError(t, err)
|
||||
_, err = c.upload(ctx, uploadRequest{ParentID: dirID, Name: "up.txt", In: strings.NewReader("up"), Size: 2})
|
||||
require.NoError(t, err)
|
||||
file, err := c.getNode(ctx, fileID)
|
||||
require.NoError(t, err)
|
||||
_, err = c.copyNode(ctx, file, dirID, "copy.txt")
|
||||
require.NoError(t, err)
|
||||
assert.Zero(t, srv.Requests()["GET /drive/v2/volumes/{volumeID}/folders/{linkID}/children"], "%v", srv.Requests())
|
||||
|
||||
srv.ResetRequests()
|
||||
listed := listAll(t, c, dirID)
|
||||
assert.Zero(t, totalRequests(srv), "listing not cached: %v", srv.Requests())
|
||||
assert.Len(t, listed, 3)
|
||||
for _, name := range []string{"sub", "up.txt", "copy.txt"} {
|
||||
assert.Contains(t, listed, name)
|
||||
}
|
||||
}
|
||||
@@ -388,13 +388,17 @@ func (c *client) uploadDraft(ctx context.Context, d *draft, r uploadRequest, sig
|
||||
ChecksumVerified: r.SHA1 != "",
|
||||
}
|
||||
commitErr := c.apiCall(ctx, &rest.Opts{Method: "PUT", Path: c.volumePath("/drive/v2/volumes", "files", d.nodeID, "revisions", d.revisionID)}, req, nil)
|
||||
c.forget(d.nodeID, d.parentID)
|
||||
// The outcome must be known even if ctx is done: deleting the draft
|
||||
// of a commit which took effect would delete the file or revision.
|
||||
readCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), draftCleanupTimeout)
|
||||
defer cancel()
|
||||
n, err = c.getNode(readCtx, d.nodeID)
|
||||
n, err = c.fetchNode(readCtx, d.nodeID)
|
||||
ours := err == nil && n.State == api.LinkStateActive && n.RevisionID == d.revisionID
|
||||
if ours {
|
||||
c.cache.add(n)
|
||||
} else {
|
||||
c.forget(d.nodeID, d.parentID)
|
||||
}
|
||||
switch {
|
||||
case ours && commitErr != nil:
|
||||
// The response of a commit which succeeded may have been lost,
|
||||
|
||||
Reference in new issue
Block a user