Compare commits
5
Commits
test
...
fix-list-resume
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d1aad23b53 | ||
|
|
07f033d586 | ||
|
|
d324e0e1b0 | ||
|
|
decd890e14 | ||
|
|
32abd40c18 |
No files matched your search
+38
-32
@@ -41,6 +41,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/resume"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -596,6 +597,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
BucketBased: true,
|
||||
BucketBasedRootOK: true,
|
||||
ChunkWriterDoesntSeek: true,
|
||||
CanResumeListing: true,
|
||||
}).Fill(ctx, f)
|
||||
// Set the test flag if required
|
||||
if opt.TestMode != "" {
|
||||
@@ -804,7 +806,7 @@ var errEndList = errors.New("end list")
|
||||
// If hidden is set then it will list the hidden (deleted) files too.
|
||||
//
|
||||
// if findFile is set it will look for files called (bucket, directory)
|
||||
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int, hidden bool, findFile bool, fn listFn) error {
|
||||
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int, hidden bool, findFile bool, startAfter string, fn listFn) error {
|
||||
if !findFile {
|
||||
if prefix != "" {
|
||||
prefix += "/"
|
||||
@@ -831,7 +833,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
Prefix: f.opt.Enc.FromStandardPath(directory),
|
||||
Delimiter: delimiter,
|
||||
}
|
||||
if directory != "" {
|
||||
if startAfter != "" {
|
||||
request.StartFileName = f.opt.Enc.FromStandardPath(startAfter)
|
||||
} else if directory != "" {
|
||||
request.StartFileName = f.opt.Enc.FromStandardPath(directory)
|
||||
}
|
||||
opts := rest.Opts{
|
||||
@@ -930,27 +934,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *api.File
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// listDir lists a single directory
|
||||
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
|
||||
last := ""
|
||||
err = f.list(ctx, bucket, directory, prefix, f.rootBucket == "", false, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return callback(entry)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// bucket must be present if listing succeeded
|
||||
f.cache.MarkOK(bucket)
|
||||
return nil
|
||||
}
|
||||
|
||||
// listBuckets returns all the buckets to out
|
||||
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
|
||||
err = f.listBucketsToFn(ctx, "", func(bucket *api.Bucket) error {
|
||||
@@ -990,9 +973,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// These need not be returned in any particular order. If
|
||||
// callback returns an error then the listing will stop
|
||||
// immediately.
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
list := list.NewHelper(callback)
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
bucket, directory := f.split(dir)
|
||||
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done(&returnErr)
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
if bucket == "" {
|
||||
if directory != "" {
|
||||
return fs.ErrorListBucketRequired
|
||||
@@ -1008,10 +997,21 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err := f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", list.Add)
|
||||
last := ""
|
||||
err := f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", false, 0, f.opt.Versions, false, startAfter, func(remote string, object *api.File, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return list.Add(entry)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.cache.MarkOK(bucket)
|
||||
}
|
||||
return list.Flush()
|
||||
}
|
||||
@@ -1032,12 +1032,18 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
//
|
||||
// Don't implement this unless you have a more efficient way
|
||||
// of listing recursively that doing a directory traversal.
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
bucket, directory := f.split(dir)
|
||||
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done(&returnErr)
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
listR := func(bucket, directory, prefix string, addBucket bool) error {
|
||||
last := ""
|
||||
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
|
||||
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, startAfter, func(remote string, object *api.File, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1064,7 +1070,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
f.cache.MarkOK(bucket)
|
||||
}
|
||||
} else {
|
||||
err = listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
|
||||
err := listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1458,7 +1464,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
|
||||
}
|
||||
|
||||
last := ""
|
||||
checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
|
||||
checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, "", func(remote string, object *api.File, isDirectory bool) error {
|
||||
if !isDirectory {
|
||||
oi, err := f.newObjectWithInfo(ctx, object.Name, object)
|
||||
if err != nil {
|
||||
@@ -1663,7 +1669,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
}
|
||||
_, err = f.NewObject(ctx, remote)
|
||||
if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
|
||||
err2 := f.list(ctx, bucket, bucketPath, f.rootDirectory, f.rootBucket == "", false, 1, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
|
||||
err2 := f.list(ctx, bucket, bucketPath, f.rootDirectory, f.rootBucket == "", false, 1, f.opt.Versions, false, "", func(remote string, object *api.File, isDirectory bool) error {
|
||||
err = nil
|
||||
return nil
|
||||
})
|
||||
@@ -1805,7 +1811,7 @@ func (o *Object) getMetaDataListing(ctx context.Context) (info *api.File, err er
|
||||
maxSearched = maxVersions
|
||||
}
|
||||
|
||||
err = o.fs.list(ctx, bucket, bucketPath, "", false, true, maxSearched, o.fs.opt.Versions, true, func(remote string, object *api.File, isDirectory bool) error {
|
||||
err = o.fs.list(ctx, bucket, bucketPath, "", false, true, maxSearched, o.fs.opt.Versions, true, "", func(remote string, object *api.File, isDirectory bool) error {
|
||||
if isDirectory {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -528,7 +528,7 @@ func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {
|
||||
func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {
|
||||
bucket, directory := f.split(dirName)
|
||||
foundFiles := []string{}
|
||||
require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
|
||||
require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, "", func(remote string, object *api.File, isDirectory bool) error {
|
||||
if !isDirectory {
|
||||
foundFiles = append(foundFiles, object.Name)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/lib/bucket"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/resume"
|
||||
)
|
||||
|
||||
// Register with Fs
|
||||
@@ -128,6 +129,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
SetTier: true,
|
||||
GetTier: true,
|
||||
SlowModTime: true,
|
||||
CanResumeListing: true,
|
||||
}).Fill(ctx, f)
|
||||
if f.rootBucket != "" && f.rootDirectory != "" && !strings.HasSuffix(root, "/") {
|
||||
// Check to see if the (bucket,directory) is actually an existing file
|
||||
@@ -270,10 +272,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// These need not be returned in any particular order. If
|
||||
// callback returns an error then the listing will stop
|
||||
// immediately.
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
list := list.NewHelper(callback)
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
bucketName, directory := f.split(dir)
|
||||
fs.Debugf(f, "listing: bucket : %v, directory: %v", bucketName, dir)
|
||||
startAfter, callback, done, err := resume.Setup(ctx, bucketName != "", f, dir, callback)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done(&returnErr)
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
if bucketName == "" {
|
||||
if directory != "" {
|
||||
return fs.ErrorListBucketRequired
|
||||
@@ -289,10 +297,21 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err := f.listDir(ctx, bucketName, directory, f.rootDirectory, f.rootBucket == "", list.Add)
|
||||
fn := func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return list.Add(entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err := f.list(ctx, bucketName, directory, f.rootDirectory, f.rootBucket == "", false, 0, startAfter, fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.cache.MarkOK(bucketName)
|
||||
}
|
||||
return list.Flush()
|
||||
}
|
||||
@@ -310,7 +329,7 @@ type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory
|
||||
// If hidden is set then it will list the hidden (deleted) files too.
|
||||
// if findFile is set it will look for files called (bucket, directory)
|
||||
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,
|
||||
fn listFn) (err error) {
|
||||
startAfter string, fn listFn) (err error) {
|
||||
if prefix != "" {
|
||||
prefix += "/"
|
||||
}
|
||||
@@ -336,6 +355,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
|
||||
if delimiter != "" {
|
||||
request.Delimiter = common.String(delimiter)
|
||||
}
|
||||
if startAfter != "" {
|
||||
request.Start = common.String(startAfter)
|
||||
}
|
||||
|
||||
for {
|
||||
var resp objectstorage.ListObjectsResponse
|
||||
@@ -442,27 +464,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *objectst
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// listDir lists a single directory
|
||||
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
|
||||
fn := func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return callback(entry)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// bucket must be present if listing succeeded
|
||||
f.cache.MarkOK(bucket)
|
||||
return nil
|
||||
}
|
||||
|
||||
// listBuckets returns all the buckets to out
|
||||
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
|
||||
if f.opt.Provider == noAuth {
|
||||
@@ -718,11 +719,17 @@ immediately.
|
||||
Don't implement this unless you have a more efficient way
|
||||
of listing recursively that doing a directory traversal.
|
||||
*/
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
bucketName, directory := f.split(dir)
|
||||
startAfter, callback, done, err := resume.Setup(ctx, bucketName != "", f, dir, callback)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done(&returnErr)
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
listR := func(bucket, directory, prefix string, addBucket bool) error {
|
||||
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
|
||||
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, startAfter, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -749,7 +756,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
f.cache.MarkOK(bucketName)
|
||||
}
|
||||
} else {
|
||||
err = listR(bucketName, directory, f.rootDirectory, f.rootBucket == "")
|
||||
err := listR(bucketName, directory, f.rootDirectory, f.rootBucket == "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+47
-34
@@ -63,6 +63,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"github.com/rclone/rclone/lib/resume"
|
||||
"github.com/rclone/rclone/lib/transferaccounter"
|
||||
"github.com/rclone/rclone/lib/version"
|
||||
)
|
||||
@@ -1831,6 +1832,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
|
||||
SetTier: provider.StorageClass.Len() > 0,
|
||||
GetTier: provider.StorageClass.Len() > 0,
|
||||
SlowModTime: true,
|
||||
CanResumeListing: true,
|
||||
}).Fill(ctx, f)
|
||||
if opt.Provider == "AWS" {
|
||||
f.features.DoubleSlash = true
|
||||
@@ -2041,6 +2043,10 @@ func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister {
|
||||
// Convert v2 req into v1 req
|
||||
//structs.SetFrom(&l.req, req)
|
||||
setFrom_s3ListObjectsInput_s3ListObjectsV2Input(&l.req, req)
|
||||
// Map v2 StartAfter to v1 Marker (same semantics: list after this key)
|
||||
if req.StartAfter != nil {
|
||||
l.req.Marker = req.StartAfter
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
@@ -2306,6 +2312,7 @@ type listOpt struct {
|
||||
versionAt fs.Time // if set only show versions <= this time
|
||||
noSkipMarkers bool // if set return dir marker objects
|
||||
restoreStatus bool // if set return restore status in listing too
|
||||
startAfter string // if set, start listing strictly after this key
|
||||
}
|
||||
|
||||
// list lists the objects into the function supplied with the opt
|
||||
@@ -2351,6 +2358,9 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
|
||||
if f.opt.RequesterPays {
|
||||
req.RequestPayer = types.RequestPayerRequester
|
||||
}
|
||||
if opt.startAfter != "" {
|
||||
req.StartAfter = &opt.startAfter
|
||||
}
|
||||
var listBucket bucketLister
|
||||
switch {
|
||||
case opt.withVersions || opt.versionAt.IsSet():
|
||||
@@ -2517,35 +2527,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *types.Ob
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// listDir lists files and directories to out
|
||||
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
|
||||
// List the objects and directories
|
||||
err = f.list(ctx, listOpt{
|
||||
bucket: bucket,
|
||||
directory: directory,
|
||||
prefix: prefix,
|
||||
addBucket: addBucket,
|
||||
withVersions: f.opt.Versions,
|
||||
versionAt: f.opt.VersionAt,
|
||||
hidden: f.opt.VersionDeleted,
|
||||
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return callback(entry)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// bucket must be present if listing succeeded
|
||||
f.cache.MarkOK(bucket)
|
||||
return nil
|
||||
}
|
||||
|
||||
// listBuckets lists the buckets to out
|
||||
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
|
||||
req := s3.ListBucketsInput{}
|
||||
@@ -2592,9 +2573,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// These need not be returned in any particular order. If
|
||||
// callback returns an error then the listing will stop
|
||||
// immediately.
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
|
||||
list := list.NewHelper(callback)
|
||||
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
bucket, directory := f.split(dir)
|
||||
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done(&returnErr)
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
if bucket == "" {
|
||||
if directory != "" {
|
||||
return fs.ErrorListBucketRequired
|
||||
@@ -2610,10 +2597,29 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
}
|
||||
}
|
||||
} else {
|
||||
err := f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", list.Add)
|
||||
err := f.list(ctx, listOpt{
|
||||
bucket: bucket,
|
||||
directory: directory,
|
||||
prefix: f.rootDirectory,
|
||||
addBucket: f.rootBucket == "",
|
||||
withVersions: f.opt.Versions,
|
||||
versionAt: f.opt.VersionAt,
|
||||
hidden: f.opt.VersionDeleted,
|
||||
startAfter: startAfter,
|
||||
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if entry != nil {
|
||||
return list.Add(entry)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f.cache.MarkOK(bucket)
|
||||
}
|
||||
return list.Flush()
|
||||
}
|
||||
@@ -2634,8 +2640,14 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
|
||||
//
|
||||
// Don't implement this unless you have a more efficient way
|
||||
// of listing recursively than doing a directory traversal.
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
bucket, directory := f.split(dir)
|
||||
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer done(&returnErr)
|
||||
|
||||
list := list.NewHelper(callback)
|
||||
listR := func(bucket, directory, prefix string, addBucket bool) error {
|
||||
return f.list(ctx, listOpt{
|
||||
@@ -2647,6 +2659,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
withVersions: f.opt.Versions,
|
||||
versionAt: f.opt.VersionAt,
|
||||
hidden: f.opt.VersionDeleted,
|
||||
startAfter: startAfter,
|
||||
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
|
||||
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
|
||||
if err != nil {
|
||||
@@ -2674,7 +2687,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
f.cache.MarkOK(bucket)
|
||||
}
|
||||
} else {
|
||||
err = listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
|
||||
err := listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2546,6 +2546,50 @@ This sets the interval between each retry specified by `--retries`
|
||||
|
||||
The default is `0`. Use `0` to disable.
|
||||
|
||||
### --resume-listings string
|
||||
|
||||
Set this to a local directory path to enable resumable listings.
|
||||
When a long-running listing operation (e.g. `ls`, `sync`, `copy`) is
|
||||
interrupted, rclone saves checkpoint files in this directory so that
|
||||
the next run can resume from where it left off rather than starting
|
||||
from scratch.
|
||||
|
||||
This is useful when working with very large buckets containing
|
||||
millions or billions of files, where listing alone can take hours or
|
||||
days.
|
||||
|
||||
How it works:
|
||||
|
||||
- During listing, rclone saves a checkpoint after each page of results
|
||||
from the backend.
|
||||
- If the operation is interrupted (e.g. Ctrl-C, crash, reboot), the
|
||||
checkpoint records how far the listing got.
|
||||
- On the next run with the same `--resume-listings` directory, rclone
|
||||
detects the checkpoint and resumes listing from that point.
|
||||
- When a listing completes successfully, its checkpoint is
|
||||
automatically deleted.
|
||||
|
||||
Only backends that support key-name-based pagination can resume
|
||||
listings. Currently supported backends include S3, B2, and Oracle
|
||||
Object Storage. Other backends will work normally but won't benefit
|
||||
from resume checkpoints.
|
||||
|
||||
Resume works with both normal listings and `--fast-list` (ListR).
|
||||
|
||||
`--resume-listings` cannot be used with `sync` or other commands that
|
||||
delete files from the destination. A resumed listing skips
|
||||
already-listed source files, which would cause them to appear missing
|
||||
and be deleted from the destination. Use `--resume-listings` with
|
||||
`copy` or `ls` instead.
|
||||
|
||||
After a resumed run completes, it is recommended to do one full run
|
||||
without `--resume-listings` to catch any files that were added in the
|
||||
already-processed key range between runs.
|
||||
|
||||
```console
|
||||
rclone copy --resume-listings /tmp/rclone-resume s3:big-bucket /local/dest
|
||||
```
|
||||
|
||||
### --server-side-across-configs
|
||||
|
||||
Allow server-side operations (e.g. copy or move) to work across
|
||||
|
||||
@@ -277,6 +277,11 @@ var ConfigOptionsInfo = Options{{
|
||||
Default: false,
|
||||
Help: "Use recursive list if available; uses more memory but fewer transactions",
|
||||
Groups: "Listing",
|
||||
}, {
|
||||
Name: "resume_listings",
|
||||
Default: "",
|
||||
Help: "Local directory path for saving listing checkpoints so interrupted listings can be resumed",
|
||||
Groups: "Listing",
|
||||
}, {
|
||||
Name: "list_cutoff",
|
||||
Default: 1_000_000,
|
||||
@@ -617,6 +622,7 @@ type ConfigInfo struct {
|
||||
Suffix string `config:"suffix"`
|
||||
SuffixKeepExtension bool `config:"suffix_keep_extension"`
|
||||
UseListR bool `config:"fast_list"`
|
||||
ResumeListings string `config:"resume_listings"`
|
||||
ListCutoff int `config:"list_cutoff"`
|
||||
BufferSize SizeSuffix `config:"buffer_size"`
|
||||
BwLimit BwTimetable `config:"bwlimit"`
|
||||
|
||||
@@ -174,6 +174,10 @@ type Features struct {
|
||||
// immediately.
|
||||
ListP func(ctx context.Context, dir string, callback ListRCallback) error
|
||||
|
||||
// CanResumeListing indicates the backend handles --resume-listings
|
||||
// internally in its ListP/ListR implementations.
|
||||
CanResumeListing bool
|
||||
|
||||
// About gets quota information from the Fs
|
||||
About func(ctx context.Context) (*Usage, error)
|
||||
|
||||
@@ -456,6 +460,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
|
||||
if mask.ListP == nil {
|
||||
ft.ListP = nil
|
||||
}
|
||||
ft.CanResumeListing = ft.CanResumeListing && mask.CanResumeListing
|
||||
if mask.About == nil {
|
||||
ft.About = nil
|
||||
}
|
||||
|
||||
@@ -1341,6 +1341,9 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
|
||||
if deleteMode != fs.DeleteModeOff && DoMove {
|
||||
return fserrors.FatalError(errors.New("can't delete and move at the same time"))
|
||||
}
|
||||
if ci.ResumeListings != "" && deleteMode != fs.DeleteModeOff {
|
||||
return fserrors.FatalError(errors.New("can't use --resume-listings with sync or other delete modes - a resumed listing skips already-listed files which would cause them to be deleted from the destination"))
|
||||
}
|
||||
// Run an extra pass to delete only
|
||||
if deleteMode == fs.DeleteModeBefore {
|
||||
if ci.TrackRenames {
|
||||
|
||||
@@ -38,6 +38,7 @@ import (
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/random"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
"github.com/rclone/rclone/lib/resume"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -1135,6 +1136,154 @@ func Run(t *testing.T, opt *Opt) {
|
||||
fstest.CheckListing(t, f, []fstest.Item{file1, file2})
|
||||
})
|
||||
|
||||
// TestFsCanResumeListing tests the resumable listing feature
|
||||
t.Run("FsCanResumeListing", func(t *testing.T) {
|
||||
skipIfNotOk(t)
|
||||
if !f.Features().CanResumeListing {
|
||||
t.Skip("FS does not support resumable listings")
|
||||
}
|
||||
require.NotNil(t, f.Features().ListP, "CanResumeListing backends must have ListP")
|
||||
|
||||
// Helper to collect all entries from ListP
|
||||
listAll := func(ctx context.Context, dir string) fs.DirEntries {
|
||||
t.Helper()
|
||||
var entries fs.DirEntries
|
||||
err := f.Features().ListP(ctx, dir, func(e fs.DirEntries) error {
|
||||
entries = append(entries, e...)
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return entries
|
||||
}
|
||||
|
||||
// List without resume - get baseline
|
||||
baseEntries := listAll(ctx, "")
|
||||
require.NotEmpty(t, baseEntries, "test requires existing files")
|
||||
|
||||
t.Run("CheckpointSavedAndCleaned", func(t *testing.T) {
|
||||
// List with --resume-listings enabled
|
||||
resumeDir := t.TempDir()
|
||||
resumeCtx, resumeCi := fs.AddConfig(ctx)
|
||||
resumeCi.ResumeListings = resumeDir
|
||||
|
||||
entries := listAll(resumeCtx, "")
|
||||
assert.Equal(t, len(baseEntries), len(entries), "resumed listing should return same entries when no checkpoint exists")
|
||||
|
||||
// After successful completion, checkpoint should be cleaned up
|
||||
dirEntries, err := os.ReadDir(resumeDir)
|
||||
require.NoError(t, err)
|
||||
// Filter out any non-json files
|
||||
var jsonFiles []os.DirEntry
|
||||
for _, e := range dirEntries {
|
||||
if strings.HasSuffix(e.Name(), ".json") {
|
||||
jsonFiles = append(jsonFiles, e)
|
||||
}
|
||||
}
|
||||
assert.Empty(t, jsonFiles, "checkpoint should be deleted after successful listing")
|
||||
})
|
||||
|
||||
t.Run("ResumeFromCheckpoint", func(t *testing.T) {
|
||||
// Sort baseline entries to find a good startAfter key
|
||||
var names []string
|
||||
for _, e := range baseEntries {
|
||||
names = append(names, e.Remote())
|
||||
}
|
||||
sort.Strings(names)
|
||||
if len(names) < 2 {
|
||||
t.Skip("need at least 2 entries to test resume")
|
||||
}
|
||||
|
||||
// Pick the first name as startAfter - resumed listing should skip it
|
||||
startAfter := names[0]
|
||||
|
||||
// Create a checkpoint that resumes after the first entry
|
||||
resumeDir := t.TempDir()
|
||||
store, err := resume.NewStore(resumeDir)
|
||||
require.NoError(t, err)
|
||||
remoteName := fs.ConfigString(f)
|
||||
require.NoError(t, store.Save(&resume.Checkpoint{
|
||||
RemoteName: remoteName,
|
||||
Dir: "",
|
||||
LastKey: startAfter,
|
||||
}))
|
||||
|
||||
// List with resume - should skip entries <= startAfter
|
||||
resumeCtx, resumeCi := fs.AddConfig(ctx)
|
||||
resumeCi.ResumeListings = resumeDir
|
||||
|
||||
resumedEntries := listAll(resumeCtx, "")
|
||||
|
||||
var resumedNames []string
|
||||
for _, e := range resumedEntries {
|
||||
resumedNames = append(resumedNames, e.Remote())
|
||||
}
|
||||
|
||||
// The resumed listing should have fewer entries
|
||||
assert.Less(t, len(resumedEntries), len(baseEntries),
|
||||
"resumed listing should have fewer entries than full listing")
|
||||
|
||||
// The startAfter entry should not appear in the resumed listing
|
||||
assert.NotContains(t, resumedNames, startAfter,
|
||||
"entry at checkpoint should not appear in resumed listing")
|
||||
|
||||
// All resumed entries should exist in the full listing
|
||||
for _, name := range resumedNames {
|
||||
assert.Contains(t, names, name,
|
||||
"resumed entry %q should exist in full listing", name)
|
||||
}
|
||||
})
|
||||
|
||||
// Test ListR resume if available
|
||||
if f.Features().ListR != nil {
|
||||
t.Run("ListRResumeFromCheckpoint", func(t *testing.T) {
|
||||
// Get baseline from ListR
|
||||
var baseNames []string
|
||||
err := f.Features().ListR(ctx, "", func(entries fs.DirEntries) error {
|
||||
for _, e := range entries {
|
||||
baseNames = append(baseNames, e.Remote())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
sort.Strings(baseNames)
|
||||
if len(baseNames) < 2 {
|
||||
t.Skip("need at least 2 entries to test ListR resume")
|
||||
}
|
||||
|
||||
startAfter := baseNames[0]
|
||||
|
||||
// Create checkpoint
|
||||
resumeDir := t.TempDir()
|
||||
store, err := resume.NewStore(resumeDir)
|
||||
require.NoError(t, err)
|
||||
remoteName := fs.ConfigString(f)
|
||||
require.NoError(t, store.Save(&resume.Checkpoint{
|
||||
RemoteName: remoteName,
|
||||
Dir: "",
|
||||
LastKey: startAfter,
|
||||
}))
|
||||
|
||||
// List with resume
|
||||
resumeCtx, resumeCi := fs.AddConfig(ctx)
|
||||
resumeCi.ResumeListings = resumeDir
|
||||
|
||||
var resumedNames []string
|
||||
err = f.Features().ListR(resumeCtx, "", func(entries fs.DirEntries) error {
|
||||
for _, e := range entries {
|
||||
resumedNames = append(resumedNames, e.Remote())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Less(t, len(resumedNames), len(baseNames),
|
||||
"resumed ListR should have fewer entries than full listing")
|
||||
assert.NotContains(t, resumedNames, startAfter,
|
||||
"entry at checkpoint should not appear in resumed ListR")
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// TestFsNewObjectDir tests NewObject on a directory which should produce fs.ErrorIsDir if possible or fs.ErrorObjectNotFound if not
|
||||
t.Run("FsNewObjectDir", func(t *testing.T) {
|
||||
skipIfNotOk(t)
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
// Package resume manages checkpoint state files for resumable listings.
|
||||
//
|
||||
// When --resume-listings is set to a local directory path, rclone
|
||||
// saves listing progress so that interrupted listings can be resumed
|
||||
// from where they left off rather than starting from scratch.
|
||||
package resume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/fs"
|
||||
)
|
||||
|
||||
const checkpointVersion = 1
|
||||
|
||||
// Checkpoint represents the saved state of a listing in progress.
|
||||
type Checkpoint struct {
|
||||
Version int `json:"version"`
|
||||
RemoteName string `json:"remoteName"` // e.g. "s3:mybucket/path"
|
||||
Dir string `json:"dir"` // directory being listed
|
||||
LastKey string `json:"lastKey"` // last key delivered to callback
|
||||
UpdatedAt time.Time `json:"updatedAt"`
|
||||
}
|
||||
|
||||
// Store manages checkpoint files in a local directory.
|
||||
type Store struct {
|
||||
dir string
|
||||
}
|
||||
|
||||
// NewStore creates a Store that persists checkpoints in dir.
|
||||
// The directory is created if it does not exist.
|
||||
func NewStore(dir string) (*Store, error) {
|
||||
if err := os.MkdirAll(dir, 0o777); err != nil {
|
||||
return nil, fmt.Errorf("resume: create store directory: %w", err)
|
||||
}
|
||||
return &Store{dir: dir}, nil
|
||||
}
|
||||
|
||||
// stateKey returns a deterministic file path for the given remote+dir.
|
||||
func (s *Store) stateKey(remoteName, dir string) string {
|
||||
h := sha256.Sum256([]byte(remoteName + ":" + dir))
|
||||
return filepath.Join(s.dir, fmt.Sprintf("%x.json", h))
|
||||
}
|
||||
|
||||
// Load reads a checkpoint for the given remote and directory.
|
||||
// Returns nil, nil if no checkpoint exists or the checkpoint doesn't
|
||||
// match (wrong version, different remote/dir).
|
||||
func (s *Store) Load(remoteName, dir string) (*Checkpoint, error) {
|
||||
path := s.stateKey(remoteName, dir)
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("resume: read checkpoint: %w", err)
|
||||
}
|
||||
|
||||
var cp Checkpoint
|
||||
if err := json.Unmarshal(data, &cp); err != nil {
|
||||
return nil, fmt.Errorf("resume: unmarshal checkpoint: %w", err)
|
||||
}
|
||||
|
||||
if cp.Version != checkpointVersion {
|
||||
return nil, nil
|
||||
}
|
||||
if cp.RemoteName != remoteName || cp.Dir != dir {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return &cp, nil
|
||||
}
|
||||
|
||||
// Save atomically writes a checkpoint to disk.
|
||||
func (s *Store) Save(cp *Checkpoint) error {
|
||||
cp.Version = checkpointVersion
|
||||
cp.UpdatedAt = time.Now().UTC()
|
||||
|
||||
payload, err := json.MarshalIndent(cp, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume: marshal checkpoint: %w", err)
|
||||
}
|
||||
|
||||
path := s.stateKey(cp.RemoteName, cp.Dir)
|
||||
|
||||
// Atomic write: temp file + rename
|
||||
tmpFile, err := os.CreateTemp(s.dir, ".rclone-resume-*.tmp")
|
||||
if err != nil {
|
||||
return fmt.Errorf("resume: create temp file: %w", err)
|
||||
}
|
||||
tmpPath := tmpFile.Name()
|
||||
|
||||
_, writeErr := tmpFile.Write(payload)
|
||||
closeErr := tmpFile.Close()
|
||||
if writeErr != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("resume: write temp file: %w", writeErr)
|
||||
}
|
||||
if closeErr != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("resume: close temp file: %w", closeErr)
|
||||
}
|
||||
|
||||
if err := os.Rename(tmpPath, path); err != nil {
|
||||
_ = os.Remove(tmpPath)
|
||||
return fmt.Errorf("resume: rename temp file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// WrapCallback loads any existing checkpoint for this remote+dir and returns:
|
||||
// - startAfter: last key from a previous interrupted listing ("" if none)
|
||||
// - wrappedCallback: tracks lastKey per page, saves checkpoint after each page
|
||||
// - done: defer this — it deletes the checkpoint only if *errp == nil
|
||||
func (s *Store) WrapCallback(remoteName, dir string, callback fs.ListRCallback) (
|
||||
startAfter string, wrappedCallback fs.ListRCallback, done func(errp *error), err error) {
|
||||
cp, err := s.Load(remoteName, dir)
|
||||
if err != nil {
|
||||
return "", nil, nil, fmt.Errorf("resume: load checkpoint: %w", err)
|
||||
}
|
||||
if cp != nil {
|
||||
startAfter = cp.LastKey
|
||||
fs.Infof(nil, "Resuming listing of %q from checkpoint %q", dir, startAfter)
|
||||
}
|
||||
|
||||
var lastKey string
|
||||
wrappedCallback = func(entries fs.DirEntries) error {
|
||||
// Track the max key in this page for checkpointing
|
||||
for _, entry := range entries {
|
||||
key := entry.Remote()
|
||||
if key > lastKey {
|
||||
lastKey = key
|
||||
}
|
||||
}
|
||||
|
||||
// Pass entries through to the original callback
|
||||
if err := callback(entries); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Save checkpoint after each page
|
||||
if lastKey != "" {
|
||||
saveCP := &Checkpoint{
|
||||
RemoteName: remoteName,
|
||||
Dir: dir,
|
||||
LastKey: lastKey,
|
||||
}
|
||||
if err := s.Save(saveCP); err != nil {
|
||||
fs.Errorf(nil, "Failed to save listing checkpoint: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
done = func(errp *error) {
|
||||
if *errp != nil {
|
||||
return
|
||||
}
|
||||
if err := s.Delete(remoteName, dir); err != nil {
|
||||
fs.Errorf(nil, "Failed to delete listing checkpoint: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return startAfter, wrappedCallback, done, nil
|
||||
}
|
||||
|
||||
// Setup is the top-level entry point for backends to add resume support.
|
||||
//
|
||||
// If --resume-listings is not configured or enabled is false, it returns
|
||||
// passthrough values: startAfter="", the original callback, and a no-op done.
|
||||
//
|
||||
// Otherwise it creates a Store, loads any existing checkpoint, and wraps the
|
||||
// callback to save progress after each page. done should be deferred — it
|
||||
// deletes the checkpoint only when *errp == nil.
|
||||
//
|
||||
// Typical usage in a backend ListP/ListR:
|
||||
//
|
||||
// func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
|
||||
// bucket, directory := f.split(dir)
|
||||
// startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
|
||||
// if err != nil { return err }
|
||||
// defer done(&returnErr)
|
||||
// ...
|
||||
// }
|
||||
func Setup(ctx context.Context, enabled bool, f fs.Fs, dir string, callback fs.ListRCallback) (
|
||||
startAfter string, wrappedCallback fs.ListRCallback, done func(errp *error), err error) {
|
||||
ci := fs.GetConfig(ctx)
|
||||
if !enabled || ci.ResumeListings == "" {
|
||||
return "", callback, func(*error) {}, nil
|
||||
}
|
||||
store, err := NewStore(ci.ResumeListings)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
return store.WrapCallback(fs.ConfigString(f), dir, callback)
|
||||
}
|
||||
|
||||
// Delete removes the checkpoint for the given remote and directory.
|
||||
// It is not an error if the checkpoint does not exist.
|
||||
func (s *Store) Delete(remoteName, dir string) error {
|
||||
path := s.stateKey(remoteName, dir)
|
||||
err := os.Remove(path)
|
||||
if err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||
return fmt.Errorf("resume: delete checkpoint: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
package resume
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/memory"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSaveLoadRoundTrip(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
cp := &Checkpoint{
|
||||
RemoteName: "s3:mybucket/path",
|
||||
Dir: "subdir/",
|
||||
LastKey: "subdir/file-1000",
|
||||
}
|
||||
require.NoError(t, store.Save(cp))
|
||||
|
||||
loaded, err := store.Load("s3:mybucket/path", "subdir/")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, loaded)
|
||||
|
||||
assert.Equal(t, checkpointVersion, loaded.Version)
|
||||
assert.Equal(t, "s3:mybucket/path", loaded.RemoteName)
|
||||
assert.Equal(t, "subdir/", loaded.Dir)
|
||||
assert.Equal(t, "subdir/file-1000", loaded.LastKey)
|
||||
assert.False(t, loaded.UpdatedAt.IsZero())
|
||||
}
|
||||
|
||||
func TestLoadNotFound(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
cp, err := store.Load("s3:mybucket", "")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, cp)
|
||||
}
|
||||
|
||||
func TestLoadRemoteMismatch(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
cp := &Checkpoint{
|
||||
RemoteName: "s3:bucket-a",
|
||||
Dir: "dir/",
|
||||
LastKey: "dir/obj-500",
|
||||
}
|
||||
require.NoError(t, store.Save(cp))
|
||||
|
||||
// Different remote name - hash differs, so file not found
|
||||
loaded, err := store.Load("s3:bucket-b", "dir/")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, loaded)
|
||||
}
|
||||
|
||||
func TestLoadDirMismatch(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
cp := &Checkpoint{
|
||||
RemoteName: "s3:bucket",
|
||||
Dir: "dir-a/",
|
||||
LastKey: "dir-a/obj-500",
|
||||
}
|
||||
require.NoError(t, store.Save(cp))
|
||||
|
||||
// Different dir - hash differs, so file not found
|
||||
loaded, err := store.Load("s3:bucket", "dir-b/")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, loaded)
|
||||
}
|
||||
|
||||
func TestDelete(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
cp := &Checkpoint{
|
||||
RemoteName: "s3:bucket",
|
||||
Dir: "",
|
||||
LastKey: "obj-100",
|
||||
}
|
||||
require.NoError(t, store.Save(cp))
|
||||
|
||||
// Verify it exists
|
||||
loaded, err := store.Load("s3:bucket", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, loaded)
|
||||
|
||||
// Delete
|
||||
require.NoError(t, store.Delete("s3:bucket", ""))
|
||||
|
||||
// Verify it's gone
|
||||
loaded, err = store.Load("s3:bucket", "")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, loaded)
|
||||
}
|
||||
|
||||
func TestDeleteNonExistent(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should not error
|
||||
require.NoError(t, store.Delete("s3:bucket", "missing/"))
|
||||
}
|
||||
|
||||
func TestWrapCallbackNoCheckpoint(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
var received fs.DirEntries
|
||||
callback := func(entries fs.DirEntries) error {
|
||||
received = append(received, entries...)
|
||||
return nil
|
||||
}
|
||||
|
||||
startAfter, wrapped, done, err := store.WrapCallback("s3:bucket", "dir", callback)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", startAfter, "no checkpoint should give empty startAfter")
|
||||
|
||||
// Send a page of entries
|
||||
entries := fs.DirEntries{
|
||||
fs.NewDir("dir/aaa", time.Time{}),
|
||||
fs.NewDir("dir/zzz", time.Time{}),
|
||||
}
|
||||
require.NoError(t, wrapped(entries))
|
||||
assert.Len(t, received, 2, "entries should pass through")
|
||||
|
||||
// Checkpoint should exist now
|
||||
cp, err := store.Load("s3:bucket", "dir")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cp)
|
||||
assert.Equal(t, "dir/zzz", cp.LastKey)
|
||||
|
||||
// Call done with nil error - checkpoint should be deleted
|
||||
var noErr error
|
||||
done(&noErr)
|
||||
cp, err = store.Load("s3:bucket", "dir")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, cp)
|
||||
}
|
||||
|
||||
func TestWrapCallbackWithCheckpoint(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
// Save an existing checkpoint
|
||||
require.NoError(t, store.Save(&Checkpoint{
|
||||
RemoteName: "s3:bucket",
|
||||
Dir: "",
|
||||
LastKey: "file-500",
|
||||
}))
|
||||
|
||||
callback := func(entries fs.DirEntries) error { return nil }
|
||||
|
||||
startAfter, _, _, err := store.WrapCallback("s3:bucket", "", callback)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "file-500", startAfter, "should resume from checkpoint")
|
||||
}
|
||||
|
||||
func TestWrapCallbackTracksMaxKey(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
callback := func(entries fs.DirEntries) error { return nil }
|
||||
_, wrapped, _, err := store.WrapCallback("s3:bucket", "", callback)
|
||||
require.NoError(t, err)
|
||||
|
||||
// First page
|
||||
require.NoError(t, wrapped(fs.DirEntries{
|
||||
fs.NewDir("bbb", time.Time{}),
|
||||
fs.NewDir("aaa", time.Time{}),
|
||||
}))
|
||||
cp, err := store.Load("s3:bucket", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cp)
|
||||
assert.Equal(t, "bbb", cp.LastKey, "should track max key across entries")
|
||||
|
||||
// Second page with higher keys
|
||||
require.NoError(t, wrapped(fs.DirEntries{
|
||||
fs.NewDir("ddd", time.Time{}),
|
||||
fs.NewDir("ccc", time.Time{}),
|
||||
}))
|
||||
cp, err = store.Load("s3:bucket", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cp)
|
||||
assert.Equal(t, "ddd", cp.LastKey, "should track max key across pages")
|
||||
}
|
||||
|
||||
func TestSetupDisabled(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.ResumeListings = "" // not configured
|
||||
|
||||
called := false
|
||||
callback := func(entries fs.DirEntries) error {
|
||||
called = true
|
||||
return nil
|
||||
}
|
||||
|
||||
startAfter, wrapped, done, err := Setup(ctx, true, nil, "", callback)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", startAfter)
|
||||
|
||||
// Should be the original callback (passthrough)
|
||||
require.NoError(t, wrapped(nil))
|
||||
assert.True(t, called)
|
||||
|
||||
// done should be safe to call
|
||||
var noErr error
|
||||
done(&noErr)
|
||||
}
|
||||
|
||||
func TestSetupEnabledFalse(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.ResumeListings = t.TempDir()
|
||||
|
||||
callback := func(entries fs.DirEntries) error { return nil }
|
||||
|
||||
// enabled=false should return passthrough even with ResumeListings set
|
||||
startAfter, _, done, err := Setup(ctx, false, nil, "", callback)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", startAfter)
|
||||
var noErr error
|
||||
done(&noErr)
|
||||
}
|
||||
|
||||
func TestSetupEnabled(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
ctx, ci := fs.AddConfig(ctx)
|
||||
ci.ResumeListings = t.TempDir()
|
||||
|
||||
memFs, err := fs.NewFs(ctx, ":memory:")
|
||||
require.NoError(t, err)
|
||||
remoteName := fs.ConfigString(memFs)
|
||||
|
||||
var received fs.DirEntries
|
||||
callback := func(entries fs.DirEntries) error {
|
||||
received = append(received, entries...)
|
||||
return nil
|
||||
}
|
||||
|
||||
startAfter, wrapped, done, err := Setup(ctx, true, memFs, "dir", callback)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "", startAfter, "no prior checkpoint")
|
||||
|
||||
// Send entries through wrapped callback
|
||||
require.NoError(t, wrapped(fs.DirEntries{
|
||||
fs.NewDir("dir/file1", time.Time{}),
|
||||
}))
|
||||
assert.Len(t, received, 1)
|
||||
|
||||
// Checkpoint should exist
|
||||
store, err := NewStore(ci.ResumeListings)
|
||||
require.NoError(t, err)
|
||||
cp, err := store.Load(remoteName, "dir")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cp)
|
||||
assert.Equal(t, "dir/file1", cp.LastKey)
|
||||
|
||||
// done() cleans up on success
|
||||
var noErr error
|
||||
done(&noErr)
|
||||
cp, err = store.Load(remoteName, "dir")
|
||||
require.NoError(t, err)
|
||||
assert.Nil(t, cp)
|
||||
}
|
||||
|
||||
func TestSaveOverwrites(t *testing.T) {
|
||||
store, err := NewStore(t.TempDir())
|
||||
require.NoError(t, err)
|
||||
|
||||
cp := &Checkpoint{
|
||||
RemoteName: "s3:bucket",
|
||||
Dir: "",
|
||||
LastKey: "obj-100",
|
||||
}
|
||||
require.NoError(t, store.Save(cp))
|
||||
|
||||
cp.LastKey = "obj-200"
|
||||
require.NoError(t, store.Save(cp))
|
||||
|
||||
loaded, err := store.Load("s3:bucket", "")
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, loaded)
|
||||
assert.Equal(t, "obj-200", loaded.LastKey)
|
||||
}
|
||||
Reference in new issue
Block a user