Compare commits

...
5 Commits
Author SHA1 Message Date
Nick Craig-Wood d1aad23b53 oracleobjectstorage: implement --resume-listings support 2026-03-11 18:36:47 +00:00
Nick Craig-Wood 07f033d586 b2: implement --resume-listings support 2026-03-11 18:36:47 +00:00
Nick Craig-Wood d324e0e1b0 s3: implement --resume-listings support 2026-03-11 18:36:47 +00:00
Nick Craig-Wood decd890e14 fs: add --resume-listings global flag and CanResumeListing feature flag
Add ResumeListings option to ConfigInfo. When set to a local directory
path, rclone saves listing checkpoints so interrupted listings can be
resumed from where they left off.
2026-03-11 18:31:01 +00:00
Nick Craig-Wood 32abd40c18 lib/resume: add shared library for resumable listing checkpoints
Add a new lib/resume package that manages checkpoint state files for
resumable listings. When --resume-listings is set to a local directory
path, rclone will save listing progress so that interrupted listings
can resume from where they left off.

The Store type provides Load, Save, and Delete operations for
Checkpoint structs, using atomic writes (temp file + rename) and
deterministic file naming based on SHA256 of remote+dir.
2026-03-11 18:01:04 +00:00
11 changed files with 835 additions and 95 deletions

No files matched your search

+38 -32
View File
@@ -41,6 +41,7 @@ import (
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/resume"
)
const (
@@ -596,6 +597,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
BucketBased: true,
BucketBasedRootOK: true,
ChunkWriterDoesntSeek: true,
CanResumeListing: true,
}).Fill(ctx, f)
// Set the test flag if required
if opt.TestMode != "" {
@@ -804,7 +806,7 @@ var errEndList = errors.New("end list")
// If hidden is set then it will list the hidden (deleted) files too.
//
// if findFile is set it will look for files called (bucket, directory)
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int, hidden bool, findFile bool, fn listFn) error {
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int, hidden bool, findFile bool, startAfter string, fn listFn) error {
if !findFile {
if prefix != "" {
prefix += "/"
@@ -831,7 +833,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
Prefix: f.opt.Enc.FromStandardPath(directory),
Delimiter: delimiter,
}
if directory != "" {
if startAfter != "" {
request.StartFileName = f.opt.Enc.FromStandardPath(startAfter)
} else if directory != "" {
request.StartFileName = f.opt.Enc.FromStandardPath(directory)
}
opts := rest.Opts{
@@ -930,27 +934,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *api.File
return o, nil
}
// listDir lists a single directory
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
last := ""
err = f.list(ctx, bucket, directory, prefix, f.rootBucket == "", false, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
if err != nil {
return err
}
if entry != nil {
return callback(entry)
}
return nil
})
if err != nil {
return err
}
// bucket must be present if listing succeeded
f.cache.MarkOK(bucket)
return nil
}
// listBuckets returns all the buckets to out
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
err = f.listBucketsToFn(ctx, "", func(bucket *api.Bucket) error {
@@ -990,9 +973,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
list := list.NewHelper(callback)
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
bucket, directory := f.split(dir)
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
if err != nil {
return err
}
defer done(&returnErr)
list := list.NewHelper(callback)
if bucket == "" {
if directory != "" {
return fs.ErrorListBucketRequired
@@ -1008,10 +997,21 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
}
}
} else {
err := f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", list.Add)
last := ""
err := f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", false, 0, f.opt.Versions, false, startAfter, func(remote string, object *api.File, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
if err != nil {
return err
}
if entry != nil {
return list.Add(entry)
}
return nil
})
if err != nil {
return err
}
f.cache.MarkOK(bucket)
}
return list.Flush()
}
@@ -1032,12 +1032,18 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
//
// Don't implement this unless you have a more efficient way
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
bucket, directory := f.split(dir)
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
if err != nil {
return err
}
defer done(&returnErr)
list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
last := ""
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, startAfter, func(remote string, object *api.File, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory, &last)
if err != nil {
return err
@@ -1064,7 +1070,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
f.cache.MarkOK(bucket)
}
} else {
err = listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
err := listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
if err != nil {
return err
}
@@ -1458,7 +1464,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b
}
last := ""
checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, "", func(remote string, object *api.File, isDirectory bool) error {
if !isDirectory {
oi, err := f.newObjectWithInfo(ctx, object.Name, object)
if err != nil {
@@ -1663,7 +1669,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
}
_, err = f.NewObject(ctx, remote)
if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
err2 := f.list(ctx, bucket, bucketPath, f.rootDirectory, f.rootBucket == "", false, 1, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
err2 := f.list(ctx, bucket, bucketPath, f.rootDirectory, f.rootBucket == "", false, 1, f.opt.Versions, false, "", func(remote string, object *api.File, isDirectory bool) error {
err = nil
return nil
})
@@ -1805,7 +1811,7 @@ func (o *Object) getMetaDataListing(ctx context.Context) (info *api.File, err er
maxSearched = maxVersions
}
err = o.fs.list(ctx, bucket, bucketPath, "", false, true, maxSearched, o.fs.opt.Versions, true, func(remote string, object *api.File, isDirectory bool) error {
err = o.fs.list(ctx, bucket, bucketPath, "", false, true, maxSearched, o.fs.opt.Versions, true, "", func(remote string, object *api.File, isDirectory bool) error {
if isDirectory {
return nil
}
+1 -1
View File
@@ -528,7 +528,7 @@ func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {
func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {
bucket, directory := f.split(dirName)
foundFiles := []string{}
require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, "", func(remote string, object *api.File, isDirectory bool) error {
if !isDirectory {
foundFiles = append(foundFiles, object.Name)
}
@@ -23,6 +23,7 @@ import (
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/resume"
)
// Register with Fs
@@ -128,6 +129,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
SetTier: true,
GetTier: true,
SlowModTime: true,
CanResumeListing: true,
}).Fill(ctx, f)
if f.rootBucket != "" && f.rootDirectory != "" && !strings.HasSuffix(root, "/") {
// Check to see if the (bucket,directory) is actually an existing file
@@ -270,10 +272,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
list := list.NewHelper(callback)
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
bucketName, directory := f.split(dir)
fs.Debugf(f, "listing: bucket : %v, directory: %v", bucketName, dir)
startAfter, callback, done, err := resume.Setup(ctx, bucketName != "", f, dir, callback)
if err != nil {
return err
}
defer done(&returnErr)
list := list.NewHelper(callback)
if bucketName == "" {
if directory != "" {
return fs.ErrorListBucketRequired
@@ -289,10 +297,21 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
}
}
} else {
err := f.listDir(ctx, bucketName, directory, f.rootDirectory, f.rootBucket == "", list.Add)
fn := func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
if err != nil {
return err
}
if entry != nil {
return list.Add(entry)
}
return nil
}
err := f.list(ctx, bucketName, directory, f.rootDirectory, f.rootBucket == "", false, 0, startAfter, fn)
if err != nil {
return err
}
f.cache.MarkOK(bucketName)
}
return list.Flush()
}
@@ -310,7 +329,7 @@ type listFn func(remote string, object *objectstorage.ObjectSummary, isDirectory
// If hidden is set then it will list the hidden (deleted) files too.
// if findFile is set it will look for files called (bucket, directory)
func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, limit int,
fn listFn) (err error) {
startAfter string, fn listFn) (err error) {
if prefix != "" {
prefix += "/"
}
@@ -336,6 +355,9 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
if delimiter != "" {
request.Delimiter = common.String(delimiter)
}
if startAfter != "" {
request.Start = common.String(startAfter)
}
for {
var resp objectstorage.ListObjectsResponse
@@ -442,27 +464,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *objectst
return o, nil
}
// listDir lists a single directory
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
fn := func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
if err != nil {
return err
}
if entry != nil {
return callback(entry)
}
return nil
}
err = f.list(ctx, bucket, directory, prefix, addBucket, false, 0, fn)
if err != nil {
return err
}
// bucket must be present if listing succeeded
f.cache.MarkOK(bucket)
return nil
}
// listBuckets returns all the buckets to out
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
if f.opt.Provider == noAuth {
@@ -718,11 +719,17 @@ immediately.
Don't implement this unless you have a more efficient way
of listing recursively that doing a directory traversal.
*/
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
bucketName, directory := f.split(dir)
startAfter, callback, done, err := resume.Setup(ctx, bucketName != "", f, dir, callback)
if err != nil {
return err
}
defer done(&returnErr)
list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, startAfter, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
if err != nil {
return err
@@ -749,7 +756,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
f.cache.MarkOK(bucketName)
}
} else {
err = listR(bucketName, directory, f.rootDirectory, f.rootBucket == "")
err := listR(bucketName, directory, f.rootDirectory, f.rootBucket == "")
if err != nil {
return err
}
+47 -34
View File
@@ -63,6 +63,7 @@ import (
"github.com/rclone/rclone/lib/pool"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/rest"
"github.com/rclone/rclone/lib/resume"
"github.com/rclone/rclone/lib/transferaccounter"
"github.com/rclone/rclone/lib/version"
)
@@ -1831,6 +1832,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
SetTier: provider.StorageClass.Len() > 0,
GetTier: provider.StorageClass.Len() > 0,
SlowModTime: true,
CanResumeListing: true,
}).Fill(ctx, f)
if opt.Provider == "AWS" {
f.features.DoubleSlash = true
@@ -2041,6 +2043,10 @@ func (f *Fs) newV1List(req *s3.ListObjectsV2Input) bucketLister {
// Convert v2 req into v1 req
//structs.SetFrom(&l.req, req)
setFrom_s3ListObjectsInput_s3ListObjectsV2Input(&l.req, req)
// Map v2 StartAfter to v1 Marker (same semantics: list after this key)
if req.StartAfter != nil {
l.req.Marker = req.StartAfter
}
return l
}
@@ -2306,6 +2312,7 @@ type listOpt struct {
versionAt fs.Time // if set only show versions <= this time
noSkipMarkers bool // if set return dir marker objects
restoreStatus bool // if set return restore status in listing too
startAfter string // if set, start listing strictly after this key
}
// list lists the objects into the function supplied with the opt
@@ -2351,6 +2358,9 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
if f.opt.RequesterPays {
req.RequestPayer = types.RequestPayerRequester
}
if opt.startAfter != "" {
req.StartAfter = &opt.startAfter
}
var listBucket bucketLister
switch {
case opt.withVersions || opt.versionAt.IsSet():
@@ -2517,35 +2527,6 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *types.Ob
return o, nil
}
// listDir lists files and directories to out
func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool, callback func(fs.DirEntry) error) (err error) {
// List the objects and directories
err = f.list(ctx, listOpt{
bucket: bucket,
directory: directory,
prefix: prefix,
addBucket: addBucket,
withVersions: f.opt.Versions,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil {
return err
}
if entry != nil {
return callback(entry)
}
return nil
})
if err != nil {
return err
}
// bucket must be present if listing succeeded
f.cache.MarkOK(bucket)
return nil
}
// listBuckets lists the buckets to out
func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
req := s3.ListBucketsInput{}
@@ -2592,9 +2573,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// These need not be returned in any particular order. If
// callback returns an error then the listing will stop
// immediately.
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
list := list.NewHelper(callback)
func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
bucket, directory := f.split(dir)
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
if err != nil {
return err
}
defer done(&returnErr)
list := list.NewHelper(callback)
if bucket == "" {
if directory != "" {
return fs.ErrorListBucketRequired
@@ -2610,10 +2597,29 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
}
}
} else {
err := f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", list.Add)
err := f.list(ctx, listOpt{
bucket: bucket,
directory: directory,
prefix: f.rootDirectory,
addBucket: f.rootBucket == "",
withVersions: f.opt.Versions,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
startAfter: startAfter,
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil {
return err
}
if entry != nil {
return list.Add(entry)
}
return nil
})
if err != nil {
return err
}
f.cache.MarkOK(bucket)
}
return list.Flush()
}
@@ -2634,8 +2640,14 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
//
// Don't implement this unless you have a more efficient way
// of listing recursively than doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
bucket, directory := f.split(dir)
startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
if err != nil {
return err
}
defer done(&returnErr)
list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, listOpt{
@@ -2647,6 +2659,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
withVersions: f.opt.Versions,
versionAt: f.opt.VersionAt,
hidden: f.opt.VersionDeleted,
startAfter: startAfter,
}, func(remote string, object *types.Object, versionID *string, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, versionID, isDirectory)
if err != nil {
@@ -2674,7 +2687,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
f.cache.MarkOK(bucket)
}
} else {
err = listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
err := listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
if err != nil {
return err
}
+44
View File
@@ -2546,6 +2546,50 @@ This sets the interval between each retry specified by `--retries`
The default is `0`. Use `0` to disable.
### --resume-listings string
Set this to a local directory path to enable resumable listings.
When a long-running listing operation (e.g. `ls`, `sync`, `copy`) is
interrupted, rclone saves checkpoint files in this directory so that
the next run can resume from where it left off rather than starting
from scratch.
This is useful when working with very large buckets containing
millions or billions of files, where listing alone can take hours or
days.
How it works:
- During listing, rclone saves a checkpoint after each page of results
from the backend.
- If the operation is interrupted (e.g. Ctrl-C, crash, reboot), the
checkpoint records how far the listing got.
- On the next run with the same `--resume-listings` directory, rclone
detects the checkpoint and resumes listing from that point.
- When a listing completes successfully, its checkpoint is
automatically deleted.
Only backends that support key-name-based pagination can resume
listings. Currently supported backends include S3, B2, and Oracle
Object Storage. Other backends will work normally but won't benefit
from resume checkpoints.
Resume works with both normal listings and `--fast-list` (ListR).
`--resume-listings` cannot be used with `sync` or other commands that
delete files from the destination. A resumed listing skips
already-listed source files, which would cause them to appear missing
and be deleted from the destination. Use `--resume-listings` with
`copy` or `ls` instead.
After a resumed run completes, it is recommended to do one full run
without `--resume-listings` to catch any files that were added in the
already-processed key range between runs.
```console
rclone copy --resume-listings /tmp/rclone-resume s3:big-bucket /local/dest
```
### --server-side-across-configs
Allow server-side operations (e.g. copy or move) to work across
+6
View File
@@ -277,6 +277,11 @@ var ConfigOptionsInfo = Options{{
Default: false,
Help: "Use recursive list if available; uses more memory but fewer transactions",
Groups: "Listing",
}, {
Name: "resume_listings",
Default: "",
Help: "Local directory path for saving listing checkpoints so interrupted listings can be resumed",
Groups: "Listing",
}, {
Name: "list_cutoff",
Default: 1_000_000,
@@ -617,6 +622,7 @@ type ConfigInfo struct {
Suffix string `config:"suffix"`
SuffixKeepExtension bool `config:"suffix_keep_extension"`
UseListR bool `config:"fast_list"`
ResumeListings string `config:"resume_listings"`
ListCutoff int `config:"list_cutoff"`
BufferSize SizeSuffix `config:"buffer_size"`
BwLimit BwTimetable `config:"bwlimit"`
+5
View File
@@ -174,6 +174,10 @@ type Features struct {
// immediately.
ListP func(ctx context.Context, dir string, callback ListRCallback) error
// CanResumeListing indicates the backend handles --resume-listings
// internally in its ListP/ListR implementations.
CanResumeListing bool
// About gets quota information from the Fs
About func(ctx context.Context) (*Usage, error)
@@ -456,6 +460,7 @@ func (ft *Features) Mask(ctx context.Context, f Fs) *Features {
if mask.ListP == nil {
ft.ListP = nil
}
ft.CanResumeListing = ft.CanResumeListing && mask.CanResumeListing
if mask.About == nil {
ft.About = nil
}
+3
View File
@@ -1341,6 +1341,9 @@ func runSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
if deleteMode != fs.DeleteModeOff && DoMove {
return fserrors.FatalError(errors.New("can't delete and move at the same time"))
}
if ci.ResumeListings != "" && deleteMode != fs.DeleteModeOff {
return fserrors.FatalError(errors.New("can't use --resume-listings with sync or other delete modes - a resumed listing skips already-listed files which would cause them to be deleted from the destination"))
}
// Run an extra pass to delete only
if deleteMode == fs.DeleteModeBefore {
if ci.TrackRenames {
+149
View File
@@ -38,6 +38,7 @@ import (
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
"github.com/rclone/rclone/lib/resume"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -1135,6 +1136,154 @@ func Run(t *testing.T, opt *Opt) {
fstest.CheckListing(t, f, []fstest.Item{file1, file2})
})
// TestFsCanResumeListing tests the resumable listing feature
t.Run("FsCanResumeListing", func(t *testing.T) {
skipIfNotOk(t)
if !f.Features().CanResumeListing {
t.Skip("FS does not support resumable listings")
}
require.NotNil(t, f.Features().ListP, "CanResumeListing backends must have ListP")
// Helper to collect all entries from ListP
listAll := func(ctx context.Context, dir string) fs.DirEntries {
t.Helper()
var entries fs.DirEntries
err := f.Features().ListP(ctx, dir, func(e fs.DirEntries) error {
entries = append(entries, e...)
return nil
})
require.NoError(t, err)
return entries
}
// List without resume - get baseline
baseEntries := listAll(ctx, "")
require.NotEmpty(t, baseEntries, "test requires existing files")
t.Run("CheckpointSavedAndCleaned", func(t *testing.T) {
// List with --resume-listings enabled
resumeDir := t.TempDir()
resumeCtx, resumeCi := fs.AddConfig(ctx)
resumeCi.ResumeListings = resumeDir
entries := listAll(resumeCtx, "")
assert.Equal(t, len(baseEntries), len(entries), "resumed listing should return same entries when no checkpoint exists")
// After successful completion, checkpoint should be cleaned up
dirEntries, err := os.ReadDir(resumeDir)
require.NoError(t, err)
// Filter out any non-json files
var jsonFiles []os.DirEntry
for _, e := range dirEntries {
if strings.HasSuffix(e.Name(), ".json") {
jsonFiles = append(jsonFiles, e)
}
}
assert.Empty(t, jsonFiles, "checkpoint should be deleted after successful listing")
})
t.Run("ResumeFromCheckpoint", func(t *testing.T) {
// Sort baseline entries to find a good startAfter key
var names []string
for _, e := range baseEntries {
names = append(names, e.Remote())
}
sort.Strings(names)
if len(names) < 2 {
t.Skip("need at least 2 entries to test resume")
}
// Pick the first name as startAfter - resumed listing should skip it
startAfter := names[0]
// Create a checkpoint that resumes after the first entry
resumeDir := t.TempDir()
store, err := resume.NewStore(resumeDir)
require.NoError(t, err)
remoteName := fs.ConfigString(f)
require.NoError(t, store.Save(&resume.Checkpoint{
RemoteName: remoteName,
Dir: "",
LastKey: startAfter,
}))
// List with resume - should skip entries <= startAfter
resumeCtx, resumeCi := fs.AddConfig(ctx)
resumeCi.ResumeListings = resumeDir
resumedEntries := listAll(resumeCtx, "")
var resumedNames []string
for _, e := range resumedEntries {
resumedNames = append(resumedNames, e.Remote())
}
// The resumed listing should have fewer entries
assert.Less(t, len(resumedEntries), len(baseEntries),
"resumed listing should have fewer entries than full listing")
// The startAfter entry should not appear in the resumed listing
assert.NotContains(t, resumedNames, startAfter,
"entry at checkpoint should not appear in resumed listing")
// All resumed entries should exist in the full listing
for _, name := range resumedNames {
assert.Contains(t, names, name,
"resumed entry %q should exist in full listing", name)
}
})
// Test ListR resume if available
if f.Features().ListR != nil {
t.Run("ListRResumeFromCheckpoint", func(t *testing.T) {
// Get baseline from ListR
var baseNames []string
err := f.Features().ListR(ctx, "", func(entries fs.DirEntries) error {
for _, e := range entries {
baseNames = append(baseNames, e.Remote())
}
return nil
})
require.NoError(t, err)
sort.Strings(baseNames)
if len(baseNames) < 2 {
t.Skip("need at least 2 entries to test ListR resume")
}
startAfter := baseNames[0]
// Create checkpoint
resumeDir := t.TempDir()
store, err := resume.NewStore(resumeDir)
require.NoError(t, err)
remoteName := fs.ConfigString(f)
require.NoError(t, store.Save(&resume.Checkpoint{
RemoteName: remoteName,
Dir: "",
LastKey: startAfter,
}))
// List with resume
resumeCtx, resumeCi := fs.AddConfig(ctx)
resumeCi.ResumeListings = resumeDir
var resumedNames []string
err = f.Features().ListR(resumeCtx, "", func(entries fs.DirEntries) error {
for _, e := range entries {
resumedNames = append(resumedNames, e.Remote())
}
return nil
})
require.NoError(t, err)
assert.Less(t, len(resumedNames), len(baseNames),
"resumed ListR should have fewer entries than full listing")
assert.NotContains(t, resumedNames, startAfter,
"entry at checkpoint should not appear in resumed ListR")
})
}
})
// TestFsNewObjectDir tests NewObject on a directory which should produce fs.ErrorIsDir if possible or fs.ErrorObjectNotFound if not
t.Run("FsNewObjectDir", func(t *testing.T) {
skipIfNotOk(t)
+214
View File
@@ -0,0 +1,214 @@
// Package resume manages checkpoint state files for resumable listings.
//
// When --resume-listings is set to a local directory path, rclone
// saves listing progress so that interrupted listings can be resumed
// from where they left off rather than starting from scratch.
package resume
import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/rclone/rclone/fs"
)
const checkpointVersion = 1
// Checkpoint represents the saved state of a listing in progress.
type Checkpoint struct {
Version int `json:"version"`
RemoteName string `json:"remoteName"` // e.g. "s3:mybucket/path"
Dir string `json:"dir"` // directory being listed
LastKey string `json:"lastKey"` // last key delivered to callback
UpdatedAt time.Time `json:"updatedAt"`
}
// Store manages checkpoint files in a local directory.
type Store struct {
dir string
}
// NewStore creates a Store that persists checkpoints in dir.
// The directory is created if it does not exist.
func NewStore(dir string) (*Store, error) {
if err := os.MkdirAll(dir, 0o777); err != nil {
return nil, fmt.Errorf("resume: create store directory: %w", err)
}
return &Store{dir: dir}, nil
}
// stateKey returns a deterministic file path for the given remote+dir.
func (s *Store) stateKey(remoteName, dir string) string {
h := sha256.Sum256([]byte(remoteName + ":" + dir))
return filepath.Join(s.dir, fmt.Sprintf("%x.json", h))
}
// Load reads a checkpoint for the given remote and directory.
// Returns nil, nil if no checkpoint exists or the checkpoint doesn't
// match (wrong version, different remote/dir).
func (s *Store) Load(remoteName, dir string) (*Checkpoint, error) {
path := s.stateKey(remoteName, dir)
data, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("resume: read checkpoint: %w", err)
}
var cp Checkpoint
if err := json.Unmarshal(data, &cp); err != nil {
return nil, fmt.Errorf("resume: unmarshal checkpoint: %w", err)
}
if cp.Version != checkpointVersion {
return nil, nil
}
if cp.RemoteName != remoteName || cp.Dir != dir {
return nil, nil
}
return &cp, nil
}
// Save atomically writes a checkpoint to disk.
func (s *Store) Save(cp *Checkpoint) error {
cp.Version = checkpointVersion
cp.UpdatedAt = time.Now().UTC()
payload, err := json.MarshalIndent(cp, "", " ")
if err != nil {
return fmt.Errorf("resume: marshal checkpoint: %w", err)
}
path := s.stateKey(cp.RemoteName, cp.Dir)
// Atomic write: temp file + rename
tmpFile, err := os.CreateTemp(s.dir, ".rclone-resume-*.tmp")
if err != nil {
return fmt.Errorf("resume: create temp file: %w", err)
}
tmpPath := tmpFile.Name()
_, writeErr := tmpFile.Write(payload)
closeErr := tmpFile.Close()
if writeErr != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("resume: write temp file: %w", writeErr)
}
if closeErr != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("resume: close temp file: %w", closeErr)
}
if err := os.Rename(tmpPath, path); err != nil {
_ = os.Remove(tmpPath)
return fmt.Errorf("resume: rename temp file: %w", err)
}
return nil
}
// WrapCallback loads any existing checkpoint for this remote+dir and returns:
// - startAfter: last key from a previous interrupted listing ("" if none)
// - wrappedCallback: tracks lastKey per page, saves checkpoint after each page
// - done: defer this — it deletes the checkpoint only if *errp == nil
func (s *Store) WrapCallback(remoteName, dir string, callback fs.ListRCallback) (
startAfter string, wrappedCallback fs.ListRCallback, done func(errp *error), err error) {
cp, err := s.Load(remoteName, dir)
if err != nil {
return "", nil, nil, fmt.Errorf("resume: load checkpoint: %w", err)
}
if cp != nil {
startAfter = cp.LastKey
fs.Infof(nil, "Resuming listing of %q from checkpoint %q", dir, startAfter)
}
var lastKey string
wrappedCallback = func(entries fs.DirEntries) error {
// Track the max key in this page for checkpointing
for _, entry := range entries {
key := entry.Remote()
if key > lastKey {
lastKey = key
}
}
// Pass entries through to the original callback
if err := callback(entries); err != nil {
return err
}
// Save checkpoint after each page
if lastKey != "" {
saveCP := &Checkpoint{
RemoteName: remoteName,
Dir: dir,
LastKey: lastKey,
}
if err := s.Save(saveCP); err != nil {
fs.Errorf(nil, "Failed to save listing checkpoint: %v", err)
}
}
return nil
}
done = func(errp *error) {
if *errp != nil {
return
}
if err := s.Delete(remoteName, dir); err != nil {
fs.Errorf(nil, "Failed to delete listing checkpoint: %v", err)
}
}
return startAfter, wrappedCallback, done, nil
}
// Setup is the top-level entry point for backends to add resume support.
//
// If --resume-listings is not configured or enabled is false, it returns
// passthrough values: startAfter="", the original callback, and a no-op done.
//
// Otherwise it creates a Store, loads any existing checkpoint, and wraps the
// callback to save progress after each page. done should be deferred — it
// deletes the checkpoint only when *errp == nil.
//
// Typical usage in a backend ListP/ListR:
//
// func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (returnErr error) {
// bucket, directory := f.split(dir)
// startAfter, callback, done, err := resume.Setup(ctx, bucket != "", f, dir, callback)
// if err != nil { return err }
// defer done(&returnErr)
// ...
// }
func Setup(ctx context.Context, enabled bool, f fs.Fs, dir string, callback fs.ListRCallback) (
startAfter string, wrappedCallback fs.ListRCallback, done func(errp *error), err error) {
ci := fs.GetConfig(ctx)
if !enabled || ci.ResumeListings == "" {
return "", callback, func(*error) {}, nil
}
store, err := NewStore(ci.ResumeListings)
if err != nil {
return "", nil, nil, err
}
return store.WrapCallback(fs.ConfigString(f), dir, callback)
}
// Delete removes the checkpoint for the given remote and directory.
// It is not an error if the checkpoint does not exist.
func (s *Store) Delete(remoteName, dir string) error {
path := s.stateKey(remoteName, dir)
err := os.Remove(path)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return fmt.Errorf("resume: delete checkpoint: %w", err)
}
return nil
}
+293
View File
@@ -0,0 +1,293 @@
package resume
import (
"context"
"testing"
"time"
_ "github.com/rclone/rclone/backend/memory"
"github.com/rclone/rclone/fs"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestSaveLoadRoundTrip(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
cp := &Checkpoint{
RemoteName: "s3:mybucket/path",
Dir: "subdir/",
LastKey: "subdir/file-1000",
}
require.NoError(t, store.Save(cp))
loaded, err := store.Load("s3:mybucket/path", "subdir/")
require.NoError(t, err)
require.NotNil(t, loaded)
assert.Equal(t, checkpointVersion, loaded.Version)
assert.Equal(t, "s3:mybucket/path", loaded.RemoteName)
assert.Equal(t, "subdir/", loaded.Dir)
assert.Equal(t, "subdir/file-1000", loaded.LastKey)
assert.False(t, loaded.UpdatedAt.IsZero())
}
func TestLoadNotFound(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
cp, err := store.Load("s3:mybucket", "")
require.NoError(t, err)
assert.Nil(t, cp)
}
func TestLoadRemoteMismatch(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
cp := &Checkpoint{
RemoteName: "s3:bucket-a",
Dir: "dir/",
LastKey: "dir/obj-500",
}
require.NoError(t, store.Save(cp))
// Different remote name - hash differs, so file not found
loaded, err := store.Load("s3:bucket-b", "dir/")
require.NoError(t, err)
assert.Nil(t, loaded)
}
func TestLoadDirMismatch(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
cp := &Checkpoint{
RemoteName: "s3:bucket",
Dir: "dir-a/",
LastKey: "dir-a/obj-500",
}
require.NoError(t, store.Save(cp))
// Different dir - hash differs, so file not found
loaded, err := store.Load("s3:bucket", "dir-b/")
require.NoError(t, err)
assert.Nil(t, loaded)
}
func TestDelete(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
cp := &Checkpoint{
RemoteName: "s3:bucket",
Dir: "",
LastKey: "obj-100",
}
require.NoError(t, store.Save(cp))
// Verify it exists
loaded, err := store.Load("s3:bucket", "")
require.NoError(t, err)
require.NotNil(t, loaded)
// Delete
require.NoError(t, store.Delete("s3:bucket", ""))
// Verify it's gone
loaded, err = store.Load("s3:bucket", "")
require.NoError(t, err)
assert.Nil(t, loaded)
}
func TestDeleteNonExistent(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
// Should not error
require.NoError(t, store.Delete("s3:bucket", "missing/"))
}
func TestWrapCallbackNoCheckpoint(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
var received fs.DirEntries
callback := func(entries fs.DirEntries) error {
received = append(received, entries...)
return nil
}
startAfter, wrapped, done, err := store.WrapCallback("s3:bucket", "dir", callback)
require.NoError(t, err)
assert.Equal(t, "", startAfter, "no checkpoint should give empty startAfter")
// Send a page of entries
entries := fs.DirEntries{
fs.NewDir("dir/aaa", time.Time{}),
fs.NewDir("dir/zzz", time.Time{}),
}
require.NoError(t, wrapped(entries))
assert.Len(t, received, 2, "entries should pass through")
// Checkpoint should exist now
cp, err := store.Load("s3:bucket", "dir")
require.NoError(t, err)
require.NotNil(t, cp)
assert.Equal(t, "dir/zzz", cp.LastKey)
// Call done with nil error - checkpoint should be deleted
var noErr error
done(&noErr)
cp, err = store.Load("s3:bucket", "dir")
require.NoError(t, err)
assert.Nil(t, cp)
}
func TestWrapCallbackWithCheckpoint(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
// Save an existing checkpoint
require.NoError(t, store.Save(&Checkpoint{
RemoteName: "s3:bucket",
Dir: "",
LastKey: "file-500",
}))
callback := func(entries fs.DirEntries) error { return nil }
startAfter, _, _, err := store.WrapCallback("s3:bucket", "", callback)
require.NoError(t, err)
assert.Equal(t, "file-500", startAfter, "should resume from checkpoint")
}
func TestWrapCallbackTracksMaxKey(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
callback := func(entries fs.DirEntries) error { return nil }
_, wrapped, _, err := store.WrapCallback("s3:bucket", "", callback)
require.NoError(t, err)
// First page
require.NoError(t, wrapped(fs.DirEntries{
fs.NewDir("bbb", time.Time{}),
fs.NewDir("aaa", time.Time{}),
}))
cp, err := store.Load("s3:bucket", "")
require.NoError(t, err)
require.NotNil(t, cp)
assert.Equal(t, "bbb", cp.LastKey, "should track max key across entries")
// Second page with higher keys
require.NoError(t, wrapped(fs.DirEntries{
fs.NewDir("ddd", time.Time{}),
fs.NewDir("ccc", time.Time{}),
}))
cp, err = store.Load("s3:bucket", "")
require.NoError(t, err)
require.NotNil(t, cp)
assert.Equal(t, "ddd", cp.LastKey, "should track max key across pages")
}
func TestSetupDisabled(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.ResumeListings = "" // not configured
called := false
callback := func(entries fs.DirEntries) error {
called = true
return nil
}
startAfter, wrapped, done, err := Setup(ctx, true, nil, "", callback)
require.NoError(t, err)
assert.Equal(t, "", startAfter)
// Should be the original callback (passthrough)
require.NoError(t, wrapped(nil))
assert.True(t, called)
// done should be safe to call
var noErr error
done(&noErr)
}
func TestSetupEnabledFalse(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.ResumeListings = t.TempDir()
callback := func(entries fs.DirEntries) error { return nil }
// enabled=false should return passthrough even with ResumeListings set
startAfter, _, done, err := Setup(ctx, false, nil, "", callback)
require.NoError(t, err)
assert.Equal(t, "", startAfter)
var noErr error
done(&noErr)
}
func TestSetupEnabled(t *testing.T) {
ctx := context.Background()
ctx, ci := fs.AddConfig(ctx)
ci.ResumeListings = t.TempDir()
memFs, err := fs.NewFs(ctx, ":memory:")
require.NoError(t, err)
remoteName := fs.ConfigString(memFs)
var received fs.DirEntries
callback := func(entries fs.DirEntries) error {
received = append(received, entries...)
return nil
}
startAfter, wrapped, done, err := Setup(ctx, true, memFs, "dir", callback)
require.NoError(t, err)
assert.Equal(t, "", startAfter, "no prior checkpoint")
// Send entries through wrapped callback
require.NoError(t, wrapped(fs.DirEntries{
fs.NewDir("dir/file1", time.Time{}),
}))
assert.Len(t, received, 1)
// Checkpoint should exist
store, err := NewStore(ci.ResumeListings)
require.NoError(t, err)
cp, err := store.Load(remoteName, "dir")
require.NoError(t, err)
require.NotNil(t, cp)
assert.Equal(t, "dir/file1", cp.LastKey)
// done() cleans up on success
var noErr error
done(&noErr)
cp, err = store.Load(remoteName, "dir")
require.NoError(t, err)
assert.Nil(t, cp)
}
func TestSaveOverwrites(t *testing.T) {
store, err := NewStore(t.TempDir())
require.NoError(t, err)
cp := &Checkpoint{
RemoteName: "s3:bucket",
Dir: "",
LastKey: "obj-100",
}
require.NoError(t, store.Save(cp))
cp.LastKey = "obj-200"
require.NoError(t, store.Save(cp))
loaded, err := store.Load("s3:bucket", "")
require.NoError(t, err)
require.NotNil(t, loaded)
assert.Equal(t, "obj-200", loaded.LastKey)
}