Version v1.75.2

golang.org/x/net v0.60.0 deprecates its HTTP/2 server and transport in
favour of net/http.

Make the h2c test use http.Transport with unencrypted HTTP/2. serve
restic --stdio still needs http2.Server to serve HTTP/2 on a single
connection as http.Server has no way of doing that.

(cherry picked from commit d3db33c6b7)

.github/workflows/build.yml

@@ -32,12 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '~1.26.5'
+            go: '~1.26.9'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -48,14 +48,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '~1.26.5'
+            go: '~1.26.9'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '~1.26.5'
+            go: '~1.26.9'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -64,14 +64,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '~1.26.5'
+            go: '~1.26.9'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '~1.26.5'
+            go: '~1.26.9'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '~1.26.5'
+            go: '~1.26.9'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.25
+          - job_name: go1.26
             os: ubuntu-latest
-            go: '~1.25.12'
+            go: '~1.26.9'
             quicktest: true
             racequicktest: true
 
@@ -179,6 +179,7 @@ jobs:
           unset HOMEBREW_NO_INSTALL_FROM_API
           brew untap --force homebrew/core
           brew untap --force homebrew/cask
+          brew untap --force aws/tap
           brew update
           brew install --cask macfuse
           brew install git-annex git-annex-remote-rclone
@@ -283,7 +284,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.26.5'
+          go-version: '~1.26.9'
           check-latest: true
           cache: false
           
@@ -398,7 +399,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.26.5'
+          go-version: '~1.26.9'
           # Caching is handled explicitly below to share the module cache
           # with the other jobs - see the build job for the rationale.
           cache: false

.golangci.yml

@@ -46,6 +46,11 @@ linters:
         - -ST1022
         # Disable quickfix checks
         - -QF*
+        # Disabled because its dataflow analysis (new in staticcheck
+        # 0.8.0) makes linting large packages more than 10x slower,
+        # which takes the CI lint job past its timeout, and it
+        # produces false positives (eg on operations.Delete).
+        - -SA4023
     gocritic:
       # With gocritic there are different settings, but since enabled-checks
       # and disabled-checks cannot both be set, for full customization the

CONTRIBUTING.md

@@ -630,10 +630,37 @@ remote or an fs.
 - **Remember** we have >50 backends to maintain so keeping them as similar as
   possible to each other is a high priority!
 
+### Managing memory
+
+Use rclone's memory pool for upload buffers rather than allocating them
+yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
+rclone's memory usage smaller.
+
+- If the provider supports multipart or parallel chunk uploads, implement
+  [fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
+  and drive it with
+  [multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
+  This gives you pooled chunk buffers, concurrency, retries and
+  `--multi-thread-streams` support for free. See s3/b2 backends for examples.
+- If you must buffer data yourself take the buffer from the global pool with
+  [multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
+  and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
+  be rewound for a retry.
+- When using a pooled buffer `Close()` it on all paths so its pages go back to
+  the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
+  attempt, set the request's `ContentLength` explicitly, and if the buffer
+  reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
+  in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
+  so the transport can't close it and free its pages before the retry.
+
 ### Unit tests
 
 - Create a config entry called `TestRemote` for the unit tests to use
 - Create a `backend/remote/remote_test.go` - copy and adjust your example remote
+- If your backend does chunked uploads, set `ChunkedUpload` in the
+  `fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
+  on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
+  `FsPutRetry` tests can run the chunked path with small chunks.
 - Make sure all tests pass with `go test -v`
 
 ### Integration tests

Dockerfile

@@ -39,6 +39,7 @@ RUN echo "**** Install Dependencies ****" && \
     apk add --no-cache \
         ca-certificates \
         fuse3 \
+        mailcap \
         tzdata && \
     echo "Enable user_allow_other in fuse" && \
     echo "user_allow_other" >> /etc/fuse.conf

Makefile

@@ -93,11 +93,15 @@ test:	rclone test_all
 	@echo "Written logs in test_all.log"
 
 # Quick test
+#
+# The timeout is raised above the go test default of 10m as the
+# cmd/gitannex end to end tests can take longer than that on slow CI
+# runners.
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
 
 racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
 
 compiletest:
 	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...

VERSION

@@ -1 +1 @@
-v1.75.0
+v1.75.2

backend/archive/archive.go

@@ -36,6 +36,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/sanitize"
 )
 
 // Register with Fs
@@ -185,8 +186,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 	foundArchive := subArchive(remote)
 	if foundArchive != nil {
 		fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
-		// Archive path
-		foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
+		// Archive path, in canonical form so that it compares equal
+		// to the cleaned entry names inside the archive
+		foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
+		if foundArchive.root == "." {
+			foundArchive.root = ""
+		}
 		// Path to the archive
 		archiveRemote := remote[:len(foundArchive.remote)]
 		// Remote is archive leaf name
@@ -491,24 +496,50 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return nil, err
 	}
 
-	entries, err = subFs.List(ctx, dir)
+	subEntries, err := subFs.List(ctx, dir)
 	if err != nil {
 		return nil, err
 	}
-	for i, entry := range entries {
+	entries = subEntries[:0]
+	skipped := 0
+	for _, entry := range subEntries {
+		remote := entry.Remote()
+		// Only pass on direct children of dir - anything else
+		// could escape the archive's namespace
+		if !isDirectChild(dir, remote) {
+			fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
+			skipped++
+			continue
+		}
 		// Can only unarchive files
 		if o, ok := entry.(fs.Object); ok {
-			remote := o.Remote()
 			archive := f.findArchive(remote)
 			if archive != nil {
 				// Overwrite entry with directory
-				entries[i] = fs.NewDir(remote, o.ModTime(ctx))
+				entry = fs.NewDir(remote, o.ModTime(ctx))
 			}
 		}
+		entries = append(entries, entry)
+	}
+	if skipped > 0 {
+		fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
 	}
 	return entries, nil
 }
 
+// isDirectChild reports whether remote names an entry directly in dir
+// ("" being the root), with no ".." or other components in between.
+func isDirectChild(dir, remote string) bool {
+	if dir != "" {
+		var ok bool
+		remote, ok = strings.CutPrefix(remote, dir+"/")
+		if !ok {
+			return false
+		}
+	}
+	return sanitize.Leaf(remote) == nil
+}
+
 // NewObject creates a new remote archive file object
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
@@ -526,6 +557,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Don't trust the archiver to have returned the object asked for
+	if o.Remote() != remote {
+		fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
+		return nil, fs.ErrorObjectNotFound
+	}
 	return o, nil
 }
 

backend/archive/archive_internal_test.go

@@ -3,6 +3,7 @@
 package archive
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"fmt"
@@ -15,6 +16,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/rclone/rclone/backend/archive/archiver"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
@@ -173,6 +175,18 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
 	checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
 
 	// Now check a single file
+	//
+	// The filter below applies to both sides, so check first that the
+	// archive exposes only that file without it
+	Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
+	require.NoError(t, err)
+	entries, err := Fsingle.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	_, err = Fsingle.List(ctx, subDir)
+	assert.Equal(t, fs.ErrorDirNotFound, err)
+	_, err = Fsingle.NewObject(ctx, subDir)
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
 	fiCtx, fi := filter.AddConfig(ctx)
 	require.NoError(t, fi.AddRule("+ "+aFile))
 	require.NoError(t, fi.AddRule("- *"))
@@ -277,3 +291,127 @@ func TestArchiveSquashfsIssue9004(t *testing.T) {
 		assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
 	})
 }
+
+// TestArchiveUncleanRoot checks that a path into an archive which isn't
+// in canonical form (with "./" or doubled slashes) still finds its
+// directory.
+func TestArchiveUncleanRoot(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create("sub/dir/a.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	zipPath := filepath.Join(t.TempDir(), "test.zip")
+	require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
+
+	for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
+		t.Run(root, func(t *testing.T) {
+			f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
+			require.NoError(t, err)
+			entries, err := f.List(ctx, "")
+			require.NoError(t, err)
+			require.Len(t, entries, 1)
+			assert.Equal(t, "a.txt", entries[0].Remote())
+		})
+	}
+}
+
+// escapingObject is an object whose remote is not where it was asked for.
+type escapingObject struct {
+	fs.Object
+	remote string
+}
+
+func (o *escapingObject) Remote() string { return o.remote }
+func (o *escapingObject) String() string { return o.remote }
+
+// escapingFs stands in for a badly behaved archiver which exposes entry
+// names outside the directory being listed.
+type escapingFs struct {
+	fs.Fs
+	prefix string
+}
+
+func (f *escapingFs) Name() string           { return "escaping" }
+func (f *escapingFs) Root() string           { return "" }
+func (f *escapingFs) String() string         { return "escaping" }
+func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
+
+func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
+	return fs.DirEntries{
+		&escapingObject{remote: path.Join(dir, "good.txt")},
+		fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
+		&escapingObject{remote: path.Join(dir, "../escape.txt")},
+		&escapingObject{remote: "../../escape.txt"},
+		&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
+		fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
+	}, nil
+}
+
+func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	return &escapingObject{remote: "../escape.txt"}, nil
+}
+
+// TestArchiveEscapingArchiver checks that the archive backend does not
+// pass on entries from an archiver which escape the directory being
+// listed, whatever the archiver does.
+func TestArchiveEscapingArchiver(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	archiver.Register(archiver.Archiver{
+		New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
+			return &escapingFs{prefix: prefix}, nil
+		},
+		Extension: ".escaping",
+	})
+
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
+	f, err := cache.Get(ctx, ":archive:"+dir)
+	require.NoError(t, err)
+
+	// Archives are discovered when their parent directory is listed
+	_, err = f.List(ctx, "")
+	require.NoError(t, err)
+
+	entries, err := f.List(ctx, "test.escaping")
+	require.NoError(t, err)
+	var remotes []string
+	for _, entry := range entries {
+		remotes = append(remotes, entry.Remote())
+	}
+	assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
+
+	_, err = f.NewObject(ctx, "test.escaping/file.txt")
+	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}
+
+// TestIsDirectChild checks the guard which decides whether an entry
+// returned by an archiver belongs directly in the directory listed.
+func TestIsDirectChild(t *testing.T) {
+	for _, test := range []struct {
+		dir, remote string
+		want        bool
+	}{
+		{"", "a.txt", true},
+		{"", "/a.txt", false},
+		{"", "a.txt/", false},
+		{"", "../a.txt", false},
+		{"", "sub/a.txt", false},
+		{"d", "d/a.txt", true},
+		{"d", "d", false},
+		{"d", "d/", false},
+		{"d", "d//a.txt", false},
+		{"d", "d/../a.txt", false},
+		{"d", "dd/a.txt", false},
+		{"d", "a.txt", false},
+	} {
+		assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
+	}
+}

backend/archive/archiver/archiver.go

@@ -8,6 +8,11 @@ import (
 )
 
 // Archiver describes an archive package
+//
+// Entry names inside an archive are attacker controlled. An archiver
+// must not expose an entry whose remote escapes the archive's own
+// namespace (for example one with a ".." component), typically by
+// validating names with lib/sanitize.
 type Archiver struct {
 	// New constructs an Fs from the (wrappedFs, remote) with the objects
 	// prefix with prefix and rooted at root

backend/archive/squashfs/squashfs.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/sanitize"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -41,6 +42,7 @@ type Fs struct {
 	prefix      string   // position for objects
 	prefixSlash string   // position for objects with a slash on
 	root        string   // position to read from within the archive
+	file        string   // remote of the single file root points at, if any
 }
 
 // recoverParsePanic converts a panic raised while parsing a squashfs image
@@ -114,10 +116,13 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
 			_, err := f.newObjectNative(native)
 			if err == nil {
 				// If it pointed to a file, find the directory above
+				// and remember the file so that only it is listed
 				f.root = path.Dir(f.root)
 				if f.root == "." || f.root == "/" {
 					f.root = ""
 				}
+				f.file = f.prefixSlash + path.Base(native)
+				singleObject = true
 			}
 		}
 	}
@@ -149,7 +154,9 @@ func (f *Fs) Name() string {
 
 // Root of the remote (as passed into NewFs)
 func (f *Fs) Root() string {
-	return f.root
+	// Include the file so the fs cache can tell this Fs apart from
+	// the one for the directory containing it
+	return path.Join(f.root, f.file)
 }
 
 // Features returns the optional features of this Fs
@@ -226,6 +233,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
 	defer recoverParsePanic(&err)
 
+	if f.file != "" && dir != "" {
+		return nil, fs.ErrorDirNotFound
+	}
+
 	nativeDir, err := f.toNative(dir)
 	if err != nil {
 		return nil, err
@@ -237,8 +248,15 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	entries = make(fs.DirEntries, 0, len(items))
+	skipped := 0
 	for _, item := range items {
 		// fs.Debugf(item.Name(), "entry = %#v", item)
+		// Skip entry names that aren't safe
+		if err := sanitize.Leaf(item.Name()); err != nil {
+			fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
+			skipped++
+			continue
+		}
 		var entry fs.DirEntry
 		if item.IsDir() {
 			remote := f.fromNative(nativeDir, item.Name())
@@ -257,9 +275,18 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
 			continue
 		}
+		if f.file != "" && entry.Remote() != f.file {
+			continue
+		}
 		entries = append(entries, entry)
 	}
 
+	// Log once per listing rather than per entry so a crafted image
+	// can't flood the log
+	if skipped > 0 {
+		fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
+	}
+
 	// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
 	return entries, nil
 }
@@ -302,6 +329,9 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
 
+	if f.file != "" && remote != f.file {
+		return nil, fs.ErrorObjectNotFound
+	}
 	nativePath, err := f.toNative(remote)
 	if err != nil {
 		return nil, err

backend/archive/squashfs/squashfs_internal_test.go

@@ -0,0 +1,32 @@
+package squashfs
+
+import (
+	"context"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	fscache "github.com/rclone/rclone/fs/cache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A backslash is an ordinary character in a file name on the systems
+// squashfs images are made on, so an entry containing one must be
+// listed and openable like any other file.
+func TestBackslashInName(t *testing.T) {
+	ctx := context.Background()
+	localFs, err := fscache.Get(ctx, "testdata")
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, "backslash.sqfs", "", "")
+	require.NoError(t, err)
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	assert.Equal(t, `back\slash.txt`, entries[0].Remote())
+
+	o, err := f.NewObject(ctx, `back\slash.txt`)
+	require.NoError(t, err)
+	assert.Equal(t, int64(6), o.Size())
+}

backend/archive/zip/zip.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"slices"
 	"strings"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/sanitize"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -135,15 +137,27 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 		return singleObject, fmt.Errorf("failed to read zip file: %w", err)
 	}
 	dt := dirtree.New()
+	skipped := 0
 	for _, file := range zr.File {
-		remote := strings.Trim(path.Clean(file.Name), "/")
-		if remote == "." {
-			remote = ""
+		// Skip entries whose name escapes the archive's own namespace
+		remote, err := sanitize.Path(file.Name)
+		if err != nil {
+			skipped++
+			continue
+		}
+		isDir := strings.HasSuffix(file.Name, "/")
+		// A file entry whose last component is "", "." or ".." names
+		// a directory rather than a file, so skip
+		if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
+			skipped++
+			continue
 		}
 		remote = path.Join(f.prefix, remote)
 		if f.root != "" {
-			// Ignore all files outside the root
-			if !strings.HasPrefix(remote, f.root) {
+			// Ignore all files outside the root, requiring a path
+			// boundary so that root "foo" does not also match a
+			// sibling entry such as "foobar"
+			if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
 				continue
 			}
 			if remote == f.root {
@@ -152,11 +166,12 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 				remote = strings.TrimPrefix(remote, f.root+"/")
 			}
 		}
-		if strings.HasSuffix(file.Name, "/") {
+		if isDir {
 			dir := fs.NewDir(remote, file.Modified)
 			dt.AddDir(dir)
 		} else {
 			if remote == "" {
+				// A file at the root is the archive member f.root points at
 				remote = path.Base(f.root)
 				singleObject = true
 				dt = dirtree.New()
@@ -173,6 +188,9 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 			}
 		}
 	}
+	if skipped > 0 {
+		fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
+	}
 	dt.CheckParents("")
 	dt.Sort()
 	f.dt = dt
@@ -200,7 +218,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return nil, fs.ErrorDirNotFound
 	}
 	fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
-	return entries, nil
+	// Return a copy as callers may filter the listing in place
+	return slices.Clone(entries), nil
 }
 
 // NewObject finds the Object at remote.

backend/archive/zip/zip_internal_test.go

@@ -0,0 +1,172 @@
+package zip
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writeZip builds a zip file with the given entry names (a name ending
+// in "/" is a directory) in dir and returns its leaf name.
+func writeZip(t *testing.T, dir, name string, names ...string) string {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, n := range names {
+		w, err := zw.Create(n)
+		require.NoError(t, err)
+		if !strings.HasSuffix(n, "/") {
+			_, err = w.Write([]byte("data for " + n))
+			require.NoError(t, err)
+		}
+	}
+	require.NoError(t, zw.Close())
+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
+	return name
+}
+
+// allRemotes returns every Object remote in the mounted archive's dirtree.
+func allRemotes(t *testing.T, f fs.Fs) []string {
+	zf, ok := f.(*Fs)
+	require.True(t, ok)
+	var remotes []string
+	for _, entries := range zf.dt {
+		for _, entry := range entries {
+			if o, ok := entry.(*Object); ok {
+				remotes = append(remotes, o.Remote())
+			}
+		}
+	}
+	// dt iteration order is nondeterministic, so sort for stable comparison.
+	sort.Strings(remotes)
+	return remotes
+}
+
+// A malicious zip whose entry names escape the archive's own namespace
+// must not be exposed by the zip backend (Zip Slip).
+func TestReadZipSlip(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "evil.zip",
+		"good.txt",
+		"../../etc/cron.d/evil",
+		"../escape.txt",
+		"sub/../../above.txt",
+		`..\..\evil.exe`,
+	)
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+
+	remotes := allRemotes(t, f)
+
+	// The one benign entry must survive.
+	assert.Contains(t, remotes, "good.txt")
+
+	// No escaping entry may be exposed.
+	for _, remote := range remotes {
+		assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
+			"escaping remote exposed: %q", remote)
+	}
+	assert.Equal(t, []string{"good.txt"}, remotes)
+}
+
+// Mounting with a non-empty root must only expose entries within that
+// root directory, not sibling directories that merely share a name
+// prefix (root "foo" must not match "foobar").
+func TestReadZipRootBoundary(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "test.zip",
+		"foo/a.txt",
+		"foobar/b.txt",
+	)
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, name, "", "foo")
+	require.NoError(t, err)
+
+	remotes := allRemotes(t, f)
+	assert.Equal(t, []string{"a.txt"}, remotes)
+}
+
+// A file entry whose name refers to a directory (".", "/", "./", "" or
+// "sub/.") must be skipped, not turn the archive or the directory
+// mounted as its root into a single file which hides every other
+// entry, or become an object named for the archive itself when the
+// archive is found by listing its directory.
+func TestReadZipDirNamedFileEntry(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "dot.zip",
+		".",
+		"/",
+		"./",
+		"",
+		"good.txt",
+		"sub/",
+		"sub/.",
+		"sub/good.txt",
+	)
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	for _, test := range []struct {
+		prefix, root string
+		want         []string
+	}{
+		{"", "", []string{"good.txt", "sub/good.txt"}},
+		{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
+		{"", "sub", []string{"good.txt"}},
+	} {
+		t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
+			f, err := New(ctx, localFs, name, test.prefix, test.root)
+			require.NoError(t, err)
+			assert.Equal(t, test.want, allRemotes(t, f))
+		})
+	}
+}
+
+// Listings are served from a cache which must survive callers
+// filtering the returned slice in place, as fs/list does.
+func TestListCacheNotAliased(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+	f, err := New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 3)
+	// Compact in place, dropping the first entry
+	copy(entries, entries[1:])
+
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	var remotes []string
+	for _, entry := range entries {
+		remotes = append(remotes, entry.Remote())
+	}
+	assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
+}

backend/azureblob/arrow.go

@@ -18,24 +18,20 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/fs"
 	"golang.org/x/sync/errgroup"
 )
 
+// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
+// response, used to tell whether the service honoured an Arrow request.
+const arrowContentType = "application/vnd.apache.arrow.stream"
+
 // listArrowParallel lists (containerName, directory) by splitting the blob name
 // keyspace into shards and listing them concurrently. opts must already request
 // the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
 // returns the number of raw items seen. directory/prefix are already normalised
 // by the caller.
-func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
-	// Parallel listing needs the arrowlist client (server-side endBefore is
-	// only honoured on the Arrow path) - without it fall back to sequential.
-	if _, err := f.arrowCntSVC(containerName); err != nil {
-		fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
-		return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
-	}
-
+func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
 	// Split the keyspace by a single-case character ladder (see
 	// arrowLadderBoundaries for why single-case matters).
 	points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
@@ -115,16 +111,13 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
 // isEndBeforeUnsupported reports whether err means the service can't honour
 // the endBefore listing bound, so sharded parallel listing must not be used.
 //
-// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
-// server answered XML with endBefore set), or the service rejecting the
-// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
-// requested operation is not allowed as EndBefore parameter support is
-// missing on this account" - endBefore is only supported on the Arrow
-// listing path of accounts with Blob Listing with Apache Arrow enabled.
+// The service rejects the endBefore parameter with 400
+// OperationNotSupportedWithFeatureMissing "The requested operation is not
+// allowed as EndBefore parameter support is missing on this account" -
+// endBefore is only supported on the Arrow listing path of accounts with
+// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
+// accept an XML page with endBefore set, but the service never sends one.)
 func isEndBeforeUnsupported(err error) bool {
-	if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
-		return true
-	}
 	var respErr *azcore.ResponseError
 	return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
 }

backend/azureblob/arrow_internal_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -212,9 +211,8 @@ func TestArrowLadderBoundaries(t *testing.T) {
 }
 
 func TestIsEndBeforeUnsupported(t *testing.T) {
-	assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
-	assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
 	assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
+	assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
 	assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
 	assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
 	assert.False(t, isEndBeforeUnsupported(nil))

backend/azureblob/arrowlist/arrow.go

@@ -1,421 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
-// public type aliases so it compiles against the released azblob module.
-
-package arrowlist
-
-import (
-	"encoding/base64"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
-	"github.com/apache/arrow-go/v18/arrow"
-	"github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/apache/arrow-go/v18/arrow/ipc"
-	"github.com/apache/arrow-go/v18/arrow/memory"
-)
-
-const (
-	// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
-	ArrowContentType = "application/vnd.apache.arrow.stream"
-
-	// ArrowAcceptHeader is the Accept header value to request Arrow format.
-	ArrowAcceptHeader = ArrowContentType
-
-	// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
-	// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
-	// in Arrow, all rows share the same schema and are distinguished by this field value.
-	resourceTypeBlobPrefix = "blobprefix"
-)
-
-// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
-func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
-	result := container.ListBlobsFlatResponse{}
-
-	// Extract response headers
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
-	// Parse Arrow IPC stream
-	items, nextMarker, err := parseArrowStream(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
-	}
-
-	result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
-		Segment: &container.BlobFlatListSegment{
-			BlobItems: items,
-		},
-		NextMarker: nextMarker,
-	}
-
-	return result, nil
-}
-
-// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
-func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
-	result := container.ListBlobsHierarchyResponse{}
-
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
-	items, nextMarker, err := parseArrowStream(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
-	}
-
-	// Separate blob items from blob prefixes based on ResourceType field.
-	var blobItems []*container.BlobItem
-	var blobPrefixes []*container.BlobPrefix
-	for _, item := range items {
-		if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
-			blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
-				Name:       item.Name,
-				Properties: item.Properties,
-			})
-		} else {
-			blobItems = append(blobItems, item)
-		}
-	}
-
-	result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
-		Segment: &container.BlobHierarchyListSegment{
-			BlobItems:    blobItems,
-			BlobPrefixes: blobPrefixes,
-		},
-		NextMarker: nextMarker,
-	}
-
-	return result, nil
-}
-
-func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
-	if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
-		*clientRequestID = &val
-	}
-	if val := resp.Header.Get("Content-Type"); val != "" {
-		*contentType = &val
-	}
-	if val := resp.Header.Get("Date"); val != "" {
-		if t, err := time.Parse(time.RFC1123, val); err == nil {
-			*date = &t
-		}
-	}
-	if val := resp.Header.Get("x-ms-request-id"); val != "" {
-		*requestID = &val
-	}
-	if val := resp.Header.Get("x-ms-version"); val != "" {
-		*version = &val
-	}
-}
-
-// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
-func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
-	reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
-	}
-	defer reader.Release()
-
-	// Extract NextMarker from schema metadata
-	var nextMarker *string
-	var numRecords int
-	if md := reader.Schema().Metadata(); md.Len() > 0 {
-		if idx := md.FindKey("NextMarker"); idx >= 0 {
-			val := md.Values()[idx]
-			if val != "" {
-				nextMarker = &val
-			}
-		}
-		if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
-			if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
-				numRecords = n
-			}
-		}
-	}
-
-	// Build column index for the schema
-	schema := reader.Schema()
-	colIndex := buildColumnIndex(schema)
-
-	// Pre-allocate if we know the count
-	items := make([]*container.BlobItem, 0, numRecords)
-
-	// Iterate over record batches
-	for reader.Next() {
-		rec := reader.RecordBatch()
-		rows := int(rec.NumRows())
-
-		for row := 0; row < rows; row++ {
-			item := &container.BlobItem{
-				Properties: &container.BlobProperties{},
-			}
-			populateBlobItem(item, rec, row, colIndex)
-			items = append(items, item)
-		}
-	}
-
-	if err := reader.Err(); err != nil {
-		return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
-	}
-
-	return items, nextMarker, nil
-}
-
-// columnIndex maps column names to their indices in the record batch for O(1) lookup.
-type columnIndex map[string]int
-
-func buildColumnIndex(schema *arrow.Schema) columnIndex {
-	idx := make(columnIndex, len(schema.Fields()))
-	for i, f := range schema.Fields() {
-		idx[f.Name] = i
-	}
-	return idx
-}
-
-// populateBlobItem fills a BlobItem from a single row across all columns.
-func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
-	// Top-level BlobItem fields
-	item.Name = getStringField(rec, row, colIdx, "Name")
-	item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
-	item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
-	item.VersionID = getStringField(rec, row, colIdx, "VersionId")
-	item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
-	item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
-
-	// Metadata maps
-	item.Metadata = getMapField(rec, row, colIdx, "Metadata")
-	item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
-
-	// Tags
-	if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
-		var blobTags []*container.BlobTag
-		for k, v := range tags {
-			key := k
-			blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
-		}
-		item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
-	}
-
-	// BlobProperties fields
-	p := item.Properties
-	p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
-	p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
-	p.ETag = getETagField(rec, row, colIdx, "Etag")
-	p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
-	p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
-	p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
-	p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
-	p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
-	p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
-	p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
-	p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
-	p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
-	p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
-	p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
-	p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
-	p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
-	p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
-	p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
-	p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
-	p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
-	p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
-	p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
-	p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
-	p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
-	p.CopyID = getStringField(rec, row, colIdx, "CopyId")
-	p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
-	p.CopySource = getStringField(rec, row, colIdx, "CopySource")
-	p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
-	p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
-	p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
-	p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
-	p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
-	p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
-	p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
-	p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
-	p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
-	p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
-	p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
-	p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
-	p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
-
-	// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
-	// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
-	// It is silently ignored for forward compatibility.
-}
-
-// Field accessor helpers that safely handle missing columns and null values.
-
-func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.String); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Boolean); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Timestamp); ok {
-		val := arr.Value(row)
-		// Arrow timestamps are stored as int64 with a unit; convert to time.Time
-		dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
-		return &dt
-	}
-	return nil
-}
-
-func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Uint64); ok {
-		val := int64(arr.Value(row))
-		return &val
-	}
-	if arr, ok := col.(*array.Int64); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Uint64); ok {
-		val := int32(arr.Value(row))
-		return &val
-	}
-	if arr, ok := col.(*array.Int32); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	etag := azcore.ETag(*s)
-	return &etag
-}
-
-func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	decoded, err := base64.StdEncoding.DecodeString(*s)
-	if err != nil {
-		return nil
-	}
-	return decoded
-}
-
-// getEnumField returns a pointer to a string-typed enum value.
-type stringEnum interface {
-	~string
-}
-
-func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	val := T(*s)
-	return &val
-}
-
-func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	mapArr, ok := col.(*array.Map)
-	if !ok {
-		return nil
-	}
-
-	// Get the offsets for this row's map entries
-	start := int(mapArr.Offsets()[row])
-	end := int(mapArr.Offsets()[row+1])
-	if start == end {
-		return nil
-	}
-
-	keys := mapArr.Keys().(*array.String)
-	values := mapArr.Items().(*array.String)
-
-	result := make(map[string]*string, end-start)
-	for i := start; i < end; i++ {
-		k := keys.Value(i)
-		if values.IsNull(i) {
-			result[k] = nil
-		} else {
-			v := values.Value(i)
-			result[k] = &v
-		}
-	}
-	return result
-}

backend/azureblob/arrowlist/arrow_test.go

@@ -1,374 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost).
-
-package arrowlist
-
-import (
-	"bytes"
-	"io"
-	"net/http"
-	"testing"
-
-	"github.com/apache/arrow-go/v18/arrow"
-	arrowArray "github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/apache/arrow-go/v18/arrow/ipc"
-	"github.com/apache/arrow-go/v18/arrow/memory"
-	"github.com/stretchr/testify/require"
-)
-
-// coreSchema returns a basic Arrow schema for testing with common blob fields.
-func coreSchema(md *arrow.Metadata) *arrow.Schema {
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
-		{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
-		{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
-		{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	return arrow.NewSchema(fields, md)
-}
-
-// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
-func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
-	t.Helper()
-	alloc := memory.DefaultAllocator
-
-	var buf bytes.Buffer
-	w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
-	builder := arrowArray.NewRecordBuilder(alloc, schema)
-	defer builder.Release()
-
-	populate(builder)
-
-	rec := builder.NewRecordBatch()
-	defer rec.Release()
-
-	require.NoError(t, w.Write(rec))
-	require.NoError(t, w.Close())
-
-	return buf.Bytes()
-}
-
-func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
-	h := http.Header{}
-	h.Set("Content-Type", ArrowContentType)
-	for k, v := range headers {
-		h.Set(k, v)
-	}
-	return &http.Response{
-		StatusCode: http.StatusOK,
-		Header:     h,
-		Body:       io.NopCloser(bytes.NewReader(data)),
-	}
-}
-
-func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
-		b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
-		b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
-		b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
-		b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
-		b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
-		b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
-		b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
-		b.Field(9).(*arrowArray.StringBuilder).Append("available")
-		b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
-		b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
-		b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
-		b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
-		b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
-		b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
-		b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
-		b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
-		b.Field(9).(*arrowArray.StringBuilder).Append("available")
-		b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	require.NotNil(t, result.NextMarker)
-	require.Equal(t, "marker123", *result.NextMarker)
-	require.Len(t, result.Segment.BlobItems, 2)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "blob1.txt", *item.Name)
-	require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
-	require.Equal(t, int64(1024), *item.Properties.ContentLength)
-	require.Equal(t, "Hot", string(*item.Properties.AccessTier))
-}
-
-func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		// No rows added
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Nil(t, result.NextMarker)
-	require.Empty(t, result.Segment.BlobItems)
-}
-
-func TestHandleFlatListResponse_NullFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
-		b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
-		b.Field(3).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(4).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
-		b.Field(6).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
-		b.Field(8).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(9).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(10).(*arrowArray.StringBuilder).AppendNull()
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "blob.txt", *item.Name)
-	require.Nil(t, item.Properties.CreationTime)
-	require.Nil(t, item.Properties.BlobType)
-	require.Nil(t, item.Properties.ETag)
-	require.Nil(t, item.Properties.ContentLength)
-	require.Nil(t, item.Properties.ContentType)
-	require.Nil(t, item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-	require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
-}
-
-func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
-	alloc := memory.DefaultAllocator
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
-	schema := coreSchema(&md)
-
-	var buf bytes.Buffer
-	w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
-	// First batch: 2 rows
-	builder := arrowArray.NewRecordBuilder(alloc, schema)
-	for i := 0; i < 2; i++ {
-		builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
-		for j := 1; j < 11; j++ {
-			builder.Field(j).AppendNull()
-		}
-	}
-	rec := builder.NewRecordBatch()
-	require.NoError(t, w.Write(rec))
-	rec.Release()
-	builder.Release()
-
-	// Second batch: 1 row
-	builder = arrowArray.NewRecordBuilder(alloc, schema)
-	builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
-	for j := 1; j < 11; j++ {
-		builder.Field(j).AppendNull()
-	}
-	rec = builder.NewRecordBatch()
-	require.NoError(t, w.Write(rec))
-	rec.Release()
-	builder.Release()
-
-	require.NoError(t, w.Close())
-
-	resp := makeHTTPResponse(buf.Bytes(), nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 3)
-}
-
-func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		// A regular blob
-		b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
-		b.Field(1).(*arrowArray.StringBuilder).AppendNull()
-
-		// A blob prefix (virtual directory)
-		b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
-		b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleHierarchyListResponse(resp)
-	require.NoError(t, err)
-
-	require.Len(t, result.Segment.BlobItems, 1)
-	require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
-
-	require.Len(t, result.Segment.BlobPrefixes, 1)
-	require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
-}
-
-func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-	})
-
-	resp := makeHTTPResponse(data, map[string]string{
-		"x-ms-request-id":        "req-123",
-		"x-ms-version":           "2026-10-06",
-		"x-ms-client-request-id": "client-req-456",
-		"Date":                   "Mon, 01 Jan 2024 00:00:00 GMT",
-	})
-
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Equal(t, "req-123", *result.RequestID)
-	require.Equal(t, "2026-10-06", *result.Version)
-	require.Equal(t, "client-req-456", *result.ClientRequestID)
-	require.NotNil(t, result.Date)
-}
-
-func TestHandleFlatListResponse_MapFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
-		{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-
-		metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
-		metaBuilder.Append(true)
-		metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
-		metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
-
-		tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
-		tagBuilder.Append(true)
-		tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
-		tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-
-	item := result.Segment.BlobItems[0]
-	require.NotNil(t, item.Metadata)
-	require.Equal(t, "value1", *item.Metadata["key1"])
-
-	require.NotNil(t, item.BlobTags)
-	require.Len(t, item.BlobTags.BlobTagSet, 1)
-}
-
-func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
-		b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
-		b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	item := result.Segment.BlobItems[0]
-	require.True(t, *item.Deleted)
-	require.False(t, *item.IsCurrentVersion)
-	require.True(t, *item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
-		b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
-		b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
-		b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
-	require.Equal(t, "snap-123", *item.Snapshot)
-	require.True(t, *item.IsCurrentVersion)
-}

backend/azureblob/arrowlist/arrowlist.go

@@ -1,259 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Package arrowlist implements the experimental "Blob Listing with Apache
-// Arrow" feature on top of the released Azure azblob SDK.
-//
-// The Azure SDK for Go has support for Arrow listing on the unreleased
-// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
-// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
-// response types. Until that ships in a tagged azblob release this package
-// fills the gap: it exposes the same options and pager interface as the
-// experimental SDK, built only on the public API of the released SDK plus
-// copies of two small internal auth policies (see sharedkey.go and
-// challenge_policy.go).
-//
-// This package is temporary. When Arrow listing ships in a released azblob
-// the whole package should be deleted and its callers pointed back at
-// container.Client.NewListBlobsHierarchyPager - the option and response
-// types here are deliberately source compatible with the SDK's to make that
-// a mechanical change.
-package arrowlist
-
-import (
-	"context"
-	"errors"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-)
-
-const (
-	// moduleName and moduleVersion identify this package in the azcore
-	// telemetry policy's User-Agent fragment.
-	moduleName    = "github.com/rclone/rclone/backend/azureblob/arrowlist"
-	moduleVersion = "v0.0.1"
-
-	// serviceVersion is the x-ms-version sent with listing requests. Arrow
-	// responses and the startFrom/endBefore parameters need at least
-	// 2026-06-06.
-	serviceVersion = "2026-06-06"
-
-	// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
-	// own clients.
-	tokenScope = "https://storage.azure.com/.default"
-)
-
-// ErrEndBeforeXMLFallback is returned by the pager when the server answered
-// with XML instead of Arrow while EndBefore was set. The service only honours
-// endBefore on the Arrow listing path, so an XML page could silently contain
-// blobs outside the requested range and cannot be used.
-var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
-
-// ListBlobsHierarchyOptions provides the configuration for
-// Client.NewListBlobsHierarchyPager.
-//
-// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
-// the released SDK's options are embedded (providing Include, Marker,
-// MaxResults, Prefix and StartFrom by promotion) with the two fields the
-// released SDK lacks added alongside.
-type ListBlobsHierarchyOptions struct {
-	container.ListBlobsHierarchyOptions
-	// EndBefore limits listing to blobs whose full container path is
-	// lexically before it (exclusive). Only honoured on Arrow responses.
-	EndBefore *string
-	// UseArrowFormat requests the response as an Apache Arrow IPC stream.
-	UseArrowFormat *bool
-}
-
-// ClientOptions contains the optional parameters for client creation.
-type ClientOptions struct {
-	azcore.ClientOptions
-}
-
-// Client issues Arrow listing requests to a single container.
-type Client struct {
-	endpoint string
-	azClient *azcore.Client
-}
-
-// newClient creates a Client for the container at containerURL sending
-// requests through a pipeline with the given auth policy (which may be nil
-// for anonymous or SAS access).
-func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
-	if options == nil {
-		options = &ClientOptions{}
-	}
-	plOpts := runtime.PipelineOptions{}
-	if authPolicy != nil {
-		plOpts.PerRetry = []policy.Policy{authPolicy}
-	}
-	azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
-	if err != nil {
-		return nil, err
-	}
-	return &Client{
-		endpoint: containerURL,
-		azClient: azClient,
-	}, nil
-}
-
-// NewClient creates a Client authenticating with a token credential.
-func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
-}
-
-// NewClientWithNoCredential creates a Client with no authentication, for
-// anonymous access or a containerURL carrying a SAS token in its query.
-func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, nil, options)
-}
-
-// NewClientWithSharedKeyCredential creates a Client signing requests with the
-// account's shared key.
-func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
-}
-
-// NewListBlobsHierarchyPager returns a pager over the container's blobs,
-// requesting Arrow responses when o.UseArrowFormat is set and returning the
-// same response type as container.Client.NewListBlobsHierarchyPager.
-//
-// If the service does not support Arrow listing (the feature not enabled on
-// the account, or an HNS account) it answers with XML, which is decoded
-// transparently - unless EndBefore is set, in which case the fetcher returns
-// ErrEndBeforeXMLFallback (see there).
-func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
-	// Copy the options so advancing the Marker doesn't mutate the caller's struct.
-	opts := ListBlobsHierarchyOptions{}
-	if o != nil {
-		opts = *o
-	}
-	useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
-	return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
-		More: func(page container.ListBlobsHierarchyResponse) bool {
-			return page.NextMarker != nil && len(*page.NextMarker) > 0
-		},
-		Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
-			if page != nil {
-				opts.Marker = page.NextMarker
-			}
-			req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
-			if err != nil {
-				return container.ListBlobsHierarchyResponse{}, err
-			}
-			resp, err := c.azClient.Pipeline().Do(req)
-			if err != nil {
-				return container.ListBlobsHierarchyResponse{}, err
-			}
-			if !runtime.HasStatusCode(resp, http.StatusOK) {
-				return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
-			}
-			if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
-				return HandleHierarchyListResponse(resp)
-			}
-			if opts.EndBefore != nil {
-				return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
-			}
-			return handleXMLResponse(resp)
-		},
-	})
-}
-
-// listCreateRequest builds the ListBlobs request, mirroring the generated
-// request builders in the SDK.
-func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
-	req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
-	if err != nil {
-		return nil, err
-	}
-	// Reading the existing query first preserves any SAS token in the endpoint.
-	reqQP := req.Raw().URL.Query()
-	reqQP.Set("comp", "list")
-	reqQP.Set("restype", "container")
-	reqQP.Set("delimiter", delimiter)
-	if include := formatInclude(opts.Include); include != "" {
-		reqQP.Set("include", include)
-	}
-	if opts.Marker != nil {
-		reqQP.Set("marker", *opts.Marker)
-	}
-	if opts.MaxResults != nil {
-		reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
-	}
-	if opts.Prefix != nil {
-		reqQP.Set("prefix", *opts.Prefix)
-	}
-	if opts.StartFrom != nil {
-		reqQP.Set("startFrom", *opts.StartFrom)
-	}
-	if useArrow && opts.EndBefore != nil {
-		reqQP.Set("endBefore", *opts.EndBefore)
-	}
-	req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
-	if useArrow {
-		// Stream the Arrow body rather than buffering it in the pipeline.
-		runtime.SkipBodyDownload(req)
-		req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
-	} else {
-		req.Raw().Header["Accept"] = []string{"application/xml"}
-	}
-	req.Raw().Header["x-ms-version"] = []string{serviceVersion}
-	return req, nil
-}
-
-// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
-// generated ListBlobHierarchySegmentHandleResponse.
-func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
-	result := container.ListBlobsHierarchyResponse{}
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-	if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
-		return container.ListBlobsHierarchyResponse{}, err
-	}
-	return result, nil
-}
-
-// formatInclude renders the include datasets as the comma separated query
-// parameter value, in the same order as the SDK's unexported
-// ListBlobsInclude.format.
-func formatInclude(l container.ListBlobsInclude) string {
-	var include []string
-	if l.Copy {
-		include = append(include, "copy")
-	}
-	if l.Deleted {
-		include = append(include, "deleted")
-	}
-	if l.DeletedWithVersions {
-		include = append(include, "deletedwithversions")
-	}
-	if l.ImmutabilityPolicy {
-		include = append(include, "immutabilitypolicy")
-	}
-	if l.LegalHold {
-		include = append(include, "legalhold")
-	}
-	if l.Metadata {
-		include = append(include, "metadata")
-	}
-	if l.Snapshots {
-		include = append(include, "snapshots")
-	}
-	if l.Tags {
-		include = append(include, "tags")
-	}
-	if l.UncommittedBlobs {
-		include = append(include, "uncommittedblobs")
-	}
-	if l.Versions {
-		include = append(include, "versions")
-	}
-	if l.Permissions {
-		include = append(include, "permissions")
-	}
-	return strings.Join(include, ",")
-}

backend/azureblob/arrowlist/arrowlist_test.go

@@ -1,199 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-package arrowlist
-
-import (
-	"context"
-	"encoding/base64"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/apache/arrow-go/v18/arrow"
-	arrowArray "github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// arrowPage builds a one row Arrow IPC listing page with the given blob name
-// and NextMarker.
-func arrowPage(t *testing.T, name, nextMarker string) []byte {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-	return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append(name)
-		b.Field(1).(*arrowArray.Uint64Builder).Append(42)
-	})
-}
-
-func TestPagerArrow(t *testing.T) {
-	var requests []*http.Request
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		requests = append(requests, r.Clone(context.Background()))
-		w.Header().Set("Content-Type", ArrowContentType)
-		if r.URL.Query().Get("marker") == "" {
-			_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
-		} else {
-			_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
-		}
-	}))
-	defer srv.Close()
-
-	// SAS style query on the endpoint must be preserved in requests.
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
-	require.NoError(t, err)
-
-	opts := &ListBlobsHierarchyOptions{
-		ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
-			Include:    container.ListBlobsInclude{Metadata: true, Tags: true},
-			Prefix:     to.Ptr("dir/"),
-			MaxResults: to.Ptr(int32(1000)),
-			StartFrom:  to.Ptr("testcontainer/dir/a"),
-		},
-		EndBefore:      to.Ptr("testcontainer/dir/n"),
-		UseArrowFormat: to.Ptr(true),
-	}
-	pager := client.NewListBlobsHierarchyPager("/", opts)
-
-	var names []string
-	for pager.More() {
-		page, err := pager.NextPage(context.Background())
-		require.NoError(t, err)
-		assert.Equal(t, ArrowContentType, *page.ContentType)
-		for _, item := range page.Segment.BlobItems {
-			names = append(names, *item.Name)
-			assert.Equal(t, int64(42), *item.Properties.ContentLength)
-		}
-	}
-	assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
-
-	require.Len(t, requests, 2)
-	q := requests[0].URL.Query()
-	assert.Equal(t, "list", q.Get("comp"))
-	assert.Equal(t, "container", q.Get("restype"))
-	assert.Equal(t, "/", q.Get("delimiter"))
-	assert.Equal(t, "metadata,tags", q.Get("include"))
-	assert.Equal(t, "dir/", q.Get("prefix"))
-	assert.Equal(t, "1000", q.Get("maxresults"))
-	assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
-	assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
-	assert.Equal(t, "fakesas", q.Get("sv"))
-	assert.Empty(t, q.Get("marker"))
-	assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
-	assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
-	assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
-
-	// The caller's options must not have been mutated by paging.
-	assert.Nil(t, opts.Marker)
-}
-
-const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
-<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
-  <Blobs>
-    <Blob>
-      <Name>file.txt</Name>
-      <Properties>
-        <Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
-        <Etag>0x8D1</Etag>
-        <Content-Length>7</Content-Length>
-        <BlobType>BlockBlob</BlobType>
-      </Properties>
-    </Blob>
-    <BlobPrefix>
-      <Name>subdir/</Name>
-    </BlobPrefix>
-  </Blobs>
-  <NextMarker />
-</EnumerationResults>`
-
-func TestPagerXMLFallback(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/xml")
-		_, _ = w.Write([]byte(xmlListResponse))
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: to.Ptr(true),
-	})
-	page, err := pager.NextPage(context.Background())
-	require.NoError(t, err)
-	assert.False(t, pager.More())
-	assert.Equal(t, "application/xml", *page.ContentType)
-	require.Len(t, page.Segment.BlobItems, 1)
-	assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
-	assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
-	require.Len(t, page.Segment.BlobPrefixes, 1)
-	assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
-}
-
-func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/xml")
-		_, _ = w.Write([]byte(xmlListResponse))
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		EndBefore:      to.Ptr("testcontainer/n"),
-		UseArrowFormat: to.Ptr(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
-}
-
-func TestPagerSharedKeyAuth(t *testing.T) {
-	var gotAuth, gotDate string
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gotAuth = r.Header.Get("Authorization")
-		gotDate = r.Header.Get("x-ms-date")
-		w.Header().Set("Content-Type", ArrowContentType)
-		_, _ = w.Write(arrowPage(t, "blob.txt", ""))
-	}))
-	defer srv.Close()
-
-	cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
-	require.NoError(t, err)
-	client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: to.Ptr(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.NoError(t, err)
-	assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
-	assert.NotEmpty(t, gotDate)
-}
-
-func TestPagerResponseError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: to.Ptr(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.Error(t, err)
-	assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
-}

backend/azureblob/arrowlist/challenge_policy.go

@@ -1,121 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
-// released SDK does not export its storage bearer-challenge policy, which is
-// needed for token authentication with the same semantics as the SDK's own
-// clients.
-
-package arrowlist
-
-import (
-	"errors"
-	"net/http"
-	"strings"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-)
-
-type storageAuthorizer struct {
-	scopes   []string
-	tenantID string
-}
-
-// NewStorageChallengePolicy returns a bearer token policy for cred which
-// handles the storage service's authentication challenges.
-func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
-	s := storageAuthorizer{scopes: []string{audience}}
-	return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
-		AuthorizationHandler: policy.AuthorizationHandler{
-			OnRequest:   s.onRequest,
-			OnChallenge: s.onChallenge,
-		},
-		InsecureAllowCredentialWithHTTP: allowHTTP,
-	})
-}
-
-func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
-	return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
-	// parse the challenge
-	err := s.parseChallenge(resp)
-	if err != nil {
-		return err
-	}
-	// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
-	return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-type challengePolicyError struct {
-	err error
-}
-
-func (c *challengePolicyError) Error() string {
-	return c.err.Error()
-}
-
-func (*challengePolicyError) NonRetriable() {
-	// marker method
-}
-
-func (c *challengePolicyError) Unwrap() error {
-	return c.err
-}
-
-// parses Tenant ID from auth challenge
-// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
-func parseTenant(url string) string {
-	if url == "" {
-		return ""
-	}
-	parts := strings.Split(url, "/")
-	if len(parts) >= 3 {
-		tenant := parts[3]
-		tenant = strings.ReplaceAll(tenant, ",", "")
-		return tenant
-	}
-	return ""
-}
-
-func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
-	authHeader := resp.Header.Get("WWW-Authenticate")
-	if authHeader == "" {
-		return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
-	}
-
-	// Strip down to auth and resource
-	// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
-	authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
-
-	parts := strings.Split(authHeader, " ")
-
-	vals := map[string]string{}
-	for _, part := range parts {
-		subParts := strings.Split(part, "=")
-		if len(subParts) == 2 {
-			stripped := strings.ReplaceAll(subParts[1], "\"", "")
-			stripped = strings.TrimSuffix(stripped, ",")
-			vals[subParts[0]] = stripped
-		}
-	}
-
-	s.tenantID = parseTenant(vals["authorization_uri"])
-
-	scope := vals["resource_id"]
-	if scope == "" {
-		return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
-	}
-
-	if !strings.HasSuffix(scope, "/.default") {
-		scope += "/.default"
-	}
-	s.scopes = []string{scope}
-	return nil
-}

backend/azureblob/arrowlist/sharedkey.go

@@ -1,318 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
-// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
-// with the internal header constants inlined. The released SDK does not
-// export its shared key signing policy, so a copy is needed to sign the
-// Arrow listing requests this package sends through its own pipeline.
-
-package arrowlist
-
-import (
-	"bytes"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/url"
-	"sort"
-	"strings"
-	"sync/atomic"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/rclone/rclone/fs"
-)
-
-// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
-// storage account's name and either its primary or secondary key.
-func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
-	c := SharedKeyCredential{accountName: accountName}
-	if err := c.SetAccountKey(accountKey); err != nil {
-		return nil, err
-	}
-	return &c, nil
-}
-
-// SharedKeyCredential contains an account's name and its primary or secondary key.
-type SharedKeyCredential struct {
-	// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
-	accountName string
-	accountKey  atomic.Value // []byte
-}
-
-// AccountName returns the Storage account's name.
-func (c *SharedKeyCredential) AccountName() string {
-	return c.accountName
-}
-
-// SetAccountKey replaces the existing account key with the specified account key.
-func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
-	_bytes, err := base64.StdEncoding.DecodeString(accountKey)
-	if err != nil {
-		return fmt.Errorf("decode account key: %w", err)
-	}
-	c.accountKey.Store(_bytes)
-	return nil
-}
-
-// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
-func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
-	h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
-	_, err := h.Write([]byte(message))
-	return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
-}
-
-func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
-	// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
-	headers := req.Header
-	contentLength := getHeader("Content-Length", headers)
-	if contentLength == "0" {
-		contentLength = ""
-	}
-
-	canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
-	if err != nil {
-		return "", err
-	}
-
-	stringToSign := strings.Join([]string{
-		req.Method,
-		getHeader("Content-Encoding", headers),
-		getHeader("Content-Language", headers),
-		contentLength,
-		getHeader("Content-MD5", headers),
-		getHeader("Content-Type", headers),
-		"", // Empty date because x-ms-date is expected (as per web page above)
-		getHeader("If-Modified-Since", headers),
-		getHeader("If-Match", headers),
-		getHeader("If-None-Match", headers),
-		getHeader("If-Unmodified-Since", headers),
-		getHeader("Range", headers),
-		c.buildCanonicalizedHeader(headers),
-		canonicalizedResource,
-	}, "\n")
-	return stringToSign, nil
-}
-
-func getHeader(key string, headers map[string][]string) string {
-	if headers == nil {
-		return ""
-	}
-	if v, ok := headers[key]; ok {
-		if len(v) > 0 {
-			return v[0]
-		}
-	}
-
-	return ""
-}
-
-func getWeightTables() [][]int {
-	tableLv0 := [...]int{
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
-		0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
-		0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
-		0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
-		0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
-		0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
-		0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
-	}
-	tableLv2 := [...]int{
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-	}
-	tables := [][]int{tableLv0[:], tableLv2[:]}
-	return tables
-}
-
-// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
-func compareHeaders(lhs, rhs string, tables [][]int) int {
-	currLevel, i, j := 0, 0, 0
-	n := len(tables)
-	lhsLen := len(lhs)
-	rhsLen := len(rhs)
-
-	for currLevel < n {
-		if currLevel == (n-1) && i != j {
-			if i > j {
-				return -1
-			}
-			if i < j {
-				return 1
-			}
-			return 0
-		}
-
-		var w1, w2 int
-
-		// Check bounds before accessing lhs[i]
-		if i < lhsLen {
-			w1 = tables[currLevel][lhs[i]]
-		} else {
-			w1 = 0x1
-		}
-
-		// Check bounds before accessing rhs[j]
-		if j < rhsLen {
-			w2 = tables[currLevel][rhs[j]]
-		} else {
-			w2 = 0x1
-		}
-
-		if w1 == 0x1 && w2 == 0x1 {
-			i = 0
-			j = 0
-			currLevel++
-		} else if w1 == w2 {
-			i++
-			j++
-		} else if w1 == 0 {
-			i++
-		} else if w2 == 0 {
-			j++
-		} else {
-			if w1 < w2 {
-				return -1
-			}
-			if w1 > w2 {
-				return 1
-			}
-			return 0
-		}
-	}
-	return 0
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
-	cm := map[string][]string{}
-	for k, v := range headers {
-		headerName := strings.TrimSpace(strings.ToLower(k))
-		if strings.HasPrefix(headerName, "x-ms-") {
-			cm[headerName] = v // NOTE: the value must not have any whitespace around it.
-		}
-	}
-	if len(cm) == 0 {
-		return ""
-	}
-
-	keys := make([]string, 0, len(cm))
-	for key := range cm {
-		keys = append(keys, key)
-	}
-	tables := getWeightTables()
-	// Sort the keys using the custom comparator
-	sort.Slice(keys, func(i, j int) bool {
-		return compareHeaders(keys[i], keys[j], tables) < 0
-	})
-	ch := bytes.NewBufferString("")
-	for i, key := range keys {
-		if i > 0 {
-			ch.WriteRune('\n')
-		}
-		ch.WriteString(key)
-		ch.WriteRune(':')
-		ch.WriteString(strings.Join(cm[key], ","))
-	}
-	return ch.String()
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
-	// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
-	cr := bytes.NewBufferString("/")
-	cr.WriteString(c.accountName)
-
-	if len(u.Path) > 0 {
-		// Any portion of the CanonicalizedResource string that is derived from
-		// the resource's URI should be encoded exactly as it is in the URI.
-		// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
-		cr.WriteString(u.EscapedPath())
-	} else {
-		// a slash is required to indicate the root path
-		cr.WriteString("/")
-	}
-
-	// params is a map[string][]string; param name is key; params values is []string
-	params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
-	if err != nil {
-		return "", fmt.Errorf("failed to parse query params: %w", err)
-	}
-
-	if len(params) > 0 { // There is at least 1 query parameter
-		var paramNames []string // We use this to sort the parameter key names
-		for paramName := range params {
-			paramNames = append(paramNames, paramName) // paramNames must be lowercase
-		}
-		sort.Strings(paramNames)
-
-		for _, paramName := range paramNames {
-			paramValues := params[paramName]
-			sort.Strings(paramValues)
-
-			// Join the sorted key values separated by ','
-			// Then prepend "keyName:"; then add this string to the buffer
-			cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
-		}
-	}
-	return cr.String(), nil
-}
-
-// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
-func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
-	return cred.computeHMACSHA256(message)
-}
-
-// SharedKeyCredPolicy is a pipeline policy that signs each request with the
-// account's shared key.
-type SharedKeyCredPolicy struct {
-	cred *SharedKeyCredential
-}
-
-// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
-func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
-	return &SharedKeyCredPolicy{cred: cred}
-}
-
-// Do signs the request and sends it via the next policy in the pipeline.
-func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
-	// skip adding the authorization header if no SharedKeyCredential was provided.
-	// this prevents a panic that might be hard to diagnose and allows testing
-	// against http endpoints that don't require authentication.
-	if s.cred == nil {
-		return req.Next()
-	}
-
-	if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
-		req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
-	}
-	stringToSign, err := s.cred.buildStringToSign(req.Raw())
-	if err != nil {
-		return nil, err
-	}
-	signature, err := s.cred.computeHMACSHA256(stringToSign)
-	if err != nil {
-		return nil, err
-	}
-	authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
-	req.Raw().Header.Set("Authorization", authHeader)
-
-	response, err := req.Next()
-	if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
-		// Service failed to authenticate request, log it
-		fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
-	}
-	return response, err
-}

backend/azureblob/azureblob.go

@@ -19,7 +19,6 @@ import (
 	"path"
 	"slices"
 	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -27,15 +26,12 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/backend/azureblob/auth"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/chunksize"
@@ -52,6 +48,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/rclone/rclone/lib/transferaccounter"
 	"golang.org/x/sync/errgroup"
 )
@@ -70,6 +67,7 @@ const (
 	defaultChunkSize      = 4 * fs.Mebi
 	defaultAccessTier     = blob.AccessTier("") // FIXME AccessTierNone
 	sasCopyValidity       = time.Hour           // how long SAS should last when doing server side copy
+	sasCopyStartSkew      = 15 * time.Minute    // how far back SAS should start to allow for clock skew
 )
 
 // setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
@@ -241,40 +239,38 @@ avoid the time out.`,
 			Advanced: true,
 		}, {
 			Name: "use_arrow_list",
-			Help: `Use the experimental Apache Arrow listing format.
+			Help: `Use the Apache Arrow listing format.
 
 If set, directory listings are fetched using the ListBlobs Apache
-Arrow response format instead of XML. This can be faster for very
-large containers.
+Arrow response format instead of XML. Arrow responses are smaller and
+much cheaper to parse, making listings of large containers several
+times faster. Combine with "list_parallelism" for the biggest gains.
 
-This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
-preview feature to be enabled on the storage account. It is NOT
-supported on accounts with a hierarchical namespace (ADLS Gen2) -
-those return a 409 error. If the feature is not enabled the server
-returns XML and the listing transparently falls back to the normal XML
-path (logged at debug level).
-
-Not supported with connection_string auth - falls back to normal
-listing.`,
+"Blob Listing with Apache Arrow" is in public preview at Microsoft and
+is only supported on flat namespace accounts. On accounts with a
+hierarchical namespace (ADLS Gen2), or where the feature is otherwise
+unavailable, the server returns XML and the listing transparently
+falls back to the normal XML path (logged at debug level).`,
 			Default:  false,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
 		}, {
 			Name: "list_parallelism",
 			Help: `Number of parallel shards to list a directory with.
 
-EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
-directory is split into this many ranges which are listed concurrently
-using the Arrow startFrom/endBefore range parameters. This can
-dramatically speed up listing containers with millions of objects, for
-both recursive (ListR) and single directory listings.
+If set greater than 1, the blob name keyspace of each directory is
+split into ranges which are listed concurrently using the Arrow
+startFrom/endBefore range parameters. This can dramatically speed up
+listing containers with millions of objects, for both recursive
+(ListR) and single directory listings. Speed keeps improving up to a
+parallelism of around 30.
 
 This has no effect unless "use_arrow_list" is also set, as Arrow is the
-only listing path that supports server-side name ranges. The default of
-0 (or 1) lists sequentially.`,
+only listing path that supports server-side name ranges. If the
+account does not support range listing (e.g. it has a hierarchical
+namespace) the listing falls back to sequential. The default of 0 (or
+1) lists sequentially.`,
 			Default:  0,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
 		}, {
 			Name: "access_tier",
 			Help: `Access tier of blob: hot, cool, cold or archive.
@@ -449,10 +445,8 @@ type Fs struct {
 	opt                Options                      // parsed config options
 	ci                 *fs.ConfigInfo               // global config
 	features           *fs.Features                 // optional features
-	cntSVCcacheMu      sync.Mutex                   // mutex to protect cntSVCcache and arrowCntSVCcache
+	cntSVCcacheMu      sync.Mutex                   // mutex to protect cntSVCcache
 	cntSVCcache        map[string]*container.Client // reference to containerClient per container
-	arrowCntSVCcache   map[string]*arrowlist.Client // reference to arrowlist client per container
-	arrowClientOpts    *arrowlist.ClientOptions     // client options for the arrowlist clients
 	arrowXMLFallback   atomic.Bool                  // set once the server answers XML so parallel listing is not retried
 	svc                *service.Client              // client to access azblob
 	cred               azcore.TokenCredential       // how to generate tokens (may be nil)
@@ -671,15 +665,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:             name,
-		opt:              *opt,
-		ci:               ci,
-		pacer:            fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		uploadToken:      pacer.NewTokenDispenser(ci.Transfers),
-		copyToken:        pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
-		cache:            bucket.NewCache(),
-		cntSVCcache:      make(map[string]*container.Client, 1),
-		arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
+		name:        name,
+		opt:         *opt,
+		ci:          ci,
+		pacer:       fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+		copyToken:   pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
+		cache:       bucket.NewCache(),
+		cntSVCcache: make(map[string]*container.Client, 1),
 	}
 	f.publicAccess = container.PublicAccessType(opt.PublicAccess)
 	f.setRoot(root)
@@ -726,15 +719,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.usingSharedKeyCred = res.UsingSharedKeyCred
 	f.anonymous = res.Anonymous
 
-	// Client options for the arrowlist clients, using the same transport and
-	// gzip policy as the SDK clients built above.
-	f.arrowClientOpts = &arrowlist.ClientOptions{
-		ClientOptions: azcore.ClientOptions{
-			Transport:       auth.Transporter(ctx),
-			PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
-		},
-	}
-
 	// if using Container level SAS put the container client into the cache
 	if opt.SASURL != "" && res.Container != "" {
 		_ = f.cntSVC(res.Container)
@@ -773,44 +757,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
 	return containerClient
 }
 
-// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
-// credentials can't be used for Arrow listing.
-var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
-
-// return the arrowlist client for the container passed in
-//
-// Returns errArrowAuthUnsupported if the configured credentials can't be
-// reused for the arrowlist client's pipeline.
-func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
-	// The container URL includes any SAS token in its query
-	url := f.cntSVC(containerName).URL()
-	f.cntSVCcacheMu.Lock()
-	defer f.cntSVCcacheMu.Unlock()
-	if client, ok := f.arrowCntSVCcache[containerName]; ok {
-		return client, nil
-	}
-	switch {
-	case f.usingSharedKeyCred:
-		// Covers account+key and the emulator (auth fills in Account/Key)
-		var cred *arrowlist.SharedKeyCredential
-		cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
-		if err == nil {
-			client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
-		}
-	case f.cred != nil:
-		client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
-	case f.anonymous || f.opt.SASURL != "":
-		client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
-	default:
-		return nil, errArrowAuthUnsupported
-	}
-	if err != nil {
-		return nil, err
-	}
-	f.arrowCntSVCcache[containerName] = client
-	return client, nil
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -1187,27 +1133,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 		delimiter = "/"
 	}
 
-	opts := &arrowlist.ListBlobsHierarchyOptions{
-		ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
-			// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
-			Include: container.ListBlobsInclude{
-				Copy:             false,
-				Metadata:         true,
-				Snapshots:        false,
-				UncommittedBlobs: false,
-				Deleted:          false,
-			},
-			Prefix:     &directory,
-			MaxResults: &maxResults,
+	opts := &container.ListBlobsHierarchyOptions{
+		// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
+		Include: container.ListBlobsInclude{
+			Copy:             false,
+			Metadata:         true,
+			Snapshots:        false,
+			UncommittedBlobs: false,
+			Deleted:          false,
 		},
+		Prefix:     &directory,
+		MaxResults: &maxResults,
 	}
-	// Experimental: request the Apache Arrow listing format. The arrowlist
-	// pager requests an Arrow IPC stream and decodes it, falling back to XML
-	// if the account doesn't have Arrow listing enabled. Skip the
-	// maxResults==1 probe (isEmpty) which doesn't benefit.
+	// Request the Apache Arrow listing format. The SDK pager requests an
+	// Arrow IPC stream and decodes it, falling back to XML if the account
+	// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
+	// (isEmpty) which doesn't benefit.
 	useArrow := f.opt.UseArrowList && maxResults != 1
 	if useArrow {
-		opts.UseArrowFormat = to.Ptr(true)
+		opts.ResponseFormat = container.StorageResponseFormatArrow
 	}
 
 	var foundItems int
@@ -1238,25 +1182,11 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 // listBlobsPager runs the hierarchy listing described by opts, calling fn for
 // each blob and subdirectory, and returns the number of raw items seen.
 // delimiter selects flat (recurse) vs hierarchical listing. If opts requests
-// the Apache Arrow format the listing goes through the arrowlist pager
-// (falling back to the SDK pager if the credentials don't support it); if the
-// service then answers with XML (Arrow listing not enabled) a debug message
-// is logged.
-func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
-	useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
-	var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
-	if useArrow {
-		arrowSVC, err := f.arrowCntSVC(containerName)
-		if err != nil {
-			fs.Debugf(f, "Not using Arrow listing: %v", err)
-			useArrow = false
-		} else {
-			pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
-		}
-	}
-	if pager == nil {
-		pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
-	}
+// the Apache Arrow format and the service answers with XML (Arrow listing
+// not enabled) a debug message is logged.
+func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
+	useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
+	pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
 	checkedArrow := false
 	for pager.More() {
 		var response container.ListBlobsHierarchyResponse
@@ -1272,7 +1202,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
 		// but not accelerated.
 		if useArrow && !checkedArrow && err == nil {
 			checkedArrow = true
-			if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
+			if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
 				fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
 			}
 		}
@@ -1827,7 +1757,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
 	}
 
 	// Validity window
-	start := time.Now().UTC()
+	start := time.Now().UTC().Add(-sasCopyStartSkew)
 	expiry := start.Add(2 * sasCopyValidity)
 	startStr := start.Format(time.RFC3339)
 	expiryStr := expiry.Format(time.RFC3339)
@@ -1876,7 +1806,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
 		// Build the SAS values
 		perms := sas.BlobPermissions{Read: true}
 		container, containerPath := o.split()
-		start := time.Now().UTC()
+		start := time.Now().UTC().Add(-sasCopyStartSkew)
 		expiry := start.Add(sasCopyValidity)
 		vals := sas.BlobSignatureValues{
 			StartTime:     start,
@@ -2346,6 +2276,16 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
 	} else {
 		size = *info.ContentLength
 	}
+	// On a range request Content-Length is the length of the range, not the object, so take the
+	// object's size from the total in Content-Range instead.
+	if info.ContentRange != nil {
+		contentRange, err := rest.ParseContentRange(*info.ContentRange)
+		if err != nil {
+			fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
+		} else if contentRange.Size >= 0 {
+			size = contentRange.Size
+		}
+	}
 	if isDirectoryMarker(size, metadata, o.remote) {
 		return fs.ErrorNotAFile
 	}
@@ -2379,22 +2319,6 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
 	// 	o.accessTier = blob.AccessTier(*info.AccessTier)
 	// }
 	o.setMetadata(metadata)
-
-	// If it was a Range request, the size is wrong, so correct it
-	if info.ContentRange != nil {
-		contentRange := *info.ContentRange
-		slash := strings.IndexRune(contentRange, '/')
-		if slash >= 0 {
-			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
-			if err == nil {
-				o.size = i
-			} else {
-				fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
-			}
-		} else {
-			fs.Debugf(o, "Failed to find length in %q", contentRange)
-		}
-	}
 	o.contentEncoding = info.ContentEncoding
 
 	// If decompressing then size and md5sum are unknown

backend/azureblob/azureblob_internal_test.go

@@ -54,6 +54,58 @@ func TestBlockIDCreator(t *testing.T) {
 	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
 }
 
+func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
+	pInt64 := func(i int64) *int64 { return &i }
+	pString := func(s string) *string { return &s }
+	newTestObject := func() *Object {
+		return &Object{
+			fs:     &Fs{},
+			remote: "test.bin",
+			size:   -2, // sentinel to check it gets set
+		}
+	}
+
+	t.Run("WholeBlob", func(t *testing.T) {
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		info.ContentLength = pInt64(12345)
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(12345), o.size)
+	})
+
+	t.Run("RangeRequest", func(t *testing.T) {
+		// On a ranged download Content-Length is the chunk length so the
+		// size must come from the total in Content-Range
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		info.ContentLength = pInt64(67108864)
+		info.ContentRange = pString("bytes 67108864-134217727/169721004032")
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(169721004032), o.size)
+	})
+
+	t.Run("BadContentRange", func(t *testing.T) {
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		info.ContentLength = pInt64(67108864)
+		info.ContentRange = pString("potato")
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(67108864), o.size)
+	})
+
+	t.Run("UnknownLength", func(t *testing.T) {
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(-1), o.size)
+	})
+}
+
+func TestCopySASTimingConstants(t *testing.T) {
+	require.Greater(t, sasCopyStartSkew, time.Duration(0))
+	require.Greater(t, sasCopyValidity, sasCopyStartSkew)
+}
+
 func (f *Fs) testFeatures(t *testing.T) {
 	// Check first feature flags are set on this remote
 	enabled := f.Features().SetTier

backend/azurefiles/azurefiles.go

@@ -696,11 +696,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	// Measure the size if it is unknown
-	if sizeUnknown {
-		counter = readers.NewCountingReader(in)
-		in = counter
-	}
+	// Count the bytes read so short reads from the source can be detected
+	counter = readers.NewCountingReader(in)
+	in = counter
 
 	// Check we have a source MD5 hash...
 	if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
@@ -735,6 +733,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if sizeUnknown {
 		// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
 		size = int64(counter.BytesRead())
+	} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
+		// The file was created at the declared size so a short source
+		// leaves the remainder padded with zeroes
+		if isNewlyCreated {
+			if _, delErr := fc.Delete(ctx, nil); delErr != nil {
+				fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
+			}
+		} else if bytesRead < size {
+			// Truncate the file to the bytes actually received so it
+			// is not left padded to the declared size with zeroes
+			if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
+				fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
+			}
+		}
+		return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
 	}
 	if hashUnknown {
 		md5Hash = hasher.Sum(nil)

backend/b2/b2.go

@@ -191,6 +191,11 @@ If the custom endpoint rewrites the requests for authentication,
 e.g., in Cloudflare Workers, this header needs to be handled properly.
 Leave blank if you want to use the endpoint provided by Backblaze.
 
+Rclone will not follow a redirect from this endpoint which downgrades
+HTTPS to HTTP, and will not send the "Authorization" header or the
+SSE-C key headers once a redirect has left the host and port of this
+endpoint.
+
 The URL provided here SHOULD have the protocol and SHOULD NOT have
 a trailing slash or specify the /file/bucket subpath as rclone will
 request files with "{download_url}/file/{bucket_name}/{path}".
@@ -529,6 +534,25 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
+// redirectSecretHeaders are the request headers carrying secrets that
+// must not be forwarded when a redirect leaves the original host.
+var redirectSecretHeaders = []string{
+	"Authorization",
+	sseAlgorithmHeader,
+	sseKeyHeader,
+	sseMd5Header,
+}
+
+// newClient makes the http client used for all B2 requests.
+//
+// The download_url option sends the requests to a host the user does
+// not control so it strips headers on cross host redirects.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
+	return client
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -581,7 +605,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:        name,
 		opt:         *opt,
 		ci:          ci,
-		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
+		srv:         rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
 		cache:       bucket.NewCache(),
 		_bucketID:   make(map[string]string, 1),
 		_bucketType: make(map[string]string, 1),

backend/b2/b2_internal_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"crypto/sha1"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"path"
 	"sort"
 	"strings"
@@ -617,3 +619,60 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// redirectTestHeaders is a deliberately literal copy of
+// redirectSecretHeaders so that a header dropped from the production
+// list fails TestRedirectSecretHeaders rather than silently losing
+// test coverage.
+var redirectTestHeaders = map[string]string{
+	"Authorization": "secret-token",
+	"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
+	"X-Bz-Server-Side-Encryption-Customer-Key":       "secret-key",
+	"X-Bz-Server-Side-Encryption-Customer-Key-Md5":   "secret-md5",
+}
+
+func TestRedirectSecretHeaders(t *testing.T) {
+	want := make([]string, 0, len(redirectTestHeaders))
+	for header := range redirectTestHeaders {
+		want = append(want, header)
+	}
+	sort.Strings(want)
+	got := append([]string(nil), redirectSecretHeaders...)
+	sort.Strings(got)
+	assert.Equal(t, want, got)
+}
+
+// redirectTestRequest makes a GET request to rawURL carrying the
+// secret headers plus a harmless marker header.
+func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
+	require.NoError(t, err)
+	for header, value := range redirectTestHeaders {
+		req.Header.Set(header, value)
+	}
+	req.Header.Set("X-Bz-Test-Mode", "marker")
+	return req
+}
+
+func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
+	client := newClient(context.Background())
+
+	redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for header := range redirectTestHeaders {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+		assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer redirectServer.Close()
+
+	initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
+	}))
+	defer initialServer.Close()
+
+	resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+}

backend/box/box.go

@@ -45,6 +45,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
 )
@@ -1460,7 +1461,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 					// Item in the cache has the same or newer SequenceID than
 					// this event. Ignore this event, it must be old.
 					f.itemMetaCacheMu.Unlock()
-					fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
+					fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
 					continue
 				}
 
@@ -1691,7 +1692,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // upload does a single non-multipart upload
 //
 // This is recommended for less than 50 MiB of content
-func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
+func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
 	upload := api.UploadFile{
 		Name:              o.fs.opt.Enc.FromStandardName(leaf),
 		ContentModifiedAt: api.Time(modTime),
@@ -1703,9 +1704,10 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 
 	var resp *http.Response
 	var result api.FolderItems
+	counter := readers.NewCountingReader(in)
 	opts := rest.Opts{
 		Method:                "POST",
-		Body:                  in,
+		Body:                  counter,
 		MultipartMetadataName: "attributes",
 		MultipartContentName:  "contents",
 		MultipartFileName:     upload.Name,
@@ -1725,6 +1727,11 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 	if err != nil {
 		return err
 	}
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload.
+	if size >= 0 && int64(counter.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
 	if result.TotalCount != 1 || len(result.Entries) != 1 {
 		return fmt.Errorf("failed to upload %v - not sure why", o)
 	}
@@ -1754,7 +1761,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Upload with simple or multipart
 	if size <= int64(o.fs.opt.UploadCutoff) {
-		err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
+		err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
 	} else {
 		err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
 	}

backend/chunker/chunker.go

@@ -801,9 +801,12 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 				fs.Debugf(f, "skip orphan data chunk %q", remote)
 				break
 			}
-			if mainObject == nil && !f.useMeta {
+			if mainObject == nil && !f.useMeta && !isSpecial {
 				// this is the "nometa" case
 				// create dummy chunked object without metadata
+				//
+				// not for temporary or control chunks as they may be all
+				// that is left of an interrupted upload
 				mainObject = f.newObject(mainRemote, nil, nil)
 				byRemote[mainRemote] = mainObject
 				if !badEntry[mainRemote] {

backend/chunker/chunker_internal_test.go

@@ -627,6 +627,22 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	_ = file.Remove(ctx)
 }
 
+// testOrphanTempChunks checks that temporary chunks left behind by an
+// interrupted upload are not listed and don't break the listing
+func testOrphanTempChunks(t *testing.T, f *Fs) {
+	const dir = "orphan"
+	ctx := context.Background()
+	defer func() {
+		_ = operations.Purge(ctx, f.base, dir)
+	}()
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
+	fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
+
+	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
+}
+
 func testMetadataInput(t *testing.T, f *Fs) {
 	const minChunkForTest = 50
 	if f.opt.ChunkSize < minChunkForTest {
@@ -928,6 +944,9 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("BackwardsCompatibility", func(t *testing.T) {
 		testBackwardsCompatibility(t, f)
 	})
+	t.Run("OrphanTempChunks", func(t *testing.T) {
+		testOrphanTempChunks(t, f)
+	})
 	t.Run("ChunkerServerSideMove", func(t *testing.T) {
 		testChunkerServerSideMove(t, f)
 	})

backend/compress/compress.go

@@ -698,6 +698,16 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		return nil, err
 	}
 
+	// Check the source supplied the number of bytes it declared -
+	// compressed data is stored under a name containing the
+	// original size so a mismatch makes the object unreadable.
+	if size := src.Size(); size >= 0 && meta.Size != size {
+		if removeErr := dataObject.Remove(ctx); removeErr != nil {
+			fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
+		}
+		return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
+	}
+
 	mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
 
 	// meta data upload may fail. in this case we try to remove the original object
@@ -1070,6 +1080,35 @@ type ObjectMetadata struct {
 	CompressionMetadataZstd *SzstdMetadata      // Metadata for Zstd compression
 }
 
+// validate checks metadata read from the wrapped remote before it is used to
+// construct a compressed reader.
+func (meta *ObjectMetadata) validate() error {
+	if meta.Mode != Gzip {
+		return nil
+	}
+	if meta.CompressionMetadataGzip == nil {
+		return errors.New("missing gzip metadata")
+	}
+	if meta.CompressionMetadataGzip.BlockSize <= 0 {
+		return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
+	}
+	if meta.Size != meta.CompressionMetadataGzip.Size {
+		return errors.New("gzip metadata size does not match object size")
+	}
+	if meta.Size < 0 {
+		return errors.New("invalid gzip object size")
+	}
+	blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
+	blocks := meta.Size / blockSize
+	if meta.Size%blockSize != 0 {
+		blocks++
+	}
+	if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
+		return errors.New("gzip block data is incomplete")
+	}
+	return nil
+}
+
 // Object with external metadata
 type Object struct {
 	fs.Object                 // Wraps around data object for this object
@@ -1093,6 +1132,9 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
 	if err = jr.Decode(meta); err != nil {
 		return nil, err
 	}
+	if err = meta.validate(); err != nil {
+		return nil, err
+	}
 	return meta, nil
 }
 

backend/compress/compress_test.go

@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/buengese/sgzip"
+
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/local"
 	_ "github.com/rclone/rclone/backend/s3"
@@ -14,6 +16,93 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
+func TestObjectMetadataValidate(t *testing.T) {
+	tests := []struct {
+		name string
+		meta ObjectMetadata
+		want bool
+	}{
+		{
+			name: "nil gzip metadata",
+			meta: ObjectMetadata{Mode: Gzip},
+			want: true,
+		},
+		{
+			name: "zero block size",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1024,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 0,
+					Size:      1024,
+					BlockData: []uint32{0},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "negative block size",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1024,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: -1,
+					Size:      1024,
+					BlockData: []uint32{0},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "mismatched sizes",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1024,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 512,
+					Size:      2048,
+					BlockData: []uint32{0, 1},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "short block data",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1025,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 512,
+					Size:      1025,
+					BlockData: []uint32{0, 1},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "valid",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1025,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 512,
+					Size:      1025,
+					BlockData: []uint32{0, 1, 2},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := test.meta.validate()
+			if (err != nil) != test.want {
+				t.Fatalf("validate() error = %v, want error: %v", err, test.want)
+			}
+		})
+	}
+}
+
 var defaultOpt = fstests.Opt{
 	RemoteName: "TestCompress:",
 	NilObject:  (*Object)(nil),

backend/crypt/cipher.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 	"strconv"
 	"strings"
 	"sync"
@@ -117,8 +118,8 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-// fileNameEncoding are the encoding methods dealing with encrypted file names
-type fileNameEncoding interface {
+// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
+type FileNameEncoding interface {
 	EncodeToString(src []byte) string
 	DecodeString(s string) ([]byte, error)
 }
@@ -153,7 +154,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
 }
 
 // NewNameEncoding creates a NameEncoding from a string
-func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
+func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
 	s = strings.ToLower(s)
 	switch s {
 	case "base32":
@@ -175,7 +176,7 @@ type Cipher struct {
 	nameTweak       [nameCipherBlockSize]byte // used to tweak the name crypto
 	block           gocipher.Block
 	mode            NameEncryptionMode
-	fileNameEnc     fileNameEncoding
+	fileNameEnc     FileNameEncoding
 	buffers         sync.Pool // encrypt/decrypt buffers
 	cryptoRand      io.Reader // read crypto random numbers from here
 	dirNameEncrypt  bool
@@ -184,7 +185,7 @@ type Cipher struct {
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
 	c := &Cipher{
 		mode:            mode,
 		fileNameEnc:     enc,
@@ -486,7 +487,12 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 }
 
 // encryptFileName encrypts a file path
-func (c *Cipher) encryptFileName(in string) string {
+//
+// If stripVersion is set, a version string (as used by --b2-versions)
+// on the last segment is kept in plain text. Set it for file names
+// only: directory names must encrypt identically whether passed alone
+// or as the parent of a file name.
+func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		// Skip directory name encryption if the user chose to
@@ -499,7 +505,7 @@ func (c *Cipher) encryptFileName(in string) string {
 		// of the file name gets encrypted/obfuscated
 		hasVersion := false
 		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
+		if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
 			var s string
 			t, s = version.Remove(segments[i])
 			// version.Remove can fail, in which case it returns segments[i]
@@ -529,19 +535,24 @@ func (c *Cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
 		return in + c.encryptedSuffix
 	}
-	return c.encryptFileName(in)
+	return c.encryptFileName(in, true)
 }
 
 // EncryptDirName encrypts a directory path
+//
+// Unlike EncryptFileName, a version string on the last segment is
+// encrypted along with the rest of the name.
 func (c *Cipher) EncryptDirName(in string) string {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in
 	}
-	return c.encryptFileName(in)
+	return c.encryptFileName(in, false)
 }
 
 // decryptFileName decrypts a file path
-func (c *Cipher) decryptFileName(in string) (string, error) {
+//
+// stripVersion is as for encryptFileName.
+func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -555,7 +566,7 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		// of the file name gets decrypted/deobfuscated
 		hasVersion := false
 		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
+		if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
 			var s string
 			t, s = version.Remove(segments[i])
 			// version.Remove can fail, in which case it returns segments[i]
@@ -601,15 +612,31 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
 		// Leave the version string on, if it was there
 		return decrypted, nil
 	}
-	return c.decryptFileName(in)
+	return c.decryptFileName(in, true)
 }
 
 // DecryptDirName decrypts a directory path
+//
+// Unlike DecryptFileName, a version string on the last segment is
+// expected to be part of the encrypted name. Directory names created
+// by rclone before v1.76 had a version-like suffix left in plain
+// text; these are still decrypted so that they appear in listings,
+// but they can't be opened or removed until renamed on the
+// underlying remote.
 func (c *Cipher) DecryptDirName(in string) (string, error) {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in, nil
 	}
-	return c.decryptFileName(in)
+	out, err := c.decryptFileName(in, false)
+	if err != nil && version.Match(path.Base(in)) {
+		var legacyErr error
+		out, legacyErr = c.decryptFileName(in, true)
+		if legacyErr == nil {
+			fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
+			return out, nil
+		}
+	}
+	return out, err
 }
 
 // NameEncryptionMode returns the encryption mode in use for names

backend/crypt/cipher_test.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 	"strings"
 	"testing"
 
@@ -682,6 +683,43 @@ func TestNonStandardDecryptDirName(t *testing.T) {
 	}
 }
 
+// Test directories whose name looks like it has a version string -
+// these encrypt verbatim, so that EncryptDirName agrees with the same
+// directory encrypted as the parent of a file name, whereas file names
+// keep the version string in plain text
+func TestVersionedDirName(t *testing.T) {
+	const dir = "dir-v2001-02-03-040506-123"
+	enc, err := NewNameEncoding("base32")
+	require.NoError(t, err)
+	for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
+		c, err := newCipher(mode, "", "", true, enc)
+		require.NoError(t, err)
+		what := fmt.Sprintf("Testing mode=%v", mode)
+		// Check EncryptDirName matches the parent of an encrypted file name
+		encryptedDir := c.EncryptDirName(dir)
+		encryptedFile := c.EncryptFileName(dir + "/file.txt")
+		assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
+		// Check the encrypted directory name round trips OK
+		decryptedDir, err := c.DecryptDirName(encryptedDir)
+		assert.NoError(t, err, what)
+		assert.Equal(t, dir, decryptedDir, what)
+		// Check a file with the same name keeps its version string in plain text
+		encryptedFile = c.EncryptFileName(dir)
+		assert.NotEqual(t, encryptedDir, encryptedFile, what)
+		assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
+		decryptedFile, err := c.DecryptFileName(encryptedFile)
+		assert.NoError(t, err, what)
+		assert.Equal(t, dir, decryptedFile, what)
+		// Check a directory with a legacy encrypted name (the file form) still decrypts
+		decryptedDir, err = c.DecryptDirName(encryptedFile)
+		assert.NoError(t, err, what)
+		assert.Equal(t, dir, decryptedDir, what)
+		// but a name which is bad in both forms is still an error
+		_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
+		assert.Error(t, err, what)
+	}
+}
+
 func TestEncryptedSize(t *testing.T) {
 	c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
 	for _, test := range []struct {

backend/crypt/crypt.go

@@ -1179,6 +1179,11 @@ func (o *ObjectInfo) Size() int64 {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
+	// If the data is unchanged then the hash of the source is the
+	// hash of the object which will be uploaded
+	if o.f.opt.NoDataEncryption {
+		return o.ObjectInfo.Hash(ctx, hash)
+	}
 	var srcObj fs.Object
 	var ok bool
 	// Get the underlying object if there is one

backend/crypt/crypt_internal_test.go

@@ -84,6 +84,10 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 
 	// Test ObjectInfo.Hash
 	wantHash := md5.Sum(outBuf.Bytes())
+	if f.opt.NoDataEncryption {
+		// If the data isn't encrypted, the hash should be that of the plaintext
+		wantHash = md5.Sum([]byte(contents))
+	}
 	gotHash, err := src.Hash(ctx, hash.MD5)
 	require.NoError(t, err)
 	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)

backend/drime/drime.go

@@ -692,13 +692,27 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // deleteObject removes an object by ID
 func (f *Fs) deleteObject(ctx context.Context, id string) error {
+	if f.opt.HardDelete {
+		// Deleting forever doesn't invalidate the server's cache of
+		// the parent folder listing, so the entry keeps being listed
+		// for about a second. Moving to the trash first does.
+		err := f.deleteEntry(ctx, id, false)
+		if err != nil {
+			return err
+		}
+	}
+	return f.deleteEntry(ctx, id, f.opt.HardDelete)
+}
+
+// deleteEntry moves the entry with ID to the trash or deletes it forever
+func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/file-entries/delete",
 	}
 	request := api.DeleteRequest{
 		EntryIDs:      []string{id},
-		DeleteForever: f.opt.HardDelete,
+		DeleteForever: deleteForever,
 	}
 	var result api.DeleteResponse
 	err := f.pacer.Call(func() (bool, error) {
@@ -1040,17 +1054,28 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
 }
 
 // copy and rename a file or folder to directoryID with leaf
-func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
-	// Can have duplicates so don't have to be careful here
-
+//
+// If existing is set it is removed once the copy has succeeded.
+func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
 	// Copy to dstDirectoryID first
 	info, err = f.copy(ctx, srcID, dstDirectoryID)
 	if err != nil {
 		return nil, err
 	}
 
+	// The server names the copy "name (1)" if the destination
+	// directory already has an entry called "name" and refuses to
+	// rename it to "name" until that entry is removed.
+	if existing != nil {
+		fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
+		err = existing.Remove(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Rename if required
-	if srcLeaf != dstLeaf {
+	if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
 		info, err = f.rename(ctx, info.ID.String(), dstLeaf)
 		if err != nil {
 			return nil, err
@@ -1074,8 +1099,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	srcLeaf := path.Base(srcObj.remote)
-
 	srcPath := srcObj.fs.rootSlash() + srcObj.remote
 	dstPath := f.rootSlash() + remote
 	if srcPath == dstPath {
@@ -1084,15 +1107,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 
 	// Find existing object
 	existingObj, err := f.NewObject(ctx, remote)
-	if err == nil {
-		defer func() {
-			// Don't remove existing object if returning an error
-			if err != nil {
-				return
-			}
-			fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
-			err = existingObj.Remove(ctx)
-		}()
+	if err != nil {
+		existingObj = nil
 	}
 
 	// Create temporary object
@@ -1102,7 +1118,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	// Copy the object
-	info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
+	info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
 	if err != nil {
 		return nil, err
 	}

backend/drive/drive.go

@@ -2766,7 +2766,12 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	}
 	trashedFiles := false
 	if check {
-		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
+		// trashedFiles is only consulted to decide between trashing and hard deleting
+		// below, so when the directory is going to be trashed anyway there is no reason
+		// to enumerate trashed children: let the server filter them out instead of
+		// paging through them. Hard deletes still need to see them (#1040).
+		includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
+		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
 			if !item.Trashed {
 				fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
 				return true

backend/dropbox/batcher.go

@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 		Entries: items,
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
+		complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
 		if retry, err := shouldRetryExclude(ctx, err); !retry {
 			return retry, err
 		}

backend/dropbox/dropbox.go

@@ -382,20 +382,20 @@ type Options struct {
 
 // Fs represents a remote dropbox server
 type Fs struct {
-	name           string         // name of this remote
-	root           string         // the path we are working on
-	opt            Options        // parsed options
-	ci             *fs.ConfigInfo // global config
-	features       *fs.Features   // optional features
-	srv            files.Client   // the connection to the dropbox server
-	svc            files.Client   // the connection to the dropbox server (unauthorized)
-	sharing        sharing.Client // as above, but for generating sharing links
-	users          users.Client   // as above, but for accessing user information
-	team           team.Client    // for the Teams API
-	slashRoot      string         // root with "/" prefix, lowercase
-	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *fs.Pacer      // To pace the API calls
-	ns             string         // The namespace we are using or "" for none
+	name           string                // name of this remote
+	root           string                // the path we are working on
+	opt            Options               // parsed options
+	ci             *fs.ConfigInfo        // global config
+	features       *fs.Features          // optional features
+	srv            files.ContextClient   // the connection to the dropbox server
+	svc            files.ContextClient   // the connection to the dropbox server (unauthorized)
+	sharing        sharing.ContextClient // as above, but for generating sharing links
+	users          users.ContextClient   // as above, but for accessing user information
+	team           team.ContextClient    // for the Teams API
+	slashRoot      string                // root with "/" prefix
+	slashRootSlash string                // root with "/" prefix and postfix
+	pacer          *fs.Pacer             // To pace the API calls
+	ns             string                // The namespace we are using or "" for none
 	batcher        *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
 	exportExts     []exportExtension
 }
@@ -577,7 +577,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
-	f.team = team.New(cfg)
+	f.team = team.NewContext(cfg)
 
 	if opt.Impersonate != "" {
 		user := team.UserSelectorArg{
@@ -588,7 +588,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		members := []*team.UserSelectorArg{&user}
 		args := team.NewMembersGetInfoArgs(members)
 
-		memberIDs, err := f.team.MembersGetInfo(args)
+		memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
@@ -603,10 +603,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		cfg.AsAdminID = opt.ImpersonateAdmin
 	}
 
-	f.srv = files.New(cfg)
-	f.svc = files.New(ucfg)
-	f.sharing = sharing.New(cfg)
-	f.users = users.New(cfg)
+	f.srv = files.NewContext(cfg)
+	f.svc = files.NewContext(ucfg)
+	f.sharing = sharing.NewContext(cfg)
+	f.users = users.NewContext(cfg)
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
 		ReadMimeType:            false,
@@ -633,13 +633,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return f, nil // our root it empty so we probably want to list shared folders
 		}
 
-		dir := path.Dir(f.root)
-		if dir == "." {
-			dir = f.root
-		}
-
 		// root is not empty so we have find the right shared folder if it exists
-		id, err := f.findSharedFolder(ctx, dir)
+		id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
 		if err != nil {
 			// if we didn't find the specified shared folder we have to bail out here
 			return nil, err
@@ -673,7 +668,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// If root starts with / then use the actual root
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
-			acc, err = f.users.GetCurrentAccount()
+			acc, err = f.users.GetCurrentAccountContext(ctx)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -693,7 +688,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// See if the root is actually an object
 	if f.root != "" {
-		_, err = f.getFileMetadata(ctx, f.slashRoot)
+		_, _, err = f.getFileMetadata(ctx, f.slashRoot)
 		if err == nil {
 			newRoot := path.Dir(f.root)
 			if newRoot == "." {
@@ -728,15 +723,16 @@ func (f *Fs) setRoot(root string) {
 }
 
 type getMetadataResult struct {
-	entry    files.IsMetadata
-	notFound bool
-	err      error
+	entry              files.IsMetadata
+	remoteIsExportPath bool
+	notFound           bool
+	err                error
 }
 
 // getMetadata gets the metadata for a file or directory
 func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
 	res.err = f.pacer.Call(func() (bool, error) {
-		res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
+		res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
 			Path: f.opt.Enc.FromStandardPath(objPath),
 		})
 		return shouldRetry(ctx, res.err)
@@ -779,6 +775,7 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
 				ch <- getMetadataResult{notFound: true}
 				return
 			}
+			res.remoteIsExportPath = true
 		}
 
 		// Return our real result or error
@@ -824,7 +821,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
 }
 
 // getFileMetadata gets the metadata for a file
-func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
+func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
 	var res getMetadataResult
 
 	// Try all possible metadatas
@@ -833,7 +830,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
 		res = <-ch
 
 		if res.err != nil {
-			return nil, res.err
+			return nil, false, res.err
 		}
 		if !res.notFound {
 			break
@@ -841,17 +838,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
 	}
 
 	if res.notFound {
-		return nil, fs.ErrorObjectNotFound
+		return nil, false, fs.ErrorObjectNotFound
 	}
 
 	fileInfo, ok := res.entry.(*files.FileMetadata)
 	if !ok {
 		if _, ok = res.entry.(*files.FolderMetadata); ok {
-			return nil, fs.ErrorIsDir
+			return nil, false, fs.ErrorIsDir
 		}
-		return nil, fs.ErrorNotAFile
+		return nil, false, fs.ErrorNotAFile
 	}
-	return fileInfo, nil
+	return fileInfo, res.remoteIsExportPath, nil
 }
 
 // getDirMetadata gets the metadata for a directory
@@ -910,7 +907,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
 				Limit: 100,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListFolders(&arg)
+				res, err = f.sharing.ListFoldersContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -922,7 +919,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
 				Cursor: res.Cursor,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListFoldersContinue(&arg)
+				res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -945,13 +942,21 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
 	return nil
 }
 
+// sharedFolderName returns the shared folder name in root, which is its first
+// path component. root must be the trimmed root as produced by setRoot (no
+// leading slash).
+func sharedFolderName(root string) string {
+	name, _, _ := strings.Cut(root, "/")
+	return name
+}
+
 // findSharedFolder find the id for a given shared folder name
 // somewhat annoyingly there is no endpoint to query a shared folder by it's name
 // so our only option is to iterate over all shared folders
 func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
 	errFoundFile := errors.New("found file")
 	err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
-		if entry.(*fs.Dir).Remote() == name {
+		if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
 			id = entry.(*fs.Dir).ID()
 			return errFoundFile
 		}
@@ -971,7 +976,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
 		SharedFolderId: id,
 	}
 	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.sharing.MountFolder(&arg)
+		_, err := f.sharing.MountFolderContext(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -988,7 +993,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 				Limit: 100,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListReceivedFiles(&arg)
+				res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1000,7 +1005,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 				Cursor: res.Cursor,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListReceivedFilesContinue(&arg)
+				res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1008,7 +1013,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 			}
 		}
 		for _, entry := range res.Entries {
-			entryPath := entry.Name
+			entryPath := f.opt.Enc.ToStandardName(entry.Name)
 			o := &Object{
 				fs:      f,
 				url:     entry.PreviewUrl,
@@ -1033,7 +1038,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
 	errFoundFile := errors.New("found file")
 	err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
-		if entry.(*Object).remote == name {
+		if strings.EqualFold(entry.(*Object).remote, name) {
 			o = entry.(*Object)
 			return errFoundFile
 		}
@@ -1107,7 +1112,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 				arg.Path = "" // Specify root folder as empty string
 			}
 			err = f.pacer.Call(func() (bool, error) {
-				res, err = f.srv.ListFolder(arg)
+				res, err = f.srv.ListFolderContext(ctx, arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1125,7 +1130,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 				Cursor: res.Cursor,
 			}
 			err = f.pacer.Call(func() (bool, error) {
-				res, err = f.srv.ListFolderContinue(&arg)
+				res, err = f.srv.ListFolderContinueContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1162,7 +1167,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 						var sfMeta *sharing.SharedFolderMetadata
 						err = f.pacer.Call(func() (bool, error) {
 							var apiErr error
-							sfMeta, apiErr = f.sharing.GetFolderMetadata(sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
+							sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
 							return shouldRetry(ctx, apiErr)
 						})
 						if err != nil {
@@ -1249,7 +1254,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 		return cErr
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.CreateFolderV2(&arg2)
+		_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -1277,7 +1282,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		}
 		var res *files.ListFolderResult
 		err = f.pacer.Call(func() (bool, error) {
-			res, err = f.srv.ListFolder(arg)
+			res, err = f.srv.ListFolderContext(ctx, arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1301,7 +1306,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	// remove it
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
+		_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -1360,7 +1365,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
-		result, err = f.srv.CopyV2(&arg)
+		result, err = f.srv.CopyV2Context(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1421,7 +1426,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
-		result, err = f.srv.MoveV2(&arg)
+		result, err = f.srv.MoveV2Context(ctx, &arg)
 		switch e := err.(type) {
 		case files.MoveV2APIError:
 			// There seems to be a bit of eventual consistency here which causes this to
@@ -1475,7 +1480,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
-		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+		linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
 		return shouldRetry(ctx, err)
 	})
 
@@ -1484,7 +1489,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		fs.Debugf(absPath, "can't create link with expiry, trying without")
 		createArg.Settings.Expires = nil
 		err = f.pacer.Call(func() (bool, error) {
-			linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+			linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
 			return shouldRetry(ctx, err)
 		})
 	}
@@ -1498,7 +1503,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		}
 		var listRes *sharing.ListSharedLinksResult
 		err = f.pacer.Call(func() (bool, error) {
-			listRes, err = f.sharing.ListSharedLinks(&listArg)
+			listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1559,7 +1564,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		},
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.MoveV2(&arg)
+		_, err = f.srv.MoveV2Context(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1573,7 +1578,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var q *users.SpaceUsage
 	err = f.pacer.Call(func() (bool, error) {
-		q, err = f.users.GetSpaceUsage()
+		q, err = f.users.GetSpaceUsageContext(ctx)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1660,7 +1665,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
 			arg.Path = ""
 		}
 
-		startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
+		startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
 
 		return shouldRetry(ctx, err)
 	})
@@ -1670,6 +1675,34 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
 	return startCursor.Cursor, nil
 }
 
+// trimPrefixFold returns s with the leading prefix removed, matching
+// case insensitively rune by rune so folded runes of differing UTF-8
+// length still match.
+//
+// If s is exactly the root that prefix names (prefix without its
+// trailing "/") it returns "". If prefix doesn't match it returns s
+// unchanged.
+func trimPrefixFold(s, prefix string) string {
+	if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
+		return ""
+	}
+
+	offset := 0
+	for prefix != "" {
+		if offset >= len(s) {
+			return s
+		}
+		sRune, sSize := utf8.DecodeRuneInString(s[offset:])
+		prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
+		if !strings.EqualFold(string(sRune), string(prefixRune)) {
+			return s
+		}
+		offset += sSize
+		prefix = prefix[prefixSize:]
+	}
+	return s[offset:]
+}
+
 func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
 	cursor := startCursor
 	var res *files.ListFolderLongpollResult
@@ -1693,7 +1726,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			Timeout: timeout,
 		}
 
-		res, err = f.svc.ListFolderLongpoll(&args)
+		res, err = f.svc.ListFolderLongpollContext(ctx, &args)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1716,7 +1749,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			Cursor: cursor,
 		}
 		err = f.pacer.Call(func() (bool, error) {
-			changeList, err = f.srv.ListFolderContinue(&arg)
+			changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1729,17 +1762,18 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			switch info := entry.(type) {
 			case *files.FolderMetadata:
 				entryType = fs.EntryDirectory
-				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
+				entryPath = info.PathDisplay
 			case *files.FileMetadata:
 				entryType = fs.EntryObject
-				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
+				entryPath = info.PathDisplay
 			case *files.DeletedMetadata:
 				entryType = fs.EntryObject
-				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
+				entryPath = info.PathDisplay
 			default:
 				fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
 				continue
 			}
+			entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
 
 			if entryPath != "" {
 				notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
@@ -1851,7 +1885,7 @@ func (o *Object) Size() int64 {
 	return o.bytes
 }
 
-func (o *Object) setMetadataForExport(info *files.FileMetadata) {
+func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
 	o.bytes = -1
 	o.hash = ""
 
@@ -1872,8 +1906,10 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata) {
 		o.exportType = exportExportable
 		// get rid of any paper extension, if present
 		o.remote = strings.TrimSuffix(o.remote, paperExtension)
-		// add the export extension
-		o.remote += "." + string(exportExt)
+		if !remoteIsExportPath {
+			// add the export extension
+			o.remote += "." + string(exportExt)
+		}
 	}
 }
 
@@ -1881,19 +1917,23 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata) {
 //
 // This isn't a complete set of metadata and has an inaccurate date
 func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
+	return o.setMetadataFromEntryWithExportPath(info, false)
+}
+
+func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
 	o.id = info.Id
 	o.bytes = int64(info.Size)
 	o.modTime = time.Time(info.ClientModified)
 	o.hash = info.ContentHash
 
 	if !info.IsDownloadable {
-		o.setMetadataForExport(info)
+		o.setMetadataForExport(info, remoteIsExportPath)
 	}
 	return nil
 }
 
 // Reads the entry for a file from dropbox
-func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
+func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
 	return o.fs.getFileMetadata(ctx, o.remotePath())
 }
 
@@ -1903,11 +1943,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	entry, err := o.readEntry(ctx)
+	entry, remoteIsExportPath, err := o.readEntry(ctx)
 	if err != nil {
 		return err
 	}
-	return o.setMetadataFromEntry(entry)
+	return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
 }
 
 // Returns the remote path for the object
@@ -1961,7 +2001,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
 	arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
 	var exportResult *files.ExportResult
 	err = o.fs.pacer.Call(func() (bool, error) {
-		exportResult, in, err = o.fs.srv.Export(&arg)
+		exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1983,7 +2023,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			Url: o.url,
 		}
 		err = o.fs.pacer.Call(func() (bool, error) {
-			_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
+			_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -2003,7 +2043,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		ExtraHeaders: headers,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, in, err = o.fs.srv.Download(&arg)
+		_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 
@@ -2027,7 +2067,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 	// start upload
 	var res *files.UploadSessionStartResult
 	err = o.fs.pacer.Call(func() (bool, error) {
-		res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
+		res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -2073,7 +2113,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 			if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
 				return false, err
 			}
-			err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
+			err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, chunk)
 			// after session is started, we retry everything
 			if err != nil {
 				// Check for incorrect offset error and retry with new offset
@@ -2098,11 +2138,23 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 					}
 				}
 			}
+			// Don't waste the low level retries if the context has
+			// been cancelled
+			if fserrors.ContextError(ctx, &err) {
+				return false, err
+			}
 			return err != nil, err
 		})
 		if err != nil {
 			return nil, err
 		}
+		if size >= 0 {
+			// Check for sources which truncate early
+			expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
+			if in.BytesRead() < expected {
+				return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
+			}
+		}
 		if appendArg.Close {
 			break
 		}
@@ -2132,7 +2184,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		entry, err = o.fs.srv.UploadSessionFinish(args, nil)
+		entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
 		if retry, err := shouldRetryExclude(ctx, err); !retry {
 			return retry, err
 		}
@@ -2200,7 +2252,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		entry, err = o.uploadChunked(ctx, in, commitInfo, size)
 	} else {
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
+			entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
 			return shouldRetry(ctx, err)
 		})
 	}
@@ -2225,7 +2277,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		return errNotSupportedInSharedMode
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
+		_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
 			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
 		})
 		return shouldRetry(ctx, err)

backend/dropbox/dropbox_internal_test.go

@@ -2,17 +2,155 @@ package dropbox
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/batcher"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+type paperMetadataClient struct {
+	files.ContextClient
+	info *files.FileMetadata
+}
+
+func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
+	if arg.Path == "document" {
+		return c.info, nil
+	}
+	return nil, files.GetMetadataAPIError{
+		APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
+		EndpointError: &files.GetMetadataError{
+			Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
+			Path: &files.LookupError{
+				Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
+			},
+		},
+	}
+}
+
+func TestInternalGetMetadataCancellation(t *testing.T) {
+	requestStarted := make(chan struct{})
+	releaseRequest := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		close(requestStarted)
+		<-releaseRequest
+	}))
+	defer server.Close()
+	defer close(releaseRequest)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	f := &Fs{
+		srv: files.NewContext(dropbox.Config{
+			Client: server.Client(),
+			URLGenerator: func(hostType string, namespace string, route string) string {
+				return server.URL
+			},
+		}),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
+	}
+
+	result := make(chan getMetadataResult, 1)
+	go func() {
+		result <- f.getMetadata(ctx, "/file")
+	}()
+
+	select {
+	case <-requestStarted:
+	case <-time.After(time.Second):
+		t.Fatal("Dropbox request did not start")
+	}
+	cancel()
+
+	select {
+	case res := <-result:
+		require.ErrorIs(t, res.err, context.Canceled)
+	case <-time.After(time.Second):
+		t.Fatal("Dropbox request did not observe cancellation")
+	}
+}
+
+type changeNotifyClient struct {
+	files.ContextClient
+	entries []files.IsMetadata
+}
+
+func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
+	return files.NewListFolderLongpollResult(true), nil
+}
+
+func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
+	return files.NewListFolderResult(c.entries, "next", false), nil
+}
+
+func TestTrimPrefixFold(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		path   string
+		prefix string
+		want   string
+	}{
+		{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
+		{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
+		{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
+		{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
+		{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
+		{name: "root remote", path: "/File", prefix: "/", want: "File"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
+		})
+	}
+}
+
+func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
+	ctx := context.Background()
+	client := changeNotifyClient{entries: []files.IsMetadata{
+		&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
+		&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
+		&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
+	}}
+	f := &Fs{
+		ci:             fs.GetConfig(ctx),
+		srv:            client,
+		svc:            client,
+		slashRootSlash: "/Docs/",
+		pacer:          fs.NewPacer(ctx, pacer.NewDefault()),
+	}
+
+	type notification struct {
+		path      string
+		entryType fs.EntryType
+	}
+	var notifications []notification
+	cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
+		notifications = append(notifications, notification{path: path, entryType: entryType})
+	}, "start")
+	require.NoError(t, err)
+	assert.Equal(t, "next", cursor)
+	assert.Equal(t, []notification{
+		{path: "Sub", entryType: fs.EntryDirectory},
+		{path: "Sub/File.TXT", entryType: fs.EntryObject},
+		{path: "Gone.md", entryType: fs.EntryObject},
+	}, notifications)
+}
+
 func TestInternalCheckPathLength(t *testing.T) {
 	rep := func(n int, r rune) (out string) {
 		rs := make([]rune, n)
@@ -50,6 +188,170 @@ func TestInternalCheckPathLength(t *testing.T) {
 	}
 }
 
+func TestInternalSharedFolderName(t *testing.T) {
+	for _, test := range []struct {
+		root string
+		want string
+	}{
+		{root: "", want: ""},
+		{root: "SharedFolder", want: "SharedFolder"},
+		{root: "SharedFolder/subdir", want: "SharedFolder"},
+		{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
+		{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
+	} {
+		assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
+	}
+}
+
+func TestPaperExportRemote(t *testing.T) {
+	ctx := context.Background()
+	info := &files.FileMetadata{
+		ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
+	}
+	f := &Fs{
+		exportExts: []exportExtension{"md"},
+		pacer:      fs.NewPacer(ctx, pacer.NewDefault()),
+		srv:        paperMetadataClient{info: info},
+	}
+
+	direct, err := f.NewObject(ctx, "document.md")
+	require.NoError(t, err)
+	assert.Equal(t, "document.md", direct.Remote())
+
+	listed, err := f.newObjectWithInfo(ctx, "document.md", info)
+	require.NoError(t, err)
+	assert.Equal(t, "document.md.md", listed.Remote())
+
+	legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
+	require.NoError(t, err)
+	assert.Equal(t, "document.md", legacy.Remote())
+}
+
+// uploadSessionClient is a mock files.ContextClient which records the
+// chunked upload calls made to it
+type uploadSessionClient struct {
+	files.ContextClient
+	appends      int    // number of UploadSessionAppendV2Context calls
+	maxAppends   int    // fail the append after this many calls to stop runaway loops
+	bytesWritten int64  // bytes received by UploadSessionAppendV2Context
+	finishCalled bool   // set if UploadSessionFinishContext was called
+	appended     func() // if set, called after each successful append
+}
+
+var errTooManyAppends = errors.New("too many appends - upload looping?")
+
+func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
+	return &files.UploadSessionStartResult{SessionId: "session"}, nil
+}
+
+func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
+	// the real client fails the request if the context is cancelled
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	c.appends++
+	if c.appends > c.maxAppends {
+		return errTooManyAppends
+	}
+	n, err := io.Copy(io.Discard, content)
+	if err != nil {
+		return err
+	}
+	c.bytesWritten += n
+	if c.appended != nil {
+		c.appended()
+	}
+	return nil
+}
+
+func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
+	c.finishCalled = true
+	return &files.FileMetadata{}, nil
+}
+
+// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
+func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
+	ctx := context.Background()
+	f := &Fs{
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
+		srv:   srv,
+	}
+	f.opt.ChunkSize = chunkSize
+	batcherOptions := defaultBatcherOptions
+	batcherOptions.Mode = "off"
+	var err error
+	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
+	require.NoError(t, err)
+	return f
+}
+
+// endlessReader supplies bytes forever
+type endlessReader struct{}
+
+func (endlessReader) Read(p []byte) (int, error) {
+	for i := range p {
+		p[i] = 'x'
+	}
+	return len(p), nil
+}
+
+func TestUploadChunkedEarlyEOF(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("MultiChunk", func(t *testing.T) {
+		// The declared size spans 4 chunks but the source ends after 1.5
+		client := &uploadSessionClient{maxAppends: 8}
+		f := newUploadTestFs(t, client, 100)
+		o := &Object{fs: f, remote: "test.bin"}
+		_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
+		require.Error(t, err)
+		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+		assert.False(t, client.finishCalled, "must not commit a truncated upload")
+	})
+
+	t.Run("SingleChunk", func(t *testing.T) {
+		// The declared size fits in one chunk but the source ends early
+		client := &uploadSessionClient{maxAppends: 8}
+		f := newUploadTestFs(t, client, 500)
+		o := &Object{fs: f, remote: "test.bin"}
+		_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
+		require.Error(t, err)
+		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+		assert.False(t, client.finishCalled, "must not commit a truncated upload")
+	})
+
+	t.Run("Complete", func(t *testing.T) {
+		// A source which supplies exactly the declared size uploads OK
+		client := &uploadSessionClient{maxAppends: 8}
+		f := newUploadTestFs(t, client, 100)
+		o := &Object{fs: f, remote: "test.bin"}
+		entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
+		require.NoError(t, err)
+		require.NotNil(t, entry)
+		assert.True(t, client.finishCalled)
+		assert.Equal(t, int64(250), client.bytesWritten)
+	})
+}
+
+func TestUploadChunkedCancel(t *testing.T) {
+	// Cancelling the context must stop the upload even though every
+	// append is succeeding
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	client := &uploadSessionClient{maxAppends: 8}
+	client.appended = func() {
+		if client.appends == 2 {
+			cancel()
+		}
+	}
+	f := newUploadTestFs(t, client, 100)
+	o := &Object{fs: f, remote: "test.bin"}
+	_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, context.Canceled)
+	assert.False(t, client.finishCalled)
+}
+
 func (f *Fs) importPaperForTest(t *testing.T) {
 	content := `# test doc
 
@@ -64,7 +366,7 @@ Lorem ipsum __dolor__ sit amet
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		reader := strings.NewReader(content)
-		_, err = f.srv.PaperCreate(&arg, reader)
+		_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
 		return shouldRetry(context.Background(), err)
 	})
 	require.NoError(t, err)
@@ -100,3 +402,136 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
+// local mock server instead of the real Dropbox API.
+func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
+	t.Helper()
+	ts := httptest.NewServer(handler)
+	t.Cleanup(ts.Close)
+
+	cfg := dropbox.Config{
+		Client: &http.Client{Transport: http.DefaultTransport},
+		URLGenerator: func(hostType, namespace, route string) string {
+			return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
+		},
+	}
+
+	return &Fs{
+		opt: Options{
+			Enc: encoder.Base |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDel |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8,
+		},
+		sharing: sharing.NewContext(cfg),
+		pacer:   fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
+	}
+}
+
+func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
+	t.Helper()
+	tm, err := time.Parse(time.RFC3339, s)
+	require.NoError(t, err)
+	return (*dropbox.DBXTime)(&tm)
+}
+
+func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.URL.Path, "list_received_files") {
+			http.NotFound(w, r)
+			return
+		}
+		resp := sharing.ListFilesResult{
+			Entries: []*sharing.SharedFileMetadata{{
+				Id:          "id:123",
+				Name:        rawName,
+				PreviewUrl:  "https://dropbox.com/preview/123",
+				Policy:      &sharing.FolderPolicy{},
+				TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
+			}},
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(resp)
+	}
+}
+
+// TestListReceivedFilesDecodesName confirms received-file names are decoded
+// via Enc.ToStandardName before being returned, matching listSharedFolders'
+// handling of shared-folder names.
+func TestListReceivedFilesDecodesName(t *testing.T) {
+	const rawName = "report.txt␠"
+	f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
+
+	wantName := f.opt.Enc.ToStandardName(rawName)
+	require.NotEqual(t, rawName, wantName)
+
+	var gotNames []string
+	err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
+		gotNames = append(gotNames, entry.Remote())
+		return nil
+	})
+	require.NoError(t, err)
+	require.Len(t, gotNames, 1)
+	assert.Equal(t, wantName, gotNames[0])
+}
+
+// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
+// a received file by its standard, decoded rclone-visible name.
+func TestFindSharedFileResolvesDecodedName(t *testing.T) {
+	const rawName = "report.txt␠"
+	f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
+
+	encodedName := f.opt.Enc.ToStandardName(rawName)
+	require.NotEqual(t, rawName, encodedName)
+
+	o, err := f.findSharedFile(context.Background(), encodedName)
+	require.NoError(t, err)
+	assert.Equal(t, encodedName, o.remote)
+}
+
+// sharedFoldersHandler serves a single shared folder from list_folders.
+func sharedFoldersHandler(name, id string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.URL.Path, "list_folders") {
+			http.NotFound(w, r)
+			return
+		}
+		resp := sharing.ListFoldersResult{
+			Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(resp)
+	}
+}
+
+// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
+// lookup must match regardless of the case of the requested name.
+func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
+	f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
+
+	for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
+		id, err := f.findSharedFolder(context.Background(), name)
+		require.NoError(t, err, name)
+		assert.Equal(t, "folder-id", id, name)
+	}
+
+	_, err := f.findSharedFolder(context.Background(), "no-such-folder")
+	assert.ErrorIs(t, err, fs.ErrorDirNotFound)
+}
+
+// The Dropbox backend advertises CaseInsensitive: true, so a received file
+// lookup must match regardless of the case of the requested name.
+func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
+	f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
+
+	for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
+		o, err := f.findSharedFile(context.Background(), name)
+		require.NoError(t, err, name)
+		assert.Equal(t, "TestFile.txt", o.remote, name)
+	}
+
+	_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
+	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}

backend/filelu/filelu_file_uploader.go

@@ -25,7 +25,9 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
 	}
 
 	if dir != "" {
-		_ = f.Mkdir(ctx, dir)
+		if _, err := f.createFolder(ctx, dir); err != nil {
+			return fmt.Errorf("failed to create multipart folder: %w", err)
+		}
 	}
 
 	folder := strings.Trim(dir, "/")

backend/filelu/filelu_object.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -141,6 +142,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 // Update updates the object with new data
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	size := src.Size()
+	counter := readers.NewCountingReader(in)
+	in = counter
 
 	if size <= int64(o.fs.opt.UploadCutoff) {
 		err := o.fs.uploadFile(ctx, in, o.remote)
@@ -155,6 +158,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload
+	if size >= 0 && int64(counter.BytesRead()) != size {
+		if removeErr := o.Remove(ctx); removeErr != nil {
+			fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
+		}
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	o.size = size
 	o.modTime = src.ModTime(ctx)
 	return nil

backend/filescom/filescom.go

@@ -19,6 +19,7 @@ import (
 	"github.com/Files-com/files-sdk-go/v3/file"
 	file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
 	"github.com/Files-com/files-sdk-go/v3/folder"
+	files_sdk_lib "github.com/Files-com/files-sdk-go/v3/lib"
 	"github.com/Files-com/files-sdk-go/v3/session"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -176,6 +177,13 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
 		}
 	}
 
+	// Errors from the upload storage servers are of this type
+	var httpErr files_sdk_lib.ResponseError
+	if errors.As(err, &httpErr) && slices.Contains(retryErrorCodes, httpErr.StatusCode) {
+		fs.Debugf(nil, "Retrying HTTP error %v", err)
+		return true, err
+	}
+
 	return fserrors.ShouldRetry(err), err
 }
 
@@ -858,7 +866,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		file.UploadWithProvidedMtime(src.ModTime(ctx)),
 	}
 
-	err := o.fs.pacer.Call(func() (bool, error) {
+	err := o.fs.pacer.CallNoRetry(func() (bool, error) {
 		err := o.fs.fileClient.Upload(uploadOpts...)
 		return shouldRetry(ctx, err)
 	})

backend/http/http.go

@@ -57,7 +57,13 @@ The input format is comma separated list of key,value pairs.  Standard
 
 For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
 
-You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.`,
+You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
+
+The headers are only sent to the host in the configured URL. If the
+server redirects to another host (including a subdomain or a different
+port) the headers are not sent to it, or to any further hop in that
+redirect chain. When headers are set, a redirect from https to http is
+refused as it would send them in cleartext.`,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
 		}, {
@@ -277,12 +283,17 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	if err != nil {
 		return false, err
 	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
+	u, err := rest.URLJoinRoot(base, f.root)
 	if err != nil {
 		return false, err
 	}
 
 	client := fshttp.NewClient(ctx)
+	// Without configured headers keep the default policy which
+	// public mirrors that redirect from https to http rely on.
+	if len(opt.Headers) > 0 {
+		client.CheckRedirect = checkRedirect(opt)
+	}
 
 	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
 	fs.Debugf(nil, "Root: %s", endpoint)
@@ -510,6 +521,25 @@ func addHeaders(req *http.Request, opt *Options) {
 	}
 }
 
+// checkRedirect returns an http.Client.CheckRedirect function which
+// follows redirects but refuses an HTTPS to HTTP downgrade with
+// rest.ErrHTTPSDowngrade and strips the configured headers when the
+// redirect chain has left the originally requested host at any point.
+func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
+	headers := make([]string, 0, len(opt.Headers)/2)
+	for i := 0; i < len(opt.Headers); i += 2 {
+		headers = append(headers, opt.Headers[i])
+	}
+	stripHeaders := rest.StripHeadersOnCrossHostRedirectFn(headers...)
+	return func(req *http.Request, via []*http.Request) error {
+		err := stripHeaders(req, via)
+		if errors.Is(err, rest.ErrHTTPSDowngrade) {
+			err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
+		}
+		return err
+	}
+}
+
 // Adds the configured headers to the request if any
 func (f *Fs) addHeaders(req *http.Request) {
 	addHeaders(req, &f.opt)

backend/http/http_internal_test.go

@@ -581,3 +581,213 @@ func TestFsNoSlashRoots(t *testing.T) {
 		}
 	}
 }
+
+// readRedirected makes an http remote with the configured headers
+// pointing at url and reads file.txt from it, checking the content
+func readRedirected(t *testing.T, url string) {
+	configfile.Install()
+	m := configmap.Simple{
+		"type":    "http",
+		"url":     url + "/",
+		"headers": strings.Join(headers, ","),
+	}
+	f, err := NewFs(context.Background(), remoteName, "", m)
+	require.NoError(t, err)
+
+	o, err := f.NewObject(context.Background(), "file.txt")
+	require.NoError(t, err)
+	fd, err := o.Open(context.Background())
+	require.NoError(t, err)
+	data, err := io.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	assert.Equal(t, "hello", string(data))
+}
+
+func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
+	var got http.Header
+	var ts *httptest.Server
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
+			return
+		}
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer ts.Close()
+
+	readRedirected(t, ts.URL)
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Equal(t, headers[i+1], got.Get(headers[i]))
+	}
+}
+
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+	// A plaintext server to be redirected to
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer target.Close()
+
+	// An HTTPS server which redirects to plaintext HTTP
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
+	}))
+	defer ts.Close()
+
+	// open makes a remote pointing at ts with or without headers
+	// and opens file.txt on it
+	open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
+		configfile.Install()
+		m := configmap.Simple{
+			"type":    "http",
+			"url":     ts.URL + "/",
+			"no_head": "true",
+		}
+		if withHeaders {
+			m.Set("headers", strings.Join(headers, ","))
+		}
+		ctx, ci := fs.AddConfig(context.Background())
+		ci.InsecureSkipVerify = true
+		f, err := NewFs(ctx, remoteName, "", m)
+		require.NoError(t, err)
+
+		// no_head means NewObject doesn't make a request so use Open to make one
+		o, err := f.NewObject(ctx, "file.txt")
+		require.NoError(t, err)
+		return o.Open(ctx)
+	}
+
+	t.Run("WithHeaders", func(t *testing.T) {
+		// Headers could leak so the downgrade must be refused
+		_, err := open(t, true)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithoutHeaders", func(t *testing.T) {
+		// Nothing to leak so the downgrade is followed
+		fd, err := open(t, false)
+		require.NoError(t, err)
+		data, err := io.ReadAll(fd)
+		require.NoError(t, err)
+		require.NoError(t, fd.Close())
+		assert.Equal(t, "hello", string(data))
+	})
+}
+
+func TestRedirectKeepsHeadersOnSameHostDifferentCase(t *testing.T) {
+	// The redirect spells the host differently but it is the same
+	// host so the headers must be kept
+	var got http.Header
+	var ts *httptest.Server
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, "http://LOCALHOST:"+port(t, ts)+"/moved"+r.URL.Path, http.StatusFound)
+			return
+		}
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer ts.Close()
+
+	readRedirected(t, "http://localhost:"+port(t, ts))
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Equal(t, headers[i+1], got.Get(headers[i]))
+	}
+}
+
+// port returns the port ts is listening on
+func port(t *testing.T, ts *httptest.Server) string {
+	u, err := url.Parse(ts.URL)
+	require.NoError(t, err)
+	return u.Port()
+}
+
+func TestRedirectStripsHeadersAfterHostChange(t *testing.T) {
+	// The configured server redirects to a second server which
+	// redirects back to the configured server. The headers must not
+	// be sent on the way back as the other host chose the path.
+	var got http.Header
+	var ts *httptest.Server
+	bouncer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
+	}))
+	defer bouncer.Close()
+
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, bouncer.URL+r.URL.Path, http.StatusFound)
+			return
+		}
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer ts.Close()
+
+	readRedirected(t, ts.URL)
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Empty(t, got.Values(headers[i]), "header %q sent after cross-host redirect", headers[i])
+	}
+}
+
+func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
+	// Serve the final response from a second server. httptest servers
+	// listen on 127.0.0.1:port so the two servers have different hosts
+	// for the purposes of the redirect check.
+	var got http.Header
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer target.Close()
+
+	redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
+	}))
+	defer redirector.Close()
+
+	readRedirected(t, redirector.URL)
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
+	}
+}
+
+func TestRootCannotChangeHost(t *testing.T) {
+	// A root starting with "//" is a network-path reference under the
+	// URL standard. It must not be able to move the remote (and the
+	// configured headers) to a host other than the configured one.
+	var requests int
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer other.Close()
+
+	m := prepareServer(t)
+	otherHost := strings.TrimPrefix(other.URL, "http://")
+	for _, root := range []string{
+		"//" + otherHost + "/",
+		"//" + otherHost + "/one%.txt",
+		"//" + otherHost,
+		"//user:pass@" + otherHost + "/",
+	} {
+		f, err := NewFs(context.Background(), remoteName, root, m)
+		assert.ErrorContains(t, err, "must not change the scheme, host or user", root)
+		assert.Nil(t, f, root)
+	}
+	assert.Equal(t, 0, requests, "request sent to the other host")
+
+	// A same-origin absolute path must still work
+	f, err := NewFs(context.Background(), remoteName, "/", m)
+	require.NoError(t, err)
+	assert.Equal(t, m["url"]+"/", f.String())
+}

backend/huaweidrive/huaweidrive.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/oauth2"
 )
@@ -1771,11 +1772,18 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	}
 
 	// Read and write file content
-	_, err = io.Copy(fileWriter, in)
+	counter := readers.NewCountingReader(in)
+	_, err = io.Copy(fileWriter, counter)
 	if err != nil {
 		return fmt.Errorf("failed to copy file content: %w", err)
 	}
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload.
+	if int64(counter.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	err = writer.Close()
 	if err != nil {
 		return fmt.Errorf("failed to close multipart writer: %w", err)

backend/iclouddrive/api/drive.go

@@ -498,7 +498,7 @@ func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*Dri
 }
 
 // CreateUpload creates an url for an upload.
-func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string) (*UploadResponse, *http.Response, error) {
+func (d *DriveService) CreateUpload(ctx context.Context, size int64, name, zone string) (*UploadResponse, *http.Response, error) {
 	// first we need to request an upload url
 	values := map[string]any{
 		"filename":     name,
@@ -513,7 +513,7 @@ func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string
 
 	opts := rest.Opts{
 		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/upload/web",
+		Path:         "/ws/" + zone + "/upload/web",
 		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
 		RootURL:      d.docsEndpoint,
 		Body:         body,
@@ -552,14 +552,14 @@ func (d *DriveService) Upload(ctx context.Context, in io.Reader, size int64, nam
 // ctx: the context.Context object for the request.
 // r: a pointer to the UpdateFileInfo struct containing the information for the file update.
 // Returns a pointer to the DriveItem struct representing the updated file, the http.Response object, and an error if any.
-func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*DriveItem, *http.Response, error) {
+func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo, zone string) (*DriveItem, *http.Response, error) {
 	body, err := IntoReader(r)
 	if err != nil {
 		return nil, nil, err
 	}
 	opts := rest.Opts{
 		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/update/documents",
+		Path:         "/ws/" + zone + "/update/documents",
 		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
 		RootURL:      d.docsEndpoint,
 		Body:         body,
@@ -572,7 +572,7 @@ func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*Driv
 
 	doc := responseInfo.Results[0].Document
 	item := DriveItem{
-		Drivewsid:    "FILE::com.apple.CloudDocs::" + doc.DocumentID,
+		Drivewsid:    ConstructDriveID(doc.DocumentID, zone, "FILE"),
 		Docwsid:      doc.DocumentID,
 		Itemid:       doc.ItemID,
 		Etag:         doc.Etag,
@@ -892,6 +892,18 @@ func DeconstructDriveID(id string) (docType, zone, docid string) {
 	return split[0], split[1], split[2]
 }
 
+// ZoneFromDriveID returns the zone a drive ID belongs to, or the default zone
+// for an ID that carries none. Items in an app container -- Obsidian, Pages,
+// Shortcuts -- live in that app's zone rather than com.apple.CloudDocs, and a
+// write addressed to the wrong zone cannot resolve its parent.
+func ZoneFromDriveID(id string) string {
+	_, zone, _ := DeconstructDriveID(id)
+	if zone == "" {
+		return defaultZone
+	}
+	return zone
+}
+
 // ConstructDriveID constructs a drive ID from the given components.
 func ConstructDriveID(id string, zone string, t string) string {
 	return strings.Join([]string{t, zone, id}, "::")

backend/iclouddrive/api/drive_test.go

@@ -0,0 +1,32 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestZoneFromDriveID(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		id   string
+		want string
+	}{
+		{"app container", "FOLDER::iCloud.md.obsidian::documents#o2v", "iCloud.md.obsidian"},
+		{"another app container", "FOLDER::com.apple.Pages::documents#7qt", "com.apple.Pages"},
+		{"ordinary folder", "FOLDER::com.apple.CloudDocs::B847FE2D", "com.apple.CloudDocs"},
+		{"file in a container", "FILE::iCloud.md.obsidian::abc123", "iCloud.md.obsidian"},
+		{"no zone", "root", defaultZone},
+		{"empty", "", defaultZone},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.want, ZoneFromDriveID(test.id))
+		})
+	}
+}
+
+func TestZoneFromDriveIDRoundTrip(t *testing.T) {
+	id := ConstructDriveID("abc123", "iCloud.md.obsidian", "FILE")
+	assert.Equal(t, "FILE::iCloud.md.obsidian::abc123", id)
+	assert.Equal(t, "iCloud.md.obsidian", ZoneFromDriveID(id))
+}

backend/iclouddrive/iclouddrive.go

@@ -92,7 +92,7 @@ func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, fou
 		item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir))
 		return shouldRetry(ctx, resp, err)
 	}); err != nil {
-		if item == nil && resp.StatusCode == 404 {
+		if item == nil && resp != nil && resp.StatusCode == 404 {
 			return nil, false, nil
 		}
 		return nil, false, err
@@ -343,7 +343,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	var item *api.DriveItem
 	if err = f.pacer.Call(func() (bool, error) {
-		item, resp, err = f.service.UpdateFile(ctx, &r)
+		item, resp, err = f.service.UpdateFile(ctx, &r, api.ZoneFromDriveID(pathID))
 		return retryResultUnknown(ctx, resp, err)
 	}); err != nil {
 		return nil, err
@@ -949,7 +949,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Create document
 	var uploadInfo *api.UploadResponse
 	if err = o.fs.pacer.Call(func() (bool, error) {
-		uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name)
+		uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name, api.ZoneFromDriveID(dirID))
 		return ignoreResultUnknown(ctx, resp, err)
 	}); err != nil {
 		return err
@@ -988,7 +988,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Update metadata
 	var item *api.DriveItem
 	if err = o.fs.pacer.Call(func() (bool, error) {
-		item, resp, err = o.fs.service.UpdateFile(ctx, &r)
+		item, resp, err = o.fs.service.UpdateFile(ctx, &r, api.ZoneFromDriveID(dirID))
 		return ignoreResultUnknown(ctx, resp, err)
 	}); err != nil {
 		return err

backend/imagekit/imagekit.go

@@ -477,7 +477,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 			FolderPath: f.EncodePath(path.Join(f.root, dir)),
 		})
 
-		if res.StatusCode == http.StatusNotFound {
+		if res != nil && res.StatusCode == http.StatusNotFound {
 			return false, fs.ErrorDirNotFound
 		}
 
@@ -502,7 +502,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 			FolderPath: f.EncodePath(remote),
 		})
 
-		if res.StatusCode == http.StatusNotFound {
+		if res != nil && res.StatusCode == http.StatusNotFound {
 			return false, fs.ErrorDirNotFound
 		}
 

backend/internetarchive/internetarchive.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -854,11 +855,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	// make a PUT request at (IAS3)/encoded(:item/:path)
+	counter := readers.NewCountingReader(in)
 	var resp *http.Response
 	opts := rest.Opts{
 		Method:        "PUT",
 		Path:          "/" + url.PathEscape(path.Join(bucket, bucketPath)),
-		Body:          in,
+		Body:          counter,
 		ContentLength: &size,
 		ExtraHeaders:  headers,
 	}
@@ -868,6 +870,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return o.fs.shouldRetry(resp, err)
 	})
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload
+	if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
+		err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	// we can't update/find metadata here as IA will "ingest" uploaded file(s)
 	// upon uploads. (you can find its progress at https://archive.org/history/ItemNameHere )
 	// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)

backend/internxt/auth.go

@@ -28,6 +28,7 @@ type userInfo struct {
 	Bucket       string
 	BridgeUser   string
 	UserID       string
+	NewToken     string
 }
 
 type userInfoConfig struct {
@@ -61,6 +62,7 @@ func getUserInfo(ctx context.Context, cfg *userInfoConfig) (*userInfo, error) {
 		Bucket:       resp.User.Bucket,
 		BridgeUser:   resp.User.BridgeUser,
 		UserID:       resp.User.UserID,
+		NewToken:     resp.NewToken,
 	}
 
 	fs.Debugf(nil, "User info: rootFolderId=%s, bucket=%s",

backend/internxt/internxt.go

@@ -367,6 +367,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.bridgeUser = userInfo.BridgeUser
 	f.userID = userInfo.UserID
 
+	// The refresh endpoint rotates the token on every successful call.
+	// Persist the rotated token so routine use keeps the stored token
+	// current; otherwise it keeps its original expiry and accounts that
+	// cannot re-login non-interactively (2FA) eventually strand.
+	if userInfo.NewToken != "" {
+		if rotated, rotErr := jwtToOAuth2Token(userInfo.NewToken); rotErr != nil {
+			fs.Debugf(f, "Not adopting rotated token from user info: %v", rotErr)
+		} else {
+			// Use the rotated token for this session even if saving it
+			// fails; persistence is best-effort.
+			f.cfg.Token = userInfo.NewToken
+			if putErr := oauthutil.PutToken(name, m, rotated, false); putErr != nil {
+				fs.Debugf(f, "Failed to save rotated token from user info: %v", putErr)
+			} else {
+				fs.Debugf(f, "Persisted rotated token from user info, expiry: %v", rotated.Expiry)
+			}
+		}
+	}
+
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
@@ -456,8 +475,10 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	if len(childFolders) > 0 {
-		return fs.ErrorDirectoryNotEmpty
+	for _, e := range childFolders {
+		if !recent.stale(e.UUID, id, e.PlainName) {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 
 	var childFiles []folders.File
@@ -469,8 +490,10 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	if len(childFiles) > 0 {
-		return fs.ErrorDirectoryNotEmpty
+	for _, e := range childFiles {
+		if !recent.stale(e.UUID, id, joinNameExt(e.PlainName, e.Type)) {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 
 	// Delete the directory
@@ -484,6 +507,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
+	recent.deleted(id)
 
 	f.dirCache.FlushDir(dir)
 	return nil
@@ -502,6 +526,9 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, e
 		return "", false, err
 	}
 	for _, e := range entries {
+		if recent.stale(e.UUID, pathID, e.PlainName) {
+			continue
+		}
 		if f.opt.Encoding.ToStandardName(e.PlainName) == leaf {
 			return e.UUID, true, nil
 		}
@@ -538,46 +565,166 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)
 	return resp.UUID, nil
 }
 
-// preUploadCheck checks if a file exists in the given directory
-// Returns the file metadata if it exists, nil if not
-func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
-	// Parse name and extension from the leaf
-	baseName := f.opt.Encoding.FromStandardName(leaf)
-	name := strings.TrimSuffix(baseName, path.Ext(baseName))
-	ext := strings.TrimPrefix(path.Ext(baseName), ".")
+// splitNameExt splits a basename into the (plainName, type) pair that Internxt
+// stores a file under.
+func splitNameExt(baseName string) (name, ext string) {
+	ext = path.Ext(baseName)
+	if ext == baseName {
+		return baseName, ""
+	}
+	return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")
+}
 
-	checkResult, err := files.CheckFilesExistence(ctx, f.cfg, directoryID, []files.FileExistenceCheck{
-		{
-			PlainName:    name,
-			Type:         ext,
-			OriginalFile: struct{}{},
-		},
+// joinNameExt reassembles the basename a stored (plainName, type) pair
+// displays as, the way List builds its entries.
+func joinNameExt(name, ext string) string {
+	if ext == "" {
+		return name
+	}
+	return name + "." + ext
+}
+
+// legacyNameExt splits a basename at the final dot, the way this backend used
+// to before it adopted the convention the other clients share. It differs from
+// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into
+// an empty name of type "bashrc".
+func legacyNameExt(baseName string) (name, ext string) {
+	return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")
+}
+
+// existenceCheck builds a lookup criterion for one (plainName, type) spelling.
+func existenceCheck(name, ext string) files.FileExistenceCheck {
+	return files.FileExistenceCheck{
+		PlainName:    name,
+		Type:         ext,
+		OriginalFile: struct{}{},
+	}
+}
+
+// staleWindow is how long the API may carry on returning an item from
+// where it was before this process moved or deleted it.
+const staleWindow = time.Minute
+
+// location is where this process last put an item. An empty parentUUID
+// means the item was deleted.
+type location struct {
+	parentUUID string
+	name       string // encoded name, including any extension
+	expires    time.Time
+}
+
+// recentChanges holds the files and folders this process has moved or
+// deleted recently, keyed by UUID.
+//
+// The API serves lookups and listings from read replicas which lag behind
+// writes, so an item can be returned from its old location for a while after
+// it was moved or deleted. Acting on such a stale entry can be destructive,
+// eg overwriting it would delete the file from where it was moved to.
+type recentChanges struct {
+	mu        sync.Mutex
+	items     map[string]location
+	lastPrune time.Time
+}
+
+// recent is shared by all the Fs in the process as UUIDs are global.
+var recent = recentChanges{items: map[string]location{}}
+
+// set records the new location of uuid, pruning expired entries.
+func (r *recentChanges) set(uuid string, loc location) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	now := time.Now()
+	if now.Sub(r.lastPrune) > staleWindow {
+		for k, v := range r.items {
+			if now.After(v.expires) {
+				delete(r.items, k)
+			}
+		}
+		r.lastPrune = now
+	}
+	loc.expires = now.Add(staleWindow)
+	r.items[uuid] = loc
+}
+
+// moved records that uuid was moved to name in parentUUID.
+func (r *recentChanges) moved(uuid, parentUUID, name string) {
+	r.set(uuid, location{parentUUID: parentUUID, name: name})
+}
+
+// deleted records that uuid was deleted.
+func (r *recentChanges) deleted(uuid string) {
+	r.set(uuid, location{})
+}
+
+// stale reports whether an API result saying uuid is called name in
+// parentUUID is out of date because of a change made by this process.
+func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	loc, ok := r.items[uuid]
+	if !ok || time.Now().After(loc.expires) {
+		return false
+	}
+	return loc.parentUUID != parentUUID || loc.name != name
+}
+
+// findFile looks up a single file by name within directoryID. Returns
+// (nil, nil) when the file does not exist; surfaces transport/API errors.
+func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
+	encodedName := f.opt.Encoding.FromStandardName(leaf)
+	name, ext := splitNameExt(encodedName)
+	checks := []files.FileExistenceCheck{existenceCheck(name, ext)}
+	if encodedName != name {
+		checks = append(checks, existenceCheck(encodedName, ""))
+	}
+	if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {
+		checks = append(checks, existenceCheck(legacyName, legacyExt))
+	}
+
+	var checkResult *files.CheckFilesExistenceResponse
+	err := f.pacer.Call(func() (bool, error) {
+		var err error
+		checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)
+		if isNotFoundError(err) {
+			return true, err
+		}
+		return f.shouldRetry(ctx, err)
 	})
-
 	if err != nil {
-		// If existence check fails, assume file doesn't exist to allow upload to proceed
+		return nil, err
+	}
+
+	var uuid string
+	for _, result := range checkResult.Files {
+		if !result.FileExists() || result.UUID == "" {
+			continue
+		}
+		if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
+			fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
+			continue
+		}
+		if joinNameExt(result.PlainName, result.Type) == encodedName {
+			uuid = result.UUID
+			break
+		}
+	}
+	if uuid == "" {
 		return nil, nil
 	}
 
-	if len(checkResult.Files) > 0 && checkResult.Files[0].FileExists() {
-		result := checkResult.Files[0]
-		if result.Type != ext {
-			return nil, nil
-		}
-
-		existingUUID := result.UUID
-		if existingUUID != "" {
-			fileMeta, err := files.GetFileMeta(ctx, f.cfg, existingUUID)
-			if err == nil && fileMeta != nil {
-				return convertFileMetaToFile(fileMeta), nil
-			}
-
-			if err != nil {
-				return nil, err
-			}
-		}
+	var fileMeta *files.FileMeta
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
 	}
-	return nil, nil
+	if fileMeta == nil {
+		return nil, nil
+	}
+	return convertFileMetaToFile(fileMeta), nil
 }
 
 // convertFileMetaToFile converts files.FileMeta to folders.File
@@ -615,6 +762,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		return nil, err
 	}
 	for _, e := range foldersList {
+		if recent.stale(e.UUID, dirID, e.PlainName) {
+			continue
+		}
 		remote := path.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
 		out = append(out, fs.NewDir(remote, e.ModificationTime))
 	}
@@ -632,6 +782,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		if len(e.Type) > 0 {
 			remote += "." + e.Type
 		}
+		if recent.stale(e.UUID, dirID, remote) {
+			continue
+		}
 		remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
 		out = append(out, newObjectWithFile(f, remote, &e))
 	}
@@ -656,8 +809,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return nil, err
 	}
 
-	// Check if file already exists
-	existingFile, err := f.preUploadCheck(ctx, leaf, directoryID)
+	existingFile, err := f.findFile(ctx, leaf, directoryID)
 	if err != nil {
 		return nil, err
 	}
@@ -705,44 +857,28 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
 	if err != nil {
 		return err
 	}
+	recent.deleted(dirID)
 	f.dirCache.FlushDir(remote)
 	return nil
 }
 
-// NewObject creates a new object
+// NewObject creates a new object by looking up a single file's metadata.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	parentDir := path.Dir(remote)
-
-	if parentDir == "." {
-		parentDir = ""
-	}
-
-	dirID, err := f.dirCache.FindDir(ctx, parentDir, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
-		return nil, fs.ErrorObjectNotFound
+		if err == fs.ErrorDirNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
 	}
-
-	var files []folders.File
-	err = f.pacer.Call(func() (bool, error) {
-		var err error
-		files, err = folders.ListAllFiles(ctx, f.cfg, dirID)
-		return f.shouldRetry(ctx, err)
-	})
+	file, err := f.findFile(ctx, leaf, directoryID)
 	if err != nil {
 		return nil, err
 	}
-	targetName := path.Base(remote)
-	for _, e := range files {
-		name := e.PlainName
-		if len(e.Type) > 0 {
-			name += "." + e.Type
-		}
-		decodedName := f.opt.Encoding.ToStandardName(name)
-		if decodedName == targetName {
-			return newObjectWithFile(f, remote, &e), nil
-		}
+	if file == nil {
+		return nil, fs.ErrorObjectNotFound
 	}
-	return nil, fs.ErrorObjectNotFound
+	return newObjectWithFile(f, remote, file), nil
 }
 
 // newObjectWithFile returns a new object by file info
@@ -846,19 +982,23 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	// Parse name and extension from the leaf
-	baseName := f.opt.Encoding.FromStandardName(leaf)
-	newName := strings.TrimSuffix(baseName, path.Ext(baseName))
-	newType := strings.TrimPrefix(path.Ext(baseName), ".")
+	newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
 
 	// Move the file server-side
 	err = f.pacer.Call(func() (bool, error) {
 		err := files.MoveFile(ctx, f.cfg, srcObj.uuid, directoryID, newName, newType)
+		// The move is checked against lagging read replicas so it can
+		// fail with 404 if the destination directory was just created
+		// or 409 if a file at the destination was just deleted.
+		if err != nil && isStaleReadError(err, true) {
+			return true, err
+		}
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
 	}
+	recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
 
 	dstObj := &Object{
 		f:       f,
@@ -894,11 +1034,17 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	encodedLeaf := f.opt.Encoding.FromStandardName(dstLeaf)
 	err = f.pacer.Call(func() (bool, error) {
 		err := folders.MoveFolder(ctx, f.cfg, srcID, dstDirectoryID, encodedLeaf)
+		// Moving a large folder can outlast the gateway timeout (520/502)
+		// yet still complete, in which case the retry reports this.
+		if err != nil && isConflictError(err) && strings.Contains(err.Error(), "already moved to that location") {
+			return false, nil
+		}
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
 	}
+	recent.moved(srcID, dstDirectoryID, encodedLeaf)
 
 	srcFs.dirCache.FlushDir(srcRemote)
 	return nil
@@ -946,9 +1092,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	remote := o.remote
 
-	origBaseName := path.Base(remote)
-	origName := strings.TrimSuffix(origBaseName, path.Ext(origBaseName))
-	origType := strings.TrimPrefix(path.Ext(origBaseName), ".")
+	origName, origType := splitNameExt(path.Base(remote))
 
 	// Create directory if it doesn't exist
 	_, dirID, err := o.f.dirCache.FindPath(ctx, remote, true)
@@ -965,11 +1109,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Step 1: If file exists, rename to backup (preserves old file during upload)
 	if oldUUID != "" {
-		// Generate unique backup name
-		baseName := path.Base(remote)
-		name := strings.TrimSuffix(baseName, path.Ext(baseName))
-		ext := strings.TrimPrefix(path.Ext(baseName), ".")
-
+		name, ext := splitNameExt(path.Base(remote))
 		backupSuffix := fmt.Sprintf(".rclone-backup-%s", random.String(8))
 		backupName = o.f.opt.Encoding.FromStandardName(name + backupSuffix)
 		backupType = ext
@@ -1042,6 +1182,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			return o.f.tooLargeError(remote, tooLarge)
 		}
 
+		if err != nil && isStaleReadError(err, backupUUID != "") {
+			o.restoreBackupFile(ctx, backupUUID, origName, origType)
+			return fserrors.RetryError(err)
+		}
+
 		if err != nil {
 			meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
 		}
@@ -1080,6 +1225,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				backupName, backupType, backupUUID, err)
 			// Don't fail the upload just because backup deletion failed
 		} else {
+			recent.deleted(backupUUID)
 			fs.Debugf(o.f, "Successfully deleted backup file")
 		}
 	}
@@ -1099,6 +1245,19 @@ func isTimeoutError(err error) bool {
 	return false
 }
 
+// isNotFoundError reports whether err is a 404 from the API.
+func isNotFoundError(err error) bool {
+	var httpErr *sdkerrors.HTTPError
+	return errors.As(err, &httpErr) && httpErr.StatusCode() == 404
+}
+
+// isStaleReadError reports whether err is the API contradicting a write this
+// backend has already made, which it does because lookups are served by read
+// replicas that lag behind the primary.
+func isStaleReadError(err error, renamed bool) bool {
+	return isNotFoundError(err) || (renamed && isConflictError(err))
+}
+
 // isConflictError checks if an error indicates a file conflict (409)
 func isConflictError(err error) bool {
 	errMsg := err.Error()
@@ -1140,40 +1299,22 @@ func (o *Object) recoverFromTimeoutConflict(ctx context.Context, uploadErr error
 	}
 
 	baseName := path.Base(remote)
-	encodedName := o.f.opt.Encoding.FromStandardName(baseName)
 
-	var meta *buckets.CreateMetaResponse
-	checkErr := o.f.pacer.Call(func() (bool, error) {
-		existingFile, err := o.f.preUploadCheck(ctx, encodedName, dirID)
-		if err != nil {
-			return o.f.shouldRetry(ctx, err)
-		}
-		if existingFile != nil {
-			name := strings.TrimSuffix(baseName, path.Ext(baseName))
-			ext := strings.TrimPrefix(path.Ext(baseName), ".")
-
-			meta = &buckets.CreateMetaResponse{
-				UUID:      existingFile.UUID,
-				FileID:    existingFile.FileID,
-				Name:      name,
-				PlainName: name,
-				Type:      ext,
-				Size:      existingFile.Size,
-			}
-			o.id = existingFile.FileID
-		}
-		return false, nil
-	})
-
-	if checkErr != nil {
+	existingFile, err := o.f.findFile(ctx, baseName, dirID)
+	if err != nil || existingFile == nil {
 		return nil, uploadErr
 	}
 
-	if meta != nil {
-		return meta, nil
-	}
-
-	return nil, uploadErr
+	name, ext := splitNameExt(baseName)
+	o.id = existingFile.FileID
+	return &buckets.CreateMetaResponse{
+		UUID:      existingFile.UUID,
+		FileID:    existingFile.FileID,
+		Name:      name,
+		PlainName: name,
+		Type:      ext,
+		Size:      existingFile.Size,
+	}, nil
 }
 
 // restoreBackupFile restores a backup file after upload failure
@@ -1191,8 +1332,13 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or
 
 // Remove deletes a file
 func (o *Object) Remove(ctx context.Context) error {
-	return o.f.pacer.Call(func() (bool, error) {
+	err := o.f.pacer.Call(func() (bool, error) {
 		err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
 		return o.f.shouldRetry(ctx, err)
 	})
+	if err != nil {
+		return err
+	}
+	recent.deleted(o.uuid)
+	return nil
 }

backend/internxt/internxt_internal_test.go

@@ -0,0 +1,191 @@
+package internxt
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/internxt/rclone-adapter/config"
+	"github.com/internxt/rclone-adapter/endpoints"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// storedFile is a file as the service holds it: a (plainName, type) pair
+// split however the client that uploaded it chose to split it.
+type storedFile struct {
+	uuid      string
+	plainName string
+	fileType  string
+}
+
+// wireType renders a stored type the way the service does: a file with no
+// extension comes back as JSON null, not an empty string.
+func wireType(fileType string) any {
+	if fileType == "" {
+		return nil
+	}
+	return fileType
+}
+
+// existenceServer answers the existence check and file meta endpoints from
+// files, matching plainName exactly and only constraining type when the
+// criterion supplies one, as the service does.
+func existenceServer(t *testing.T, stored []storedFile) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(r.URL.Path, "/meta") {
+			uuid := strings.Split(strings.TrimPrefix(r.URL.Path, "/drive/files/"), "/")[0]
+			for _, f := range stored {
+				if f.uuid == uuid {
+					require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+						"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
+					}))
+					return
+				}
+			}
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		var req struct {
+			Files []struct {
+				PlainName string `json:"plainName"`
+				Type      string `json:"type"`
+			} `json:"files"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
+		var existent []map[string]any
+		for _, f := range stored {
+			for _, c := range req.Files {
+				if c.PlainName == f.plainName && (c.Type == "" || c.Type == f.fileType) {
+					existent = append(existent, map[string]any{
+						"exists": true, "status": "EXISTS",
+						"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
+					})
+					break
+				}
+			}
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"existentFiles": existent}))
+	}))
+}
+
+func testFs(t *testing.T, server *httptest.Server) *Fs {
+	t.Helper()
+	f := &Fs{
+		opt: Options{Encoding: encoder.EncodeInvalidUtf8},
+		cfg: &config.Config{
+			HTTPClient: server.Client(),
+			Endpoints:  endpoints.NewConfig(server.URL),
+		},
+	}
+	f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+	return f
+}
+
+func TestSplitJoinNameExt(t *testing.T) {
+	for _, test := range []struct {
+		baseName string
+		name     string
+		ext      string
+	}{
+		{"foo.txt", "foo", "txt"},
+		{"foo.tar.gz", "foo.tar", "gz"},
+		{"README", "README", ""},
+		{".bashrc", ".bashrc", ""},
+		{"..hidden", ".", "hidden"},
+		{"", "", ""},
+	} {
+		name, ext := splitNameExt(test.baseName)
+		assert.Equal(t, test.name, name, test.baseName)
+		assert.Equal(t, test.ext, ext, test.baseName)
+		assert.Equal(t, test.baseName, joinNameExt(name, ext), test.baseName)
+	}
+}
+
+func TestFindFile(t *testing.T) {
+	for _, test := range []struct {
+		what     string
+		stored   []storedFile
+		leaf     string
+		wantUUID string
+	}{{
+		what:     "split as rclone splits it",
+		stored:   []storedFile{{"uuid-1", "foo.tar", "gz"}},
+		leaf:     "foo.tar.gz",
+		wantUUID: "uuid-1",
+	}, {
+		what:     "whole name stored as plainName by another client",
+		stored:   []storedFile{{"uuid-2", "foo.tar.gz", ""}},
+		leaf:     "foo.tar.gz",
+		wantUUID: "uuid-2",
+	}, {
+		what:     "extensionless name is not confused with an extended one",
+		stored:   []storedFile{{"uuid-3", "README", "md"}, {"uuid-4", "README", ""}},
+		leaf:     "README",
+		wantUUID: "uuid-4",
+	}, {
+		what:     "dotfile stored the way the web client splits it",
+		stored:   []storedFile{{"uuid-6", ".bashrc", ""}},
+		leaf:     ".bashrc",
+		wantUUID: "uuid-6",
+	}, {
+		what:     "dotfile stored the way older rclone versions split it",
+		stored:   []storedFile{{"uuid-7", "", "bashrc"}},
+		leaf:     ".bashrc",
+		wantUUID: "uuid-7",
+	}, {
+		what:     "trailing dot kept in plainName by a rename",
+		stored:   []storedFile{{"uuid-8", "trailing.", ""}},
+		leaf:     "trailing.",
+		wantUUID: "uuid-8",
+	}, {
+		what:   "missing file",
+		stored: []storedFile{{"uuid-5", "other", "txt"}},
+		leaf:   "foo.txt",
+	}} {
+		t.Run(test.what, func(t *testing.T) {
+			server := existenceServer(t, test.stored)
+			defer server.Close()
+			file, err := testFs(t, server).findFile(context.Background(), test.leaf, "dir-uuid")
+			require.NoError(t, err)
+			if test.wantUUID == "" {
+				assert.Nil(t, file)
+				return
+			}
+			require.NotNil(t, file)
+			assert.Equal(t, test.wantUUID, file.UUID)
+		})
+	}
+}
+
+func TestFindFileIgnoresStale(t *testing.T) {
+	ctx := context.Background()
+	server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})
+	defer server.Close()
+	f := testFs(t, server)
+
+	recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")
+	recent.deleted("uuid-deleted")
+	recent.moved("uuid-renamed", "dir-uuid", "baz.txt")
+
+	file, err := f.findFile(ctx, "foo.txt", "dir-uuid")
+	require.NoError(t, err)
+	assert.Nil(t, file, "moved away")
+
+	file, err = f.findFile(ctx, "bar.txt", "dir-uuid")
+	require.NoError(t, err)
+	assert.Nil(t, file, "deleted")
+
+	file, err = f.findFile(ctx, "baz.txt", "dir-uuid")
+	require.NoError(t, err)
+	require.NotNil(t, file, "moved here")
+	assert.Equal(t, "uuid-renamed", file.UUID)
+}

backend/internxt/upload.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"path"
 	"sort"
-	"strings"
 	"sync"
 
 	"github.com/internxt/rclone-adapter/buckets"
@@ -263,9 +262,7 @@ func (w *internxtChunkWriter) Close(ctx context.Context) error {
 	}
 
 	// Create file metadata in Internxt Drive
-	baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))
-	name := strings.TrimSuffix(baseName, path.Ext(baseName))
-	ext := strings.TrimPrefix(path.Ext(baseName), ".")
+	name, ext := splitNameExt(w.f.opt.Encoding.FromStandardName(path.Base(w.remote)))
 
 	var meta *buckets.CreateMetaResponse
 	err = w.f.pacer.Call(func() (bool, error) {

backend/local/local.go

@@ -438,6 +438,7 @@ var (
 	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
 	errLinksNeedsSuffix  = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
 	errPathEscapes       = errors.New("file name is not a path within the local root - check the encoding")
+	errLinkNameInvalid   = errors.New("translated link needs a file name before the \"" + fs.LinkSuffix + "\" suffix")
 )
 
 // NewFs constructs an Fs from the path
@@ -565,30 +566,26 @@ func (f *Fs) caseInsensitive() bool {
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
-// translateLink checks whether the remote is a translated link
-// and returns a new path, removing the suffix as needed,
-// It also returns whether this is a translated link at all
-//
-// for regular files, localPath is returned unchanged
-func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
-	newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
-	return newLocalPath, isTranslatedLink
-}
-
 // newObject makes a half completed Object
 func (f *Fs) newObject(remote string) (*Object, error) {
-	translatedLink := false
-	localPath, err := f.localPath(remote)
+	name, translatedLink := remote, false
+	if f.opt.TranslateSymlinks {
+		name, translatedLink = strings.CutSuffix(remote, fs.LinkSuffix)
+	}
+	if translatedLink {
+		// Check this makes a valid native file name
+		native := f.nativePath(name)
+		switch native[strings.LastIndex(native, string(filepath.Separator))+1:] {
+		case "", ".", "..":
+			return nil, fserrors.NoRetryError(fmt.Errorf("%q: %w", remote, errLinkNameInvalid))
+		}
+	}
+	// Check the name after any suffix removal
+	localPath, err := f.localPath(name)
 	if err != nil {
 		return nil, err
 	}
 
-	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for localPath
-		localPath, translatedLink = translateLink(remote, localPath)
-	}
-
 	return &Object{
 		fs:             f,
 		remote:         remote,
@@ -637,15 +634,23 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 }
 
 // Create new directory object from the info passed in
+//
+// Directories are never translated links, so the name is used as is
+// whatever suffix it has.
 func (f *Fs) newDirectory(dir string, fi os.FileInfo) (*Directory, error) {
-	o, err := f.newObject(dir)
+	localPath, err := f.localPath(dir)
 	if err != nil {
 		return nil, err
 	}
-	o.setMetadata(fi)
-	return &Directory{
-		Object: *o,
-	}, nil
+	d := &Directory{
+		Object: Object{
+			fs:     f,
+			remote: dir,
+			path:   localPath,
+		},
+	}
+	d.setMetadata(fi)
+	return d, nil
 }
 
 // List the objects and directories in dir into entries.  The
@@ -688,6 +693,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}()
 
 	for {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return nil, ctxErr
+		}
 		var fis []os.FileInfo
 		if useReadDir {
 			// Windows and Plan9 read the directory entries with the stat information in which
@@ -782,6 +790,12 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					continue
 				}
 				fso, err := f.newObjectWithInfo(newRemote, fi)
+				if errors.Is(err, errLinkNameInvalid) {
+					// Skip a badly named ".rclonelink" file
+					fs.Errorf(newRemote, "Listing error: %v", err)
+					_ = accounting.Stats(ctx).Error(err)
+					continue
+				}
 				if err != nil {
 					return nil, err
 				}
@@ -811,14 +825,19 @@ func (f *Fs) cleanRemote(dir, filename string) (remote string) {
 	return
 }
 
+// nativePath returns name in the OS encoding with OS path separators,
+// relative to f.root, without cleaning it.
+func (f *Fs) nativePath(name string) string {
+	return filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
+}
+
 // localPath returns the OS path for the object called name, which is
 // always underneath f.root.
 //
 // It returns errPathEscapes if name resolves outside f.root which can
 // happen depending on the encoding.
 func (f *Fs) localPath(name string) (string, error) {
-	native := filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
-	localPath := filepath.Join(f.root, native)
+	localPath := filepath.Join(f.root, f.nativePath(name))
 	rel, err := filepath.Rel(f.root, localPath)
 	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
 		return "", fserrors.NoRetryError(fmt.Errorf("%q: %w", name, errPathEscapes))
@@ -1315,7 +1334,7 @@ func (o *Object) setTimes(atime, mtime time.Time) (err error) {
 	if o.translatedLink {
 		err = lChtimes(o.path, atime, mtime)
 	} else {
-		err = os.Chtimes(o.path, atime, mtime)
+		err = o.fs.chtimes(o.path, atime, mtime)
 	}
 	return err
 }
@@ -1420,6 +1439,13 @@ func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err
 	if err != nil {
 		return nil, err
 	}
+	// Clamp offset into range to avoid panic
+	if offset < 0 {
+		offset = 0
+	}
+	if offset > int64(len(linkdst)) {
+		offset = int64(len(linkdst))
+	}
 	return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
 }
 
@@ -1565,6 +1591,45 @@ func (f *Fs) symlink(target, localPath string) (err error) {
 	return root.Symlink(target, rel)
 }
 
+// chmod changes the mode of localPath.
+func (f *Fs) chmod(localPath string, mode os.FileMode) (err error) {
+	if !f.opt.TranslateSymlinks {
+		return os.Chmod(localPath, mode)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(root, &err)
+	return root.Chmod(rel, mode)
+}
+
+// chown changes the ownership of localPath.
+func (f *Fs) chown(localPath string, uid, gid int) (err error) {
+	if !f.opt.TranslateSymlinks {
+		return os.Chown(localPath, uid, gid)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(root, &err)
+	return root.Chown(rel, uid, gid)
+}
+
+// chtimes changes the atime and mtime of localPath.
+func (f *Fs) chtimes(localPath string, atime, mtime time.Time) (err error) {
+	if !f.opt.TranslateSymlinks {
+		return os.Chtimes(localPath, atime, mtime)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(root, &err)
+	return root.Chtimes(rel, atime, mtime)
+}
+
 // mkdirAll makes all the directories needed to store the object
 func (o *Object) mkdirAll() error {
 	return o.fs.mkdirAll(filepath.Dir(o.path))

backend/local/local_internal_diskfull_windows_test.go

@@ -0,0 +1,37 @@
+//go:build windows
+
+package local
+
+import (
+	"os"
+	"testing"
+
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/sys/windows"
+)
+
+func TestUpdateFatalIfNoSpaceWindows(t *testing.T) {
+	tests := []struct {
+		name string
+		err  error
+	}{
+		{"ERROR_DISK_FULL", windows.ERROR_DISK_FULL},
+		{"openat PathError", &os.PathError{Op: "openat", Path: "test.txt", Err: windows.ERROR_DISK_FULL}},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Run("off", func(t *testing.T) {
+				err := updateWithReader(t, false, test.err)
+				require.Error(t, err)
+				assert.False(t, fserrors.IsFatalError(err))
+			})
+			t.Run("on", func(t *testing.T) {
+				err := updateWithReader(t, true, test.err)
+				require.Error(t, err)
+				assert.True(t, fserrors.IsFatalError(err))
+			})
+		})
+	}
+}

backend/local/local_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"os"
 	"path"
 	"path/filepath"
@@ -96,7 +97,19 @@ func TestVerifyCopy(t *testing.T) {
 	assert.Error(t, err)
 }
 
+// skipIfNoSymlinks skips the test if this process can't create symlinks.
+//
+// Windows grants the privilege only to an elevated process or one running with
+// Developer Mode enabled, so an ordinary user gets ERROR_PRIVILEGE_NOT_HELD.
+func skipIfNoSymlinks(t *testing.T) {
+	t.Helper()
+	if err := os.Symlink("target", filepath.Join(t.TempDir(), "link")); err != nil {
+		t.Skipf("Skipping as symlinks are unavailable: %v", err)
+	}
+}
+
 func TestSymlink(t *testing.T) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 	r := fstest.NewRun(t)
 	f := r.Flocal.(*Fs)
@@ -198,6 +211,32 @@ func TestSymlink(t *testing.T) {
 	require.NoError(t, in.Close())
 }
 
+// TestSymlinkRangeBeyondEnd checks range requests on a translated
+// symlink's target string don't panic.
+func TestSymlinkRangeBeyondEnd(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	const target = "file.txt"
+	require.NoError(t, putLink(ctx, f, "symlink.txt", target))
+
+	o, err := f.NewObject(ctx, "symlink.txt"+fs.LinkSuffix)
+	require.NoError(t, err)
+
+	// An offset just past the end and a wildly large offset must both read
+	// empty rather than panicking.
+	for _, start := range []int64{int64(len(target)), int64(len(target)) + 1, math.MaxInt64} {
+		in, err := o.Open(ctx, &fs.RangeOption{Start: start, End: -1})
+		require.NoError(t, err)
+		contents, err := io.ReadAll(in)
+		require.NoError(t, err)
+		require.Empty(t, string(contents))
+		require.NoError(t, in.Close())
+	}
+}
+
 func TestSymlinkError(t *testing.T) {
 	m := configmap.Simple{
 		"links":      "true",
@@ -238,6 +277,7 @@ func linksMode(f *Fs) {
 // faithful backup of the source) but must refuse to write through it, so
 // nothing lands outside the destination (CWE-59).
 func TestSymlinkEscapeWriteThroughBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 
 	// A directory outside the destination the attacker wants to write into
@@ -273,6 +313,7 @@ func TestSymlinkEscapeWriteThroughBlocked(t *testing.T) {
 // re-validated against the root, so the write-through is refused and nothing
 // escapes.
 func TestSymlinkEscapeNestedBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 
 	evil := t.TempDir()
@@ -324,6 +365,7 @@ func TestSymlinkEscapeConcurrent(t *testing.T) {
 // use: an in-tree symlink to a sibling directory can still be created and
 // written through, since that write stays inside the destination.
 func TestSymlinkInTreeWriteThroughWorks(t *testing.T) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 
 	r := fstest.NewRun(t)
@@ -341,6 +383,133 @@ func TestSymlinkInTreeWriteThroughWorks(t *testing.T) {
 	require.Equal(t, "world", string(got))
 }
 
+// TestDirMetadataThroughPlantedSymlinkBlocked checks metadata +
+// symlinks can't write outside the root.
+func TestDirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("symlinks and unix modes not applicable on Windows")
+	}
+	ctx := context.Background()
+
+	// A directory outside the destination whose metadata the attacker targets.
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	evilMtime := fstest.Time("2016-06-07T08:09:10Z")
+	require.NoError(t, os.Chtimes(evilDir, evilMtime, evilMtime))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// The source now presents "pwn" as a directory with attacker-chosen mode
+	// and mtime. Applying it must not reach through the planted symlink.
+	metadata := fs.Metadata{
+		"mode":  "0777",
+		"mtime": "2001-02-03T04:05:06Z",
+	}
+	_, err := f.MkdirMetadata(ctx, "pwn", metadata)
+	require.Error(t, err, "applying metadata through a planted symlink should be refused")
+
+	// The outside directory's mode and mtime must be unchanged.
+	fi, err := os.Stat(evilDir)
+	require.NoError(t, err)
+	require.Equal(t, os.FileMode(0700), fi.Mode().Perm(), "chmod escaped through planted symlink to %q", evilDir)
+	require.True(t, fi.ModTime().Equal(evilMtime), "chtimes escaped through planted symlink to %q", evilDir)
+}
+
+// TestDirSetModTimeThroughPlantedSymlinkBlocked checks we can't
+// chtimes outside the root with --links
+func TestDirSetModTimeThroughPlantedSymlinkBlocked(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("symlinks not applicable on Windows")
+	}
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	evilMtime := fstest.Time("2016-06-07T08:09:10Z")
+	require.NoError(t, os.Chtimes(evilDir, evilMtime, evilMtime))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	err := f.DirSetModTime(ctx, "pwn", fstest.Time("2001-02-03T04:05:06Z"))
+	require.Error(t, err, "setting dir modtime through a planted symlink should be refused")
+
+	fi, err := os.Stat(evilDir)
+	require.NoError(t, err)
+	require.True(t, fi.ModTime().Equal(evilMtime), "chtimes escaped through planted symlink to %q", evilDir)
+}
+
+// TestDirMetadataInTreeWorks checks the root confinement doesn't
+// break legitimate dir metadata.
+func TestDirMetadataInTreeWorks(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("unix modes not applicable on Windows")
+	}
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	metadata := fs.Metadata{
+		"mode":  "0705",
+		"mtime": "2001-02-03T04:05:06Z",
+	}
+	_, err := f.MkdirMetadata(ctx, "sub", metadata)
+	require.NoError(t, err)
+
+	fi, err := os.Stat(filepath.Join(f.root, "sub"))
+	require.NoError(t, err)
+	require.Equal(t, os.FileMode(0705), fi.Mode().Perm())
+	require.True(t, fi.ModTime().Equal(fstest.Time("2001-02-03T04:05:06Z")))
+}
+
+// TestDirBTimeThroughPlantedSymlinkBlocked checks a btime write
+// through a symlink can't escape the root.
+func TestDirBTimeThroughPlantedSymlinkBlocked(t *testing.T) {
+	if !haveSetBTime {
+		t.Skip("birth time is not settable on this OS")
+	}
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+
+	// Read the outside dir's btime through a local Fs rooted at evil.
+	evilFsRaw, err := NewFs(ctx, "local", evil, configmap.Simple{})
+	require.NoError(t, err)
+	evilFs := evilFsRaw.(*Fs)
+	readBTime := func() string {
+		o, err := evilFs.newObject("secret.d")
+		require.NoError(t, err)
+		require.NoError(t, o.lstat())
+		m, err := o.Metadata(ctx)
+		require.NoError(t, err)
+		return m["btime"]
+	}
+	before := readBTime()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	_, _ = f.MkdirMetadata(ctx, "pwn", fs.Metadata{"btime": "2001-02-03T04:05:06Z"})
+
+	require.Equal(t, before, readBTime(), "btime escaped through planted symlink to %q", evilDir)
+}
+
 // TestEncodingEscapeBlocked checks that a name from a malicious source can't
 // be decoded into path syntax which writes outside the destination.
 func TestEncodingEscapeBlocked(t *testing.T) {
@@ -452,6 +621,154 @@ func TestLocalPath(t *testing.T) {
 	}
 }
 
+// TestLinkSuffixEscapeBlocked checks that in --links mode a name which
+// passes the root check with its ".rclonelink" suffix can't escape the
+// destination once the suffix is removed: "...rclonelink" must not
+// become "..", the parent of the root.
+func TestLinkSuffixEscapeBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+	outer := t.TempDir()
+	newLinksFs := func(dir, encoding string) *Fs {
+		fRaw, err := NewFs(ctx, "local", filepath.Join(outer, dir), configmap.Simple{"encoding": encoding})
+		require.NoError(t, err)
+		f := fRaw.(*Fs)
+		linksMode(f)
+		require.NoError(t, f.Mkdir(ctx, ""))
+		return f
+	}
+	f := newLinksFs("dst", encoder.OS.String())
+	s := newLinksFs("slash", "Slash")
+
+	// Stamp the parent of the destinations so a write to it shows up.
+	before := fstest.Time("2011-12-25T12:59:59Z")
+	require.NoError(t, os.Chtimes(outer, before, before))
+	parentUnchanged := func() {
+		t.Helper()
+		fi, err := os.Lstat(outer)
+		require.NoError(t, err)
+		assert.True(t, fi.ModTime().Equal(before), "the parent of the destination was modified")
+	}
+
+	// A directory called "...rclonelink" is created under its own name
+	// and its metadata goes there, not to the parent of the root which
+	// trimming the suffix off its path would address.
+	remote := ".." + fs.LinkSuffix
+	metadata := fs.Metadata{"mtime": "2001-02-03T04:05:06Z"}
+	_, err := f.MkdirMetadata(ctx, remote, metadata)
+	require.NoError(t, err)
+	parentUnchanged()
+
+	// A file called "...rclonelink" is a link called "..", which the
+	// default encoding stores as fullwidth dots like any other name.
+	require.NoError(t, putFile(ctx, f, remote, "target"))
+	fi, err := os.Lstat(filepath.Join(f.root, "．．"))
+	require.NoError(t, err)
+	assert.NotZero(t, fi.Mode()&os.ModeSymlink)
+	parentUnchanged()
+
+	// Ordinary translated links still resolve to the file without the suffix.
+	o, err := f.newObject("sub/link" + fs.LinkSuffix)
+	require.NoError(t, err)
+	assert.True(t, o.translatedLink)
+	assert.Equal(t, filepath.Join(f.root, "sub", "link"), o.path)
+
+	// A name which merely contains the suffix is a regular file.
+	o, err = f.newObject("sub/link" + fs.LinkSuffix + ".txt")
+	require.NoError(t, err)
+	assert.False(t, o.translatedLink)
+	assert.Equal(t, filepath.Join(f.root, "sub", "link"+fs.LinkSuffix+".txt"), o.path)
+
+	// An encoding without Dot leaves ".." as a real ".." once the suffix
+	// is gone, so the name is refused instead.
+	require.ErrorIs(t, putFile(ctx, s, remote, "PWNED"), errLinkNameInvalid)
+	_, err = s.MkdirMetadata(ctx, remote, metadata)
+	require.NoError(t, err)
+	parentUnchanged()
+
+	// As is any other name which doesn't leave a file name once the
+	// suffix is removed and the encoding applied.
+	for _, name := range []string{
+		remote,
+		"sub/.." + fs.LinkSuffix,
+		"sub/." + fs.LinkSuffix,
+		"sub/．．" + fs.LinkSuffix,
+		"sub/sub2/．．" + fs.LinkSuffix,
+		"sub/．" + fs.LinkSuffix,
+		"sub/" + fs.LinkSuffix,
+		fs.LinkSuffix,
+	} {
+		_, err = s.newObject(name)
+		assert.ErrorIs(t, err, errLinkNameInvalid, name)
+		assert.True(t, fserrors.IsNoRetryError(err), name)
+	}
+
+	// The root check still applies to the name of the link.
+	_, err = s.newObject("../link" + fs.LinkSuffix)
+	require.ErrorIs(t, err, errPathEscapes)
+
+	if runtime.GOOS == "windows" {
+		// A backslash is a path separator on Windows, so an encoding
+		// without BackSlash can hide ".." behind one.
+		w := newLinksFs("win", "Slash,Dot")
+		_, err = w.newObject(`sub\..` + fs.LinkSuffix)
+		assert.ErrorIs(t, err, errLinkNameInvalid)
+	}
+}
+
+// TestLinkSuffixListSkipsInvalidName checks that with --links a regular
+// file whose name is just the suffix is reported as an error and skipped
+// rather than failing the listing of its whole directory.
+func TestLinkSuffixListSkipsInvalidName(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+	when := fstest.Time("2001-02-03T04:05:06Z")
+	r.WriteFile("sub/"+fs.LinkSuffix, "not a link", when)
+	r.WriteFile("sub/file.txt", "hello", when)
+
+	accounting.Stats(ctx).ResetErrors()
+	entries, err := f.List(ctx, "sub")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	assert.Equal(t, "sub/file.txt", entries[0].Remote())
+	assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors())
+	accounting.Stats(ctx).ResetErrors()
+}
+
+// TestLinkSuffixDirectory checks that in --links mode a directory whose
+// name ends in ".rclonelink" is an ordinary directory. Only files are
+// translated into links, so the suffix must stay in its path rather
+// than making it address a sibling.
+func TestLinkSuffixDirectory(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	dir := "sub" + fs.LinkSuffix
+	want := filepath.Join(f.root, dir)
+	mtime := fstest.Time("2001-02-03T04:05:06Z")
+
+	d, err := f.MkdirMetadata(ctx, dir, fs.Metadata{"mtime": mtime.Format(time.RFC3339Nano)})
+	require.NoError(t, err)
+	assert.Equal(t, want, d.(*Directory).path)
+	fi, err := os.Lstat(want)
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+	assert.True(t, fi.ModTime().Equal(mtime), "metadata was not written to the directory")
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	got, ok := entries[0].(*Directory)
+	require.True(t, ok)
+	assert.Equal(t, dir, got.Remote())
+	assert.Equal(t, want, got.path)
+	assert.False(t, got.translatedLink)
+}
+
 func TestHashWithTypeNone(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
@@ -529,6 +846,7 @@ func TestHashOnDelete(t *testing.T) {
 }
 
 func TestMetadata(t *testing.T) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 	r := fstest.NewRun(t)
 	const filePath = "metafile.txt"
@@ -805,6 +1123,7 @@ func TestFilter(t *testing.T) {
 }
 
 func testFilterSymlink(t *testing.T, copyLinks bool) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 	r := fstest.NewRun(t)
 	defer r.Finalise()
@@ -909,6 +1228,7 @@ func TestFilterSymlinkLinks(t *testing.T) {
 }
 
 func TestCopySymlink(t *testing.T) {
+	skipIfNoSymlinks(t)
 	ctx := context.Background()
 	r := fstest.NewRun(t)
 	defer r.Finalise()

backend/local/metadata.go

@@ -105,7 +105,10 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	}
 	if haveSetBTime {
 		if btimeOK {
-			if o.translatedLink {
+			// When translating symlinks, never follow the path. A planted symlink must
+			// not redirect the birth-time write out of the root. The NOFOLLOW open is a
+			// no-op on a real file or directory
+			if o.translatedLink || o.fs.opt.TranslateSymlinks {
 				err = lsetBTime(o.path, btime)
 			} else {
 				err = setBTime(o.path, btime)
@@ -128,7 +131,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 			if o.translatedLink {
 				err = os.Lchown(o.path, uid, gid)
 			} else {
-				err = os.Chown(o.path, uid, gid)
+				err = o.fs.chown(o.path, uid, gid)
 			}
 			if err != nil {
 				outErr = fmt.Errorf("failed to change ownership: %w", err)
@@ -155,7 +158,7 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 						err = nil
 					}
 				} else {
-					err = os.Chmod(o.path, fileMode)
+					err = o.fs.chmod(o.path, fileMode)
 				}
 				if err != nil {
 					outErr = fmt.Errorf("failed to change permissions: %w", err)

backend/netstorage/netstorage.go

@@ -236,9 +236,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("couldn't parse URL %q: %w", opt.Endpoint, err)
 	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(root))
+	u, err := rest.URLJoinRoot(base, root)
 	if err != nil {
-		return nil, fmt.Errorf("couldn't join URL %q and %q: %w", base.String(), root, err)
+		return nil, err
 	}
 	client := fshttp.NewClient(ctx)
 
@@ -573,10 +573,12 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 				fs.Errorf(nil, "Unable to parse URL %q: %v", f.endpointURL, err)
 				return fs.ErrorDirNotFound
 			}
-			resumeURL, err := rest.URLJoin(u, rest.URLPathEscape(resumeStart))
+			// A bad resume start must fail the listing rather than
+			// return fs.ErrorDirNotFound which sync treats as an empty
+			// directory
+			resumeURL, err := rest.URLJoinRoot(u, resumeStart)
 			if err != nil {
-				fs.Errorf(nil, "Unable to join URL %q for resumeStart %s: %v", f.endpointURL, resumeStart, err)
-				return fs.ErrorDirNotFound
+				return fmt.Errorf("unable to join URL %q for resumeStart %q: %w", f.endpointURL, resumeStart, err)
 			}
 			URL = resumeURL.String()
 		}

backend/netstorage/netstorage_internal_test.go

@@ -0,0 +1,61 @@
+package netstorage
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListResumeCannotChangeHost(t *testing.T) {
+	// The resume start returned by a list request comes from the
+	// server. It must not be able to move the signed continuation
+	// request to another host.
+	var requests int
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		http.Error(w, "not found", http.StatusNotFound)
+	}))
+	defer other.Close()
+	otherHost := strings.TrimPrefix(other.URL, "http://")
+
+	var listRequests int
+	configured := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		action := r.Header.Get("X-Akamai-ACS-Action")
+		switch {
+		case strings.Contains(action, "action=stat"):
+			_, _ = w.Write([]byte(`<stat directory="/123456/dir"><file type="dir" name="dir" mtime="0"/></stat>`))
+		case strings.Contains(action, "action=list"):
+			listRequests++
+			_, _ = w.Write([]byte(`<list><file type="file" name="123456/dir/a.txt" size="1" mtime="0"/><resume start="//` + otherHost + `/123456/dir/b.txt"/></list>`))
+		default:
+			http.Error(w, "unexpected action "+action, http.StatusBadRequest)
+		}
+	}))
+	defer configured.Close()
+
+	m := configmap.Simple{
+		"type":     "netstorage",
+		"protocol": "http",
+		"host":     strings.TrimPrefix(configured.URL, "http://") + "/123456/",
+		"account":  "account",
+		"secret":   obscure.MustObscure("secret"),
+	}
+	f, err := NewFs(context.Background(), "TestNetStorage", "dir", m)
+	require.NoError(t, err)
+
+	// The listing must fail loudly rather than with
+	// fs.ErrorDirNotFound which sync treats as an empty directory
+	err = f.(*Fs).ListR(context.Background(), "", func(entries fs.DirEntries) error { return nil })
+	assert.ErrorContains(t, err, "must not change the scheme, host or user")
+	assert.NotErrorIs(t, err, fs.ErrorDirNotFound)
+	assert.Equal(t, 1, listRequests, "list request sent to the configured host")
+	assert.Equal(t, 0, requests, "request sent to the other host")
+}

backend/onedrive/metadata.go

@@ -20,7 +20,7 @@ import (
 const (
 	dirMimeType   = "inode/directory"
 	timeFormatIn  = time.RFC3339
-	timeFormatOut = "2006-01-02T15:04:05.999Z" // mS for OneDrive Personal, otherwise only S
+	timeFormatOut = "2006-01-02T15:04:05.999Z" // keeps mS if present, though OneDrive only stores S
 )
 
 // system metadata keys which this backend owns
@@ -32,17 +32,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 		ReadOnly: true,
 	},
 	"mtime": {
-		Help:    "Time of last modification with S accuracy (mS for OneDrive Personal).",
+		Help:    "Time of last modification with S accuracy.",
 		Type:    "RFC 3339",
 		Example: "2006-01-02T15:04:05Z",
 	},
 	"btime": {
-		Help:    "Time of file birth (creation) with S accuracy (mS for OneDrive Personal).",
+		Help:    "Time of file birth (creation) with S accuracy.",
 		Type:    "RFC 3339",
 		Example: "2006-01-02T15:04:05Z",
 	},
 	"utime": {
-		Help:     "Time of upload with S accuracy (mS for OneDrive Personal).",
+		Help:     "Time of upload with S accuracy.",
 		Type:     "RFC 3339",
 		Example:  "2006-01-02T15:04:05Z",
 		ReadOnly: true,
@@ -113,7 +113,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 		ReadOnly: true,
 	},
 	"shared-time": {
-		Help:     "Time when the item was shared, with S accuracy (mS for OneDrive Personal).",
+		Help:     "Time when the item was shared, with S accuracy.",
 		Type:     "RFC 3339",
 		Example:  "2006-01-02T15:04:05Z",
 		ReadOnly: true,

backend/onedrive/onedrive.go

@@ -275,16 +275,13 @@ cases, rclone will fall back to normal copy (which will be slightly slower).`,
 			Default: false,
 			Help: `Remove all versions on modifying operations.
 
-Onedrive for business creates versions when rclone uploads new files
+Onedrive creates versions when rclone uploads new files
 overwriting an existing one and when it sets the modification time.
 
 These versions take up space out of the quota.
 
 This flag checks for versions after file upload and setting
 modification time and removes all but the last version.
-
-**NB** Onedrive personal can't currently delete versions so don't use
-this flag there.
 `,
 			Advanced: true,
 		}, {
@@ -294,8 +291,7 @@ this flag there.
 Normally files will get sent to the recycle bin on deletion. Setting
 this flag causes them to be permanently deleted. Use with care.
 
-OneDrive personal accounts do not support the permanentDelete API,
-it only applies to OneDrive for Business and SharePoint document libraries.
+This works with OneDrive for Business, SharePoint document libraries, and OneDrive personal accounts, including free accounts.
 `,
 			Advanced: true,
 			Default:  false,
@@ -324,14 +320,18 @@ it only applies to OneDrive for Business and SharePoint document libraries.
 				Help:  "Creates a read-write link to the item.",
 			}, {
 				Value: "embed",
-				Help:  "Creates an embeddable link to the item.",
+				Help:  "Creates an embeddable link to the item.\nOnly available in OneDrive personal.",
 			}},
 		}, {
 			Name:    "link_password",
 			Default: "",
 			Help: `Set the password for links created by the link command.
 
-At the time of writing this only works with OneDrive personal paid accounts.
+At the time of writing this works with OneDrive for Business and
+OneDrive personal paid accounts.
+
+OneDrive personal free accounts can't set a password or an expiry time
+(with --expire) on links.
 `,
 			Advanced:  true,
 			Sensitive: true,
@@ -549,31 +549,34 @@ func chooseDrive(ctx context.Context, name string, m configmap.Mapper, srv *rest
 	// We don't have the final ID yet?
 	// query Microsoft Graph
 	if opt.finalDriveID == "" {
-		_, err := srv.CallJSON(ctx, &opt.opts, nil, &drives)
-		if err != nil {
-			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
+		_, drivesErr := srv.CallJSON(ctx, &opt.opts, nil, &drives)
+		if drivesErr != nil {
+			fs.Debugf(nil, "Failed to query /me/drives: %v - trying /me/drive", drivesErr)
 		}
-
 		// Also call /me/drive as sometimes /me/drives doesn't return it #4068
 		if opt.opts.Path == "/me/drives" {
-			opt.opts.Path = "/me/drive"
+			meDriveOpts := opt.opts
+			meDriveOpts.Path = "/me/drive"
 			meDrive := api.DriveResource{}
-			_, err := srv.CallJSON(ctx, &opt.opts, nil, &meDrive)
-			if err != nil {
-				return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", err))
-			}
-			found := false
-			for _, drive := range drives.Drives {
-				if drive.DriveID == meDrive.DriveID {
-					found = true
-					break
+			_, meDriveErr := srv.CallJSON(ctx, &meDriveOpts, nil, &meDrive)
+			if meDriveErr == nil {
+				found := false
+				for _, drive := range drives.Drives {
+					if drive.DriveID == meDrive.DriveID {
+						found = true
+						break
+					}
 				}
+				// add the me drive if not found already
+				if !found {
+					fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
+					drives.Drives = append(drives.Drives, meDrive)
+				}
+			} else if drivesErr != nil {
+				return fs.ConfigError("driveid", fmt.Sprintf("Failed to query available drives: /me/drives: %v; /me/drive: %v\nEnter the drive ID manually instead", drivesErr, meDriveErr))
 			}
-			// add the me drive if not found already
-			if !found {
-				fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
-				drives.Drives = append(drives.Drives, meDrive)
-			}
+		} else if drivesErr != nil {
+			return fs.ConfigError("choose_type", fmt.Sprintf("Failed to query available drives: %v", drivesErr))
 		}
 	} else {
 		drives.Drives = append(drives.Drives, api.DriveResource{
@@ -2678,9 +2681,8 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			}
 			return true, fmt.Errorf("retry this chunk skipping %d bytes: %w", skip, err)
 		} else if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound {
-			fs.Debugf(o, "Received 404 error: assuming eventual consistency problem with session - retrying chunk: %v", err)
-			time.Sleep(5 * time.Second) // a little delay to help things along
-			return true, err
+			fs.Debugf(o, "Received 404 error: upload session not found - not retrying: %v", err)
+			return false, fserrors.NoLowLevelRetryError(err)
 		}
 		if err != nil {
 			return shouldRetry(ctx, resp, err)

backend/oracleobjectstorage/byok.go

@@ -116,11 +116,14 @@ func useBYOKCopyObject(fs *Fs, request *objectstorage.CopyObjectRequest) {
 	}
 	if fs.opt.SSECustomerAlgorithm != "" {
 		request.OpcSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
+		request.OpcSourceSseCustomerAlgorithm = common.String(fs.opt.SSECustomerAlgorithm)
 	}
 	if fs.opt.SSECustomerKey != "" {
 		request.OpcSseCustomerKey = common.String(fs.opt.SSECustomerKey)
+		request.OpcSourceSseCustomerKey = common.String(fs.opt.SSECustomerKey)
 	}
 	if fs.opt.SSECustomerKeySha256 != "" {
 		request.OpcSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
+		request.OpcSourceSseCustomerKeySha256 = common.String(fs.opt.SSECustomerKeySha256)
 	}
 }

backend/oracleobjectstorage/byok_test.go

@@ -0,0 +1,25 @@
+//go:build !plan9 && !solaris && !js
+
+package oracleobjectstorage
+
+import (
+	"testing"
+
+	"github.com/oracle/oci-go-sdk/v65/objectstorage"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestUseBYOKCopyObject(t *testing.T) {
+	f := &Fs{opt: Options{
+		SSECustomerAlgorithm: "AES256",
+		SSECustomerKey:       "customer-key",
+		SSECustomerKeySha256: "customer-key-sha256",
+	}}
+	req := &objectstorage.CopyObjectRequest{}
+
+	useBYOKCopyObject(f, req)
+
+	assert.Equal(t, &f.opt.SSECustomerAlgorithm, req.OpcSourceSseCustomerAlgorithm)
+	assert.Equal(t, &f.opt.SSECustomerKey, req.OpcSourceSseCustomerKey)
+	assert.Equal(t, &f.opt.SSECustomerKeySha256, req.OpcSourceSseCustomerKeySha256)
+}

backend/oracleobjectstorage/object.go

@@ -3,6 +3,7 @@
 package oracleobjectstorage
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/base64"
@@ -396,6 +397,9 @@ func isZeroLength(streamReader io.Reader) bool {
 		return v.Len() == 0
 	case *strings.Reader:
 		return v.Len() == 0
+	case *bufio.Reader:
+		_, err := v.Peek(1)
+		return err == io.EOF
 	case *os.File:
 		fi, err := v.Stat()
 		if err != nil {

backend/oracleobjectstorage/oracleobjectstorage.go

@@ -4,6 +4,7 @@
 package oracleobjectstorage
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"io"
@@ -553,7 +554,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // PutStream uploads to the remote path with the modTime given of indeterminate size
 func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return f.Put(ctx, in, src, options...)
+	return f.Put(ctx, bufio.NewReader(in), src, options...)
 }
 
 // Mkdir creates the bucket if it doesn't exist

backend/oracleobjectstorage/oracleobjectstorage_test.go

@@ -3,11 +3,16 @@
 package oracleobjectstorage
 
 import (
+	"bufio"
 	"bytes"
 	"compress/gzip"
 	"context"
 	"crypto/md5"
+	"errors"
 	"fmt"
+	"io"
+	"os"
+	"strings"
 	"testing"
 
 	"github.com/rclone/rclone/fs"
@@ -15,6 +20,7 @@ import (
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -31,6 +37,51 @@ func TestIntegration(t *testing.T) {
 	})
 }
 
+func TestIsZeroLength(t *testing.T) {
+	newFile := func(contents string) *os.File {
+		file, err := os.CreateTemp(t.TempDir(), "stream")
+		require.NoError(t, err)
+		_, err = file.WriteString(contents)
+		require.NoError(t, err)
+		_, err = file.Seek(0, io.SeekStart)
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = file.Close() })
+		return file
+	}
+	closedFile := newFile("")
+	require.NoError(t, closedFile.Close())
+
+	for _, test := range []struct {
+		name         string
+		in           io.Reader
+		want         bool
+		wantContents string
+	}{
+		{name: "EmptyBuffer", in: bytes.NewBuffer(nil), want: true},
+		{name: "NonEmptyBuffer", in: bytes.NewBufferString("contents"), want: false},
+		{name: "EmptyBytesReader", in: bytes.NewReader(nil), want: true},
+		{name: "NonEmptyBytesReader", in: bytes.NewReader([]byte("contents")), want: false},
+		{name: "EmptyStringsReader", in: strings.NewReader(""), want: true},
+		{name: "NonEmptyStringsReader", in: strings.NewReader("contents"), want: false},
+		{name: "EmptyBufferedStream", in: bufio.NewReader(io.TeeReader(bytes.NewReader(nil), io.Discard)), want: true},
+		{name: "NonEmptyBufferedStream", in: bufio.NewReader(io.TeeReader(bytes.NewBufferString("contents"), io.Discard)), want: false, wantContents: "contents"},
+		{name: "BufferedStreamError", in: bufio.NewReader(readers.ErrorReader{Err: errors.New("stream error")}), want: false},
+		{name: "EmptyFile", in: newFile(""), want: true},
+		{name: "NonEmptyFile", in: newFile("contents"), want: false},
+		{name: "ClosedFile", in: closedFile, want: false},
+		{name: "OpaqueEmptyStream", in: io.TeeReader(bytes.NewReader(nil), io.Discard), want: false},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.want, isZeroLength(test.in))
+			if test.wantContents != "" {
+				contents, err := io.ReadAll(test.in)
+				require.NoError(t, err)
+				assert.Equal(t, test.wantContents, string(contents))
+			}
+		})
+	}
+}
+
 func gz(t *testing.T, s string) string {
 	var buf bytes.Buffer
 	zw := gzip.NewWriter(&buf)

backend/overview/overview.go

@@ -26,7 +26,7 @@ type BackendConfig struct {
 	Remote           string   `yaml:"remote"`
 	Features         []string `yaml:"features"`
 	Hashes           []string `yaml:"hashes"`
-	Precision        int      `yaml:"precision"`
+	Precision        int64    `yaml:"precision"`
 }
 
 // GetBackendConfig from docs/data/backends

backend/overview/overview_test.go

@@ -1,8 +1,10 @@
 package overview
 
 import (
+	"strings"
 	"testing"
 
+	"github.com/rclone/rclone/docs/data/backends"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -53,3 +55,17 @@ func TestGetBackendConfig(t *testing.T) {
 	assert.Equal(t, expectedMemoryConfig, conf, "parsed memory.yaml should match")
 
 }
+
+// Check every embedded backend YAML file parses on all architectures
+func TestGetBackendConfigAll(t *testing.T) {
+	entries, err := backends.BackendFS.ReadDir(".")
+	require.NoError(t, err)
+	require.NotEmpty(t, entries)
+
+	for _, entry := range entries {
+		name := strings.TrimSuffix(entry.Name(), ".yaml")
+		conf, err := GetBackendConfig(name)
+		assert.NoError(t, err, "failed to load %s", entry.Name())
+		assert.NotNil(t, conf)
+	}
+}

backend/pcloud/pcloud.go

@@ -14,6 +14,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -56,6 +57,10 @@ var (
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
 	}
+
+	// hostnameRe matches a lower case API hostname within pCloud's
+	// domain, eg api.pcloud.com or eapi.pcloud.com
+	hostnameRe = regexp.MustCompile(`^([a-z0-9]([a-z0-9-]*[a-z0-9])?\.)+pcloud\.com$`)
 )
 
 // Update the TokenURL with the actual hostname
@@ -63,6 +68,19 @@ func updateTokenURL(oauthConfig *oauthutil.Config, hostname string) {
 	oauthConfig.TokenURL = "https://" + hostname + "/oauth2_token"
 }
 
+// callbackHostname checks and returns the API hostname pCloud sent back in the
+// OAuth callback, lower cased, or defaultHostname if it sent none.
+func callbackHostname(hostname string) (string, error) {
+	if hostname == "" {
+		return defaultHostname, nil
+	}
+	hostname = strings.ToLower(hostname)
+	if !hostnameRe.MatchString(hostname) {
+		return "", fmt.Errorf("invalid hostname %q in oauth response", hostname)
+	}
+	return hostname, nil
+}
+
 // Register with Fs
 func init() {
 	updateTokenURL(oauthConfig, defaultHostname)
@@ -81,9 +99,9 @@ func init() {
 				if auth == nil || auth.Form == nil {
 					return errors.New("form not found in response")
 				}
-				hostname := auth.Form.Get("hostname")
-				if hostname == "" {
-					hostname = defaultHostname
+				hostname, err := callbackHostname(auth.Form.Get("hostname"))
+				if err != nil {
+					return err
 				}
 				// Save the hostname in the config
 				m.Set("hostname", hostname)
@@ -95,7 +113,6 @@ func init() {
 			return oauthutil.ConfigOut("", &oauthutil.Options{
 				OAuth2Config: oauthConfig,
 				CheckAuth:    checkAuth,
-				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
 			})
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{

backend/pcloud/pcloud_internal_test.go

@@ -0,0 +1,46 @@
+package pcloud
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// TestCallbackHostname checks that only hosts within pCloud's domain
+// are accepted.
+func TestCallbackHostname(t *testing.T) {
+	for _, test := range []struct {
+		in   string
+		want string // "" means an error is expected
+	}{
+		{"", defaultHostname},
+		{"api.pcloud.com", "api.pcloud.com"},
+		{"eapi.pcloud.com", "eapi.pcloud.com"},
+		{"EAPI.pCloud.COM", "eapi.pcloud.com"},
+		{"new-region.api.pcloud.com", "new-region.api.pcloud.com"},
+		{"pcloud.com", ""},
+		{".pcloud.com", ""},
+		{"-.pcloud.com", ""},
+		{"a-.pcloud.com", ""},
+		{"a..pcloud.com", ""},
+		{"api.pcloud.com.", ""},
+		{"api.pclоud.com", ""}, // Cyrillic о
+		{"127.0.0.1:18443", ""},
+		{"evil.example.com", ""},
+		{"api.pcloud.com.evil.example.com", ""},
+		{"api.pcloud.com:443", ""},
+		{"api.pcloud.com/path", ""},
+		{"user@evil.example.com#.pcloud.com", ""},
+		{"evil.example.com?.pcloud.com", ""},
+	} {
+		t.Run("["+test.in+"]", func(t *testing.T) {
+			got, err := callbackHostname(test.in)
+			if test.want == "" {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, test.want, got)
+		})
+	}
+}

backend/pikpak/multipart.go

@@ -128,6 +128,13 @@ func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
 		return err
 	}
 
+	// Check the source supplied the number of bytes it declared before
+	// finalising, otherwise a truncated file would be stored as a good
+	// upload. Returning an error here aborts the upload.
+	if size >= 0 && off != size {
+		return fmt.Errorf("multipart upload: expected %d bytes in input, but got %d: %w", size, off, io.ErrUnexpectedEOF)
+	}
+
 	err = w.Close(ctx)
 	if err != nil {
 		return fmt.Errorf("multipart upload: failed to finalise: %w", err)

backend/pikpak/pikpak.go

@@ -58,6 +58,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/oauth2"
 )
@@ -1447,10 +1448,11 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, s
 	if err != nil {
 		return fmt.Errorf("failed to create upload client: %w", err)
 	}
+	counter := readers.NewCountingReader(in)
 	req := &s3.PutObjectInput{
 		Bucket: &p.Bucket,
 		Key:    &p.Key,
-		Body:   io.NopCloser(in),
+		Body:   io.NopCloser(counter),
 	}
 	// Apply upload options
 	for _, option := range options {
@@ -1478,7 +1480,14 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, s
 		_, err = client.PutObject(ctx, req, s3opts...)
 		return f.shouldRetry(ctx, nil, err)
 	})
-	return
+	if err != nil {
+		return err
+	}
+	// Check for truncation of input
+	if size >= 0 && int64(counter.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+	return nil
 }
 
 func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, gcid string, size int64, options ...fs.OpenOption) (info *api.File, err error) {

backend/pixeldrain/api_client.go

@@ -213,7 +213,7 @@ func (f *Fs) put(
 	// exist yet
 	params.Set("make_parents", "true")
 
-	return node, f.pacer.Call(func() (bool, error) {
+	return node, f.pacer.CallNoRetry(func() (bool, error) {
 		resp, err := f.srv.CallJSON(
 			ctx,
 			&rest.Opts{

backend/premiumizeme/premiumizeme.go

@@ -993,6 +993,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}()
 	}
 
+	// The upload server truncates the file name at the first ";" so
+	// upload under a temporary name and rename it afterwards.
+	uploadLeaf := leaf
+	if strings.Contains(leaf, ";") {
+		uploadLeaf = "rclone-upload-" + random.String(16)
+	}
+
 	opts = rest.Opts{
 		Method:  "POST",
 		RootURL: info.URL,
@@ -1000,8 +1007,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		MultipartParams: url.Values{
 			"token": {info.Token},
 		},
-		MultipartContentName: "file", // ..name of the parameter which is the attached file
-		MultipartFileName:    leaf,   // ..name of the file for the attached file
+		MultipartContentName: "file",     // ..name of the parameter which is the attached file
+		MultipartFileName:    uploadLeaf, // ..name of the file for the attached file
 		ContentLength:        &size,
 	}
 	var result api.Response
@@ -1016,6 +1023,29 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("upload file: %w", err)
 	}
 
+	if uploadLeaf != leaf {
+		var newID string
+		var found bool
+		_, found, err = o.fs.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {
+			if item.Name == uploadLeaf {
+				newID = item.ID
+				return true
+			}
+			return false
+		})
+		if err != nil {
+			return fmt.Errorf("upload find temporary file: %w", err)
+		}
+		if !found {
+			return fmt.Errorf("upload find temporary file %q: %w", uploadLeaf, fs.ErrorObjectNotFound)
+		}
+		err = o.fs.renameLeaf(ctx, true, newID, leaf)
+		if err != nil {
+			_ = o.fs.remove(ctx, newID)
+			return fmt.Errorf("upload rename temporary file: %w", err)
+		}
+	}
+
 	// on successful upload, remove old file if it exists
 	uploaded = true
 	if o.hasMetaData {

backend/protondrive/protondrive.go

@@ -1114,7 +1114,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	modTime := src.ModTime(ctx)
 	var linkID string
 	var fileSystemAttrs *proton.RevisionXAttrCommon
-	if err = o.fs.pacer.Call(func() (bool, error) {
+	if err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		linkID, fileSystemAttrs, err = o.fs.protonDrive.UploadFileByReader(ctx, folderLinkID, leaf, modTime, in, 0)
 		return shouldRetry(ctx, err)
 	}); err != nil {

backend/quatrix/quatrix.go

@@ -1126,12 +1126,14 @@ func (o *Object) dynamicUpload(ctx context.Context, size int64, modTime time.Tim
 
 		_, err := io.CopyN(rw, in, localChunk)
 		if err != nil {
+			_ = rw.Close()
 			return fmt.Errorf("read chunk with offset %d size %d: %w", offset, localChunk, err)
 		}
 
 		start := time.Now()
 
 		err = o.upload(ctx, uploadSession.UploadKey, rw, size, offset, localChunk, options...)
+		_ = rw.Close()
 		if err != nil {
 			return fmt.Errorf("upload chunk with offset %d size %d: %w", offset, localChunk, err)
 		}
@@ -1218,7 +1220,7 @@ func (o *Object) uploadSession(ctx context.Context, parentID, name string) (uplo
 	return o.fs.uploadLink(ctx, parentID, encName)
 }
 
-func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) {
+func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.ReadSeeker, fullSize int64, offset int64, chunkSize int64, options ...fs.OpenOption) (err error) {
 	opts := rest.Opts{
 		Method:        "POST",
 		RootURL:       fmt.Sprintf(uploadURL, o.fs.opt.Host) + uploadKey,
@@ -1231,6 +1233,11 @@ func (o *Object) upload(ctx context.Context, uploadKey string, chunk io.Reader,
 	var fileID string
 
 	err = o.fs.pacer.Call(func() (bool, error) {
+		// Rewind the chunk so a retry re-sends it in full
+		_, err := chunk.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &fileID)
 		return shouldRetry(ctx, resp, err)
 	})

backend/s3/provider/Ceph.yaml

@@ -19,3 +19,7 @@ quirks:
   force_path_style: true
   list_url_encode: false
   use_already_exists: false
+  # Ceph RGW (and reverse proxies such as Cloudflare in front of it) can
+  # mutate Accept-Encoding and break SigV4 when that header is signed.
+  # See: https://github.com/rclone/rclone/issues/8206
+  sign_accept_encoding: false

backend/s3/provider/Linode.yaml

@@ -24,3 +24,8 @@ endpoint:
   us-iad-10.linodeobjects.com: Washington, DC, US (us-iad-10)
 acl: {}
 bucket_acl: true
+quirks:
+  # Linode Object Storage is Ceph-backed; signing Accept-Encoding causes
+  # SignatureDoesNotMatch with rclone 1.68+. Match the Ceph quirk.
+  # See: https://github.com/rclone/rclone/issues/8206
+  sign_accept_encoding: false

backend/s3/provider/Mega.yaml

@@ -1,17 +1,20 @@
 name: Mega
 description: MEGA S4 Object Storage
 endpoint:
-  s3.eu-amsterdam.megas4.com: Mega S4 Amsterdam
-  s3.eu-luxembourg.megas4.com: Mega S4 Luxembourg
-  s3.eu-paris.megas4.com: Mega S4 Paris
-  s3.eu-barcelona.megas4.com: Mega S4 Barcelona
-  s3.ca-montreal.megas4.com: Mega S4 Montreal
-  s3.ca-vancouver.megas4.com: Mega S4 Vancouver
-  s3.ap-tokyo.megas4.com: Mega S4 Tokyo
-  s3.eu-central-1.s4.mega.io: Mega S4 eu-central-1 (Amsterdam, legacy)
-  s3.eu-central-2.s4.mega.io: Mega S4 eu-central-2 (Bettembourg, legacy)
-  s3.ca-central-1.s4.mega.io: Mega S4 ca-central-1 (Montreal, legacy)
-  s3.ca-west-1.s4.mega.io: Mega S4 ca-west-1 (Vancouver, legacy)
+  s3.eu-luxembourg-1.megas4.com: Mega S4 Luxembourg 1
+  s3.eu-luxembourg-2.megas4.com: Mega S4 Luxembourg 2
+  s3.eu-amsterdam-1.megas4.com: Mega S4 Amsterdam 1
+  s3.eu-amsterdam-2.megas4.com: Mega S4 Amsterdam 2
+  s3.eu-paris-1.megas4.com: Mega S4 Paris 1
+  s3.eu-paris-2.megas4.com: Mega S4 Paris 2
+  s3.eu-barcelona-1.megas4.com: Mega S4 Barcelona 1
+  s3.eu-barcelona-2.megas4.com: Mega S4 Barcelona 2
+  s3.ca-montreal-1.megas4.com: Mega S4 Montreal 1
+  s3.ca-montreal-2.megas4.com: Mega S4 Montreal 2
+  s3.ca-vancouver-1.megas4.com: Mega S4 Vancouver 1
+  s3.ca-vancouver-2.megas4.com: Mega S4 Vancouver 2
+  s3.ap-tokyo-1.megas4.com: Mega S4 Tokyo 1
+  s3.ap-tokyo-2.megas4.com: Mega S4 Tokyo 2
 bucket_acl: true
 quirks:
   list_version: 2

backend/s3/s3.go

@@ -1357,10 +1357,10 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
 }
 
 // s3RedirectSecretHeaders are the request headers carrying origin-bound
-// secrets that must not be forwarded when a redirect crosses a host or
-// downgrades the scheme. Go strips Authorization on a hostname change but not
-// on a scheme downgrade, and has no knowledge that the SSE-C headers hold raw
-// encryption keys, so we strip them all ourselves.
+// secrets that must not be forwarded when a redirect leaves the original
+// host. Go strips Authorization on a hostname change but keeps it for a
+// subdomain or a different port, and has no knowledge that the SSE-C
+// headers hold raw encryption keys, so we strip them all ourselves.
 var s3RedirectSecretHeaders = []string{
 	"X-Amz-Security-Token",  // AWS STS session token
 	"X-Amz-S3session-Token", // S3 Express (directory bucket) session token
@@ -1375,36 +1375,10 @@ var s3RedirectSecretHeaders = []string{
 	"Referer", // may be a presigned request
 }
 
-func s3CheckRedirect(req *http.Request, via []*http.Request) error {
-	if len(via) >= 10 {
-		return errors.New("stopped after 10 redirects")
-	}
-	// Never follow a redirect that downgrades the transport from HTTPS to
-	// HTTP. An S3 endpoint has no legitimate reason to do this, and replaying
-	// the request over plaintext would expose whatever credential it carries.
-	if via[len(via)-1].URL.Scheme == "https" && req.URL.Scheme == "http" {
-		return fmt.Errorf("refusing to follow insecure redirect from HTTPS to HTTP: %s", req.URL.Redacted())
-	}
-	if s3RedirectCrossesHost(req, via) {
-		for _, header := range s3RedirectSecretHeaders {
-			req.Header.Del(header)
-		}
-	}
-	return nil
-}
-
-func s3RedirectCrossesHost(req *http.Request, via []*http.Request) bool {
-	if len(via) == 0 {
-		return false
-	}
-	scheme, host := via[0].URL.Scheme, via[0].URL.Host
-	for _, redirect := range via[1:] {
-		if redirect.URL.Host != host || redirect.URL.Scheme != scheme {
-			return true
-		}
-	}
-	return host != req.URL.Host || scheme != req.URL.Scheme
-}
+// s3CheckRedirect is the http.Client CheckRedirect policy for S3. It
+// refuses HTTPS to HTTP downgrades and strips s3RedirectSecretHeaders
+// once a redirect has left the original host.
+var s3CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(s3RedirectSecretHeaders...)
 
 // Fixup the request if needed.
 //
@@ -2317,7 +2291,31 @@ func versionLess(a, b *types.ObjectVersion) bool {
 // types.ObjectVersion with Size = isDeleteMarker to tell them apart
 //
 // We then merge them back into the Versions in the correct order
-func mergeDeleteMarkers(oldVersions []types.ObjectVersion, deleteMarkers []types.DeleteMarkerEntry) (newVersions []types.ObjectVersion) {
+func mergeDeleteMarkers(oldVersions []types.ObjectVersion, deleteMarkers []types.DeleteMarkerEntry, urlEncoded bool) (newVersions []types.ObjectVersion) {
+	encodedKeys := make(map[string]string)
+	if urlEncoded {
+		// URL encoding can change key order, so compare decoded keys and restore the encoded keys for the caller.
+		oldVersions = append([]types.ObjectVersion(nil), oldVersions...)
+		deleteMarkers = append([]types.DeleteMarkerEntry(nil), deleteMarkers...)
+		decodeKey := func(key **string) {
+			if *key == nil {
+				return
+			}
+			encodedKey := **key
+			decodedKey, err := url.QueryUnescape(encodedKey)
+			if err != nil {
+				return
+			}
+			encodedKeys[decodedKey] = encodedKey
+			*key = &decodedKey
+		}
+		for i := range oldVersions {
+			decodeKey(&oldVersions[i].Key)
+		}
+		for i := range deleteMarkers {
+			decodeKey(&deleteMarkers[i].Key)
+		}
+	}
 	newVersions = make([]types.ObjectVersion, 0, len(oldVersions)+len(deleteMarkers))
 	for _, deleteMarker := range deleteMarkers {
 		var obj types.ObjectVersion
@@ -2332,6 +2330,11 @@ func mergeDeleteMarkers(oldVersions []types.ObjectVersion, deleteMarkers []types
 	}
 	// Merge any remaining versions
 	newVersions = append(newVersions, oldVersions...)
+	for i := range newVersions {
+		if encodedKey, ok := encodedKeys[deref(newVersions[i].Key)]; ok {
+			newVersions[i].Key = &encodedKey
+		}
+	}
 	return newVersions
 }
 
@@ -2371,7 +2374,7 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
 
 	// Merge in delete Markers as types.ObjectVersion if we need them
 	if ls.hidden || ls.usingVersionAt {
-		respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers)
+		respVersions.Versions = mergeDeleteMarkers(respVersions.Versions, respVersions.DeleteMarkers, ls.req.EncodingType == types.EncodingTypeUrl)
 	}
 
 	// Convert the Versions and the DeleteMarkers into an array of types.Object
@@ -2398,6 +2401,8 @@ func (ls *versionsList) List(ctx context.Context) (resp *s3.ListObjectsV2Output,
 		var obj types.Object
 		//structs.SetFrom(obj, objVersion)
 		setFrom_typesObject_typesObjectVersion(&obj, &objVersion)
+		// StorageClass has a different type in ObjectVersion so isn't copied by setFrom
+		obj.StorageClass = types.ObjectStorageClass(objVersion.StorageClass)
 		// Adjust the file names
 		if !ls.usingVersionAt && (!deref(objVersion.IsLatest) || objVersion.Size == isDeleteMarker) {
 			if obj.Key != nil && objVersion.LastModified != nil {
@@ -3278,6 +3283,16 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
+	// With NoHeadObject no metadata was read for the new object, so carry
+	// the size and MD5 over from the source as a server-side copy produces
+	// an object with identical content.
+	if f.opt.NoHeadObject {
+		if dstObject, ok := dstObj.(*Object); ok {
+			dstObject.bytes = srcObj.bytes
+			dstObject.md5 = srcObj.md5
+		}
+	}
+
 	// Set Object Lock via separate API calls if requested
 	if f.opt.ObjectLockSetAfterUpload {
 		if dstObject, ok := dstObj.(*Object); ok {
@@ -4657,6 +4672,16 @@ func (w *s3ChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader
 			// retry all chunks once have done the first few
 			return true, err
 		}
+		if uout == nil || uout.ETag == nil {
+			// A successful UploadPart without an ETag header is unusable: the
+			// part ETag is required by CompleteMultipartUpload. Proxies and
+			// load balancers have been observed emitting empty 200 responses
+			// under load - see #9822. Treat it as a retryable error so the
+			// pacer retries this chunk, instead of dereferencing a nil ETag
+			// in the debug log below or completing the upload with a broken
+			// part list.
+			return true, fmt.Errorf("UploadPart response for chunk %d has no ETag", chunkNumber+1)
+		}
 		return false, nil
 	})
 	if err != nil {

backend/s3/s3_internal_test.go

@@ -7,6 +7,9 @@ import (
 	"crypto/md5"
 	"errors"
 	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"path"
 	"strings"
 	"testing"
@@ -144,6 +147,33 @@ func (f *Fs) InternalTestNoHead(t *testing.T) {
 
 }
 
+func (f *Fs) InternalTestNoHeadObjectCopy(t *testing.T) {
+	ctx := context.Background()
+	contents := random.String(1000)
+	item := fstest.NewItem("test-no-head-object-copy-src", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+	src := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+	defer func() {
+		assert.NoError(t, src.Remove(ctx))
+	}()
+	// Set NoHeadObject for this test so the copied object's metadata is not read back
+	f.opt.NoHeadObject = true
+	defer func() {
+		f.opt.NoHeadObject = false
+	}()
+	dst, err := f.Copy(ctx, src, "test-no-head-object-copy-dst")
+	require.NoError(t, err)
+	defer func() {
+		assert.NoError(t, dst.Remove(ctx))
+	}()
+	assert.Equal(t, src.Size(), dst.Size())
+	srcHash, err := src.Hash(ctx, hash.MD5)
+	require.NoError(t, err)
+	dstHash, err := dst.Hash(ctx, hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, srcHash, dstHash)
+	assert.NotEqual(t, "", dstHash)
+}
+
 func (f *Fs) InternalTestHasChildren(t *testing.T) {
 	ctx := context.Background()
 	contents := random.String(100)
@@ -275,11 +305,64 @@ func TestMergeDeleteMarkers(t *testing.T) {
 			},
 		},
 	} {
-		got := mergeDeleteMarkers(test.versions, test.markers)
+		got := mergeDeleteMarkers(test.versions, test.markers, false)
 		assert.Equal(t, test.want, got, fmt.Sprintf("%d: %+v", n, test))
 	}
 }
 
+func TestMergeDeleteMarkersWithURLEncodedKeys(t *testing.T) {
+	plainKey := "images/reservations/photo.png"
+	encodedKey := "images/%D0%9F%D1%80%D0%B8%D0%B2%D0%B5%D1%82-01.jpg"
+	t1 := fstest.Time("2022-01-21T12:00:00+01:00")
+	t2 := fstest.Time("2022-01-21T12:00:01+01:00")
+	versions := []types.ObjectVersion{
+		{Key: &plainKey, LastModified: &t2},
+		{Key: &encodedKey, LastModified: &t1},
+	}
+	markers := []types.DeleteMarkerEntry{
+		{Key: &encodedKey, LastModified: &t2},
+	}
+
+	got := mergeDeleteMarkers(versions, markers, true)
+	want := []types.ObjectVersion{
+		{Key: &plainKey, LastModified: &t2},
+		{Key: &encodedKey, LastModified: &t2, Size: isDeleteMarker},
+		{Key: &encodedKey, LastModified: &t1},
+	}
+	assert.Equal(t, want, got)
+}
+
+func TestVersionsListStorageClass(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.WriteString(w, `<ListVersionsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
+  <Name>bucket</Name>
+  <IsTruncated>false</IsTruncated>
+  <Version>
+    <Key>archived.bin</Key>
+    <VersionId>v1</VersionId>
+    <IsLatest>true</IsLatest>
+    <LastModified>2026-09-26T12:00:00.000Z</LastModified>
+    <ETag>"d41d8cd98f00b204e9800998ecf8427e"</ETag>
+    <Size>0</Size>
+    <StorageClass>DEEP_ARCHIVE</StorageClass>
+  </Version>
+</ListVersionsResult>`)
+	}))
+	defer srv.Close()
+
+	f := &Fs{c: s3.New(s3.Options{
+		Region:       "us-east-1",
+		BaseEndpoint: aws.String(srv.URL),
+		UsePathStyle: true,
+		Credentials:  aws.AnonymousCredentials{},
+	})}
+	ls := f.newVersionsList(&s3.ListObjectsV2Input{Bucket: aws.String("bucket")}, false, time.Time{})
+	resp, _, err := ls.List(context.Background())
+	require.NoError(t, err)
+	require.Len(t, resp.Contents, 1)
+	assert.Equal(t, types.ObjectStorageClassDeepArchive, resp.Contents[0].StorageClass)
+}
+
 func TestRemoveAWSChunked(t *testing.T) {
 	ps := func(s string) *string {
 		return &s
@@ -810,6 +893,7 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("NoHead", f.InternalTestNoHead)
+	t.Run("NoHeadObjectCopy", f.InternalTestNoHeadObjectCopy)
 	t.Run("HasChildren", f.InternalTestHasChildren)
 	t.Run("Versions", f.InternalTestVersions)
 	t.Run("ObjectLock", f.InternalTestObjectLock)

backend/s3/s3_test.go

@@ -228,54 +228,6 @@ func TestS3CheckRedirectRejectsSchemeDowngrade(t *testing.T) {
 	}
 }
 
-func TestRedirectCrossesHost(t *testing.T) {
-	mustReq := func(method, url string) *http.Request {
-		req, err := http.NewRequest(method, url, nil)
-		require.NoError(t, err)
-		return req
-	}
-	for _, test := range []struct {
-		name string
-		via  []string
-		req  string
-		want bool
-	}{
-		{
-			name: "SameHost",
-			via:  []string{"https://bucket.example.com/"},
-			req:  "https://bucket.example.com/redirected",
-			want: false,
-		},
-		{
-			name: "DifferentHost",
-			via:  []string{"https://bucket.example.com/"},
-			req:  "https://evil.example.com/redirected",
-			want: true,
-		},
-		{
-			name: "SchemeDowngradeSameHost",
-			via:  []string{"https://bucket.example.com/"},
-			req:  "http://bucket.example.com/redirected",
-			want: true,
-		},
-		{
-			name: "SchemeDowngradeMidChain",
-			via:  []string{"https://bucket.example.com/", "http://bucket.example.com/middle"},
-			req:  "http://bucket.example.com/final",
-			want: true,
-		},
-	} {
-		t.Run(test.name, func(t *testing.T) {
-			via := make([]*http.Request, len(test.via))
-			for i, url := range test.via {
-				via[i] = mustReq(http.MethodGet, url)
-			}
-			got := s3RedirectCrossesHost(mustReq(http.MethodGet, test.req), via)
-			assert.Equal(t, test.want, got)
-		})
-	}
-}
-
 func TestClientStopsAfterTenRedirects(t *testing.T) {
 	_, _, client := SetupS3Test(t)
 

backend/seafile/object.go

@@ -88,28 +88,22 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	// The upload sometimes return a temporary 500 error
-	// We cannot use the pacer to retry uploading the file as the upload link is single use only
-	for retry := 0; retry <= 3; retry++ {
-		uploadLink, err := o.fs.getUploadLink(ctx, o.libraryID)
-		if err != nil {
-			return err
-		}
-
-		uploaded, err := o.fs.upload(ctx, in, uploadLink, o.pathInLibrary)
-		if err == ErrorInternalDuringUpload {
-			// This is a temporary error, try again with a new upload link
-			continue
-		}
-		if err != nil {
-			return err
-		}
-		// Set the properties from the upload back to the object
-		o.size = uploaded.Size
-
-		return nil
+	uploadLink, err := o.fs.getUploadLink(ctx, o.libraryID)
+	if err != nil {
+		return err
 	}
-	return ErrorInternalDuringUpload
+
+	// The upload can't be retried here as the input stream can't be re-read and
+	// the upload link is single use, so upload returns a retry
+	// error for the caller to retry with a fresh stream.
+	uploaded, err := o.fs.upload(ctx, in, uploadLink, o.pathInLibrary)
+	if err != nil {
+		return err
+	}
+	// Set the properties from the upload back to the object
+	o.size = uploaded.Size
+
+	return nil
 }
 
 // Remove this object

backend/seafile/webapi.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/rclone/rclone/backend/seafile/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 )
@@ -721,8 +722,8 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 500 {
-				// This is a temporary error - we will get a new upload link before retrying
-				return nil, ErrorInternalDuringUpload
+				// This is a temporary error - the caller will get a new upload link when it retries
+				return nil, fserrors.RetryError(ErrorInternalDuringUpload)
 			}
 		}
 		return nil, fmt.Errorf("failed to upload file: %w", err)

backend/sftp/sftp.go

@@ -2791,6 +2791,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
 	if err != nil {
+		// Close the handle otherwise the server keeps the removed file's space allocated
+		_ = file.Close()
 		o.fs.putSftpConnection(&c, err)
 		remove()
 		return fmt.Errorf("Update ReadFrom failed: %w", err)

backend/shade/shade.go

@@ -380,9 +380,11 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	//Need to check if destination exists
 	fullPath := f.buildFullPath(dstRemote)
 	var response api.ListDirResponse
-	res, _ := f.callAPI(ctx, "GET", fmt.Sprintf("/%s/fs/attr?path=%s", f.drive, fullPath), &response)
-
-	if res.StatusCode != http.StatusNotFound {
+	res, err := f.callAPI(ctx, "GET", fmt.Sprintf("/%s/fs/attr?path=%s", f.drive, fullPath), &response)
+	if res == nil || res.StatusCode != http.StatusNotFound {
+		if err != nil {
+			return fmt.Errorf("failed to check destination directory: %w", err)
+		}
 		return fs.ErrorDirExists
 	}
 

backend/sia/sia.go

@@ -24,6 +24,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -176,11 +177,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // Update the object with the contents of the io.Reader
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	size := src.Size()
+	counter := readers.NewCountingReader(in)
 	var resp *http.Response
 	opts := rest.Opts{
 		Method:        "POST",
 		Path:          path.Join("/renter/uploadstream/", o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
-		Body:          in,
+		Body:          counter,
 		ContentLength: &size,
 		Parameters:    url.Values{},
 	}
@@ -191,6 +193,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return o.fs.shouldRetry(resp, err)
 	})
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload.
+	// The partially uploaded file is cleaned up by Put which copes
+	// with it appearing asynchronously after the failed upload.
+	if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	if err == nil {
 		err = o.readMetaData(ctx)
 	}

backend/smb/smb.go

@@ -49,8 +49,7 @@ func init() {
 			Sensitive: true,
 		}, {
 			Name:      "user",
-			Help:      "SMB username.",
-			Default:   currentUser,
+			Help:      "SMB username.\n\nLeave blank to use the current user name.",
 			Sensitive: true,
 		}, {
 			Name:    "port",
@@ -191,6 +190,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
+	if opt.User == "" {
+		opt.User = currentUser
+	}
 
 	// if root is empty or ends with / (must be a directory)
 	isRootDir := isPathDir(root)
@@ -836,6 +838,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("Update Close failed: %w", err)
 	}
 
+	// Return the connection so the SetModTime below can reuse it rather than
+	// dialling a second one - the file is closed so remove() no longer needs
+	// it. This nils cn out, making the deferred putConnection a no-op.
+	o.fs.putConnection(&cn, nil)
+
 	// Set the modified time and also o.statResult
 	err = o.SetModTime(ctx, src.ModTime(ctx))
 	if err != nil {

backend/smb/smb_internal_test.go

@@ -1,7 +1,98 @@
 // Unit tests for internal SMB functions
 package smb
 
-import "testing"
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDialClosesConnectionOnSetupError(t *testing.T) {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	defer func() { require.NoError(t, listener.Close()) }()
+
+	type acceptResult struct {
+		conn net.Conn
+		err  error
+	}
+	accepted := make(chan acceptResult, 1)
+	go func() {
+		conn, err := listener.Accept()
+		accepted <- acceptResult{conn: conn, err: err}
+	}()
+
+	f := &Fs{opt: Options{Pass: "invalid"}}
+	_, err = f.dial(context.Background(), "tcp", listener.Addr().String())
+	require.Error(t, err)
+
+	var result acceptResult
+	select {
+	case result = <-accepted:
+	case <-time.After(time.Second):
+		t.Fatal("timed out waiting for server to accept connection")
+	}
+	require.NoError(t, result.err)
+	defer func() { require.NoError(t, result.conn.Close()) }()
+	require.NoError(t, result.conn.SetReadDeadline(time.Now().Add(time.Second)))
+
+	buffer := make([]byte, 1)
+	n, err := result.conn.Read(buffer)
+	require.Zero(t, n)
+	require.ErrorIs(t, err, io.EOF)
+}
+
+// TestUploadConnectionReuse checks an upload leaves only one connection in the
+// pool, ie the connection it used is available again for the SetModTime which
+// follows it rather than a second one being dialled.
+//
+// This needs a real SMB server so it is skipped if one isn't configured.
+func TestUploadConnectionReuse(t *testing.T) {
+	ctx := context.Background()
+	fstest.Initialise()
+	remoteName := *fstest.RemoteName
+	if remoteName == "" {
+		remoteName = "TestSMB:rclone"
+	}
+	remote, err := fs.NewFs(ctx, remoteName)
+	if errors.Is(err, fs.ErrorNotFoundInConfigFile) {
+		t.Skipf("skipping as %q is not configured", remoteName)
+	}
+	require.NoError(t, err)
+	f, ok := remote.(*Fs)
+	if !ok {
+		t.Skipf("skipping as %q is not an SMB remote", remoteName)
+	}
+
+	defer func() { require.NoError(t, f.Shutdown(ctx)) }()
+
+	// Empty the pool so the connections counted below are only the upload's
+	require.NoError(t, f.drainPool(ctx))
+
+	const contents = "connection reuse test"
+	remotePath := fmt.Sprintf("rclone-test-connection-reuse-%d.txt", time.Now().UnixNano())
+	src := object.NewStaticObjectInfo(remotePath, time.Now(), int64(len(contents)), true, nil, nil)
+	o, err := f.Put(ctx, strings.NewReader(contents), src)
+	require.NoError(t, err)
+	defer func() { require.NoError(t, o.Remove(ctx)) }()
+
+	f.poolMu.Lock()
+	pooled := len(f.pool)
+	f.poolMu.Unlock()
+	assert.Equal(t, 1, pooled, "upload should leave exactly one connection in the pool")
+}
 
 // TestIsPathDir tests the isPathDir function logic
 func TestIsPathDir(t *testing.T) {
@@ -39,3 +130,33 @@ func TestIsPathDir(t *testing.T) {
 		})
 	}
 }
+
+func TestNewFsUser(t *testing.T) {
+	ctx := context.Background()
+	for _, test := range []struct {
+		user string
+		want string
+	}{
+		{user: "", want: currentUser},
+		{user: currentUser, want: currentUser},
+		{user: "someone", want: "someone"},
+	} {
+		m := configmap.Simple{"host": "localhost", "user": test.user}
+		f, err := NewFs(ctx, "TestSMB", "", m)
+		require.NoError(t, err)
+		assert.Equal(t, test.want, f.(*Fs).opt.User, "user=%q", test.user)
+	}
+
+	// No user in the config at all
+	f, err := NewFs(ctx, "TestSMB", "", configmap.Simple{"host": "localhost"})
+	require.NoError(t, err)
+	assert.Equal(t, currentUser, f.(*Fs).opt.User)
+}
+
+// The default is blank so that a user name which happens to match the
+// current user is still written to the config file.
+func TestUserDefault(t *testing.T) {
+	ri, err := fs.Find("smb")
+	require.NoError(t, err)
+	assert.Equal(t, "", ri.Options.Get("user").Default)
+}

backend/webdav/webdav.go

@@ -135,6 +135,13 @@ The input format is comma separated list of key,value pairs.  Standard
 For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
 
 You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
+
+The headers are only sent to the host in the configured URL. If the
+server redirects to another host (including a subdomain or a different
+port) the headers are not sent to it, or to any further hop in that
+redirect chain. The exception is the directory listing and file
+download requests when auth_redirect is set, as that keeps all
+credentials across redirects.
 `,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
@@ -233,7 +240,10 @@ type Fs struct {
 	ntlmAuthMu         sync.Mutex    // mutex to serialize NTLM auth roundtrips
 	chunksUploadURL    string        // upload URL for nextcloud chunked
 	canChunk           bool          // set if nextcloud and nextcloud_chunk_size is set
+	canRecalcHash      bool          // set if the server can recalculate checksums with PATCH (nextcloud)
 	authSingleflight   *singleflight.Group
+	// redirect policy which keeps opt.Headers on the configured host
+	checkRedirect func(req *http.Request, via []*http.Request) error
 }
 
 // Object describes a webdav object
@@ -344,6 +354,41 @@ func itemIsDir(item *api.Response) bool {
 	return false
 }
 
+// sabreNotFound is the exception SabreDAV based servers report for a
+// path which doesn't exist
+var sabreNotFound = []byte(`Sabre\DAV\Exception\NotFound`)
+
+// callPropfind does the PROPFIND in opts and decodes the response into result
+//
+// ownCloud 10.16 starts a 207 Multi-Status response before it finds
+// that the path doesn't exist, then appends a Sabre NotFound error to
+// it which makes it invalid XML. This is returned as the 404 *api.Error
+// which servers normally give for a path which doesn't exist.
+func (f *Fs) callPropfind(ctx context.Context, opts *rest.Opts, result *api.Multistatus) (resp *http.Response, err error) {
+	resp, err = f.srv.Call(ctx, opts)
+	if err != nil {
+		return resp, err
+	}
+	// Keep the start of the body to look for the error in
+	start := make([]byte, 4096)
+	n, _ := io.ReadFull(resp.Body, start)
+	start = start[:n]
+	body := resp.Body
+	resp.Body = struct {
+		io.Reader
+		io.Closer
+	}{io.MultiReader(bytes.NewReader(start), body), body}
+	err = rest.DecodeXML(resp, result)
+	if err != nil && bytes.Contains(start, sabreNotFound) {
+		return resp, &api.Error{
+			Exception:  string(sabreNotFound),
+			Status:     http.StatusText(http.StatusNotFound),
+			StatusCode: http.StatusNotFound,
+		}
+	}
+	return resp, err
+}
+
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Prop, err error) {
 	// FIXME how do we read back additional properties?
@@ -353,7 +398,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Pr
 		ExtraHeaders: map[string]string{
 			"Depth": "0",
 		},
-		CheckRedirect: rest.PreserveMethodRedirectFn,
+		CheckRedirect: f.preserveMethodRedirect,
 	}
 	if f.hasOCMD5 || f.hasOCSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
@@ -364,7 +409,11 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Pr
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		var attempt api.Multistatus
+		resp, err = f.callPropfind(ctx, &opts, &attempt)
+		if err == nil {
+			result = attempt
+		}
 		return f.shouldRetry(ctx, resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
@@ -515,8 +564,17 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			rt: ntlmssp.Negotiator{RoundTripper: t},
 		}
 	}
-	// Refuse redirects that downgrade HTTPS to plaintext HTTP.
-	client.CheckRedirect = rest.RefuseHTTPSDowngradeRedirectFn
+	// Refuse redirects that downgrade HTTPS to plaintext HTTP and
+	// don't send the configured headers to another host.
+	//
+	// The auth_redirect option bypasses this for the requests it applies
+	// to as it is an explicit opt-in to keep credentials across redirects.
+	headerNames := make([]string, 0, len(opt.Headers)/2)
+	for i := 0; i < len(opt.Headers); i += 2 {
+		headerNames = append(headerNames, opt.Headers[i])
+	}
+	f.checkRedirect = rest.StripHeadersOnCrossHostRedirectFn(headerNames...)
+	client.CheckRedirect = f.checkRedirect
 	f.srv = rest.NewClient(client).SetRoot(u.String())
 
 	f.features = (&fs.Features{
@@ -595,6 +653,16 @@ func (f *Fs) fetchBearerToken(cmd fs.SpaceSepList) (string, error) {
 	return stdoutString, nil
 }
 
+// preserveMethodRedirect is an http.Client.CheckRedirect function
+// which applies f.checkRedirect and then preserves the original HTTP
+// method like rest.PreserveMethodRedirectFn.
+func (f *Fs) preserveMethodRedirect(req *http.Request, via []*http.Request) error {
+	if err := f.checkRedirect(req, via); err != nil {
+		return err
+	}
+	return rest.PreserveMethodRedirectFn(req, via)
+}
+
 // Adds the configured headers to the request if any
 func (f *Fs) addHeaders(headers fs.CommaSepList) {
 	for i := 0; i < len(headers); i += 2 {
@@ -664,6 +732,7 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		f.propsetMtime = true
 		f.hasOCSHA1 = true
 		f.canChunk = true
+		f.canRecalcHash = true
 
 		if f.opt.ChunkSize == 0 {
 			fs.Logf(nil, "Chunked uploads are disabled because nextcloud_chunk_size is set to 0")
@@ -813,7 +882,11 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
+		var attempt api.Multistatus
+		resp, err = f.callPropfind(ctx, &opts, &attempt)
+		if err == nil {
+			result = attempt
+		}
 		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
@@ -1471,11 +1544,16 @@ var owncloudPropsetWithChecksum = `<?xml version="1.0" encoding="utf-8" ?>
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if o.fs.propsetMtime {
+		// Setting the modification time discards the stored
+		// checksums so they are set again in the same request.
+		// Servers which can't do that recalculate them afterwards.
 		checksums := ""
-		if o.fs.hasOCSHA1 && o.sha1 != "" {
-			checksums = "SHA1:" + o.sha1
-		} else if o.fs.hasOCMD5 && o.md5 != "" {
-			checksums = "MD5:" + o.md5
+		if !o.fs.canRecalcHash {
+			if o.fs.hasOCSHA1 && o.sha1 != "" {
+				checksums = "SHA1:" + o.sha1
+			} else if o.fs.hasOCMD5 && o.md5 != "" {
+				checksums = "MD5:" + o.md5
+			}
 		}
 
 		opts := rest.Opts{
@@ -1510,6 +1588,12 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 			// in-memory modtime identical to the one a fresh
 			// listing returns
 			o.modTime = modTime.Truncate(time.Second)
+			if o.fs.canRecalcHash && (o.sha1 != "" || o.md5 != "") {
+				err = o.recalculateHash(ctx)
+				if err != nil {
+					return fmt.Errorf("couldn't restore checksum after setting modified time: %w", err)
+				}
+			}
 			return nil
 		}
 		// got an error, but it's possible it actually worked, so double-check
@@ -1596,7 +1680,54 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	// read metadata from remote
 	o.hasMetaData = false
-	return o.readMetaData(ctx)
+	err = o.readMetaData(ctx)
+	if err != nil {
+		return err
+	}
+	// The server only stores a checksum supplied with the upload,
+	// so ask the server to calculate one if it is missing.
+	if o.fs.canRecalcHash && o.sha1 == "" && o.md5 == "" {
+		err = o.recalculateHash(ctx)
+		if err != nil {
+			return fmt.Errorf("couldn't calculate checksum after upload: %w", err)
+		}
+	}
+	return nil
+}
+
+// recalculateHash asks the server to calculate and store the checksum
+// of the object's contents, updating the cached hash from the result.
+//
+// This uses the nextcloud PATCH extension with the X-Recalculate-Hash
+// header which returns the checksum in the OC-Checksum header.
+func (o *Object) recalculateHash(ctx context.Context) error {
+	hashName := ""
+	if o.fs.hasOCSHA1 {
+		hashName = "sha1"
+	} else if o.fs.hasOCMD5 {
+		hashName = "md5"
+	} else {
+		return nil
+	}
+	opts := rest.Opts{
+		Method:       "PATCH",
+		Path:         o.filePath(),
+		NoResponse:   true,
+		ExtraHeaders: map[string]string{"X-Recalculate-Hash": hashName},
+	}
+	var resp *http.Response
+	err := o.fs.pacer.Call(func() (bool, error) {
+		var err error
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	hashes := (&api.Prop{Checksums: []string{resp.Header.Get("OC-Checksum")}}).Hashes()
+	o.sha1 = hashes[hash.SHA1]
+	o.md5 = hashes[hash.MD5]
+	return nil
 }
 
 func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string]string {

backend/webdav/webdav_internal_test.go

@@ -3,9 +3,12 @@ package webdav_test
 import (
 	"context"
 	"fmt"
+	"io"
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"testing"
 
 	"github.com/rclone/rclone/backend/webdav"
@@ -120,6 +123,114 @@ func TestListAllAuthRedirect(t *testing.T) {
 	assert.NotEmpty(t, targetAuth, "Authorization header should be preserved across redirect")
 }
 
+// fileInfoResponse is a PROPFIND response describing a single file.
+const fileInfoResponse = `<d:multistatus xmlns:d="DAV:">
+<d:response>
+ <d:href>/file.txt</d:href>
+ <d:propstat>
+  <d:prop>
+   <d:getcontentlength>10</d:getcontentlength>
+   <d:resourcetype/>
+  </d:prop>
+  <d:status>HTTP/1.1 200 OK</d:status>
+ </d:propstat>
+</d:response>
+</d:multistatus>`
+
+// redirectTestHeaders are the headers option for the redirect tests.
+// The one starting with "*" is sent without canonicalising.
+var redirectTestHeaders = []string{"X-Potato", "sausage", "*x-marrow", "pea"}
+
+// redirectHeaders makes a webdav remote with redirectTestHeaders
+// pointing at a server which redirects every request to a second
+// server on hostname, or back to itself if hostname is empty, and
+// exercises each of the backend's redirect policies against it: the
+// PROPFIND used to look up an object, the GET used to read it and the
+// PROPFIND used to list a directory. It returns the headers the
+// redirect target saw on each of those requests.
+func redirectHeaders(t *testing.T, hostname string) []http.Header {
+	var target string
+	var mu sync.Mutex
+	var seen []http.Header
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, target+"/moved"+r.URL.Path, http.StatusTemporaryRedirect)
+			return
+		}
+		mu.Lock()
+		seen = append(seen, r.Header.Clone())
+		mu.Unlock()
+		switch r.Method {
+		case "PROPFIND":
+			w.WriteHeader(http.StatusMultiStatus)
+			_, err := fmt.Fprint(w, fileInfoResponse)
+			assert.NoError(t, err)
+		case http.MethodGet:
+			_, err := fmt.Fprint(w, "hello")
+			assert.NoError(t, err)
+		default:
+			http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
+		}
+	})
+	ts := httptest.NewServer(handler)
+	defer ts.Close()
+	if hostname == "" {
+		target = ts.URL
+	} else {
+		// Serve the final response from a second server on a different host
+		other := httptest.NewServer(handler)
+		defer other.Close()
+		target = strings.Replace(other.URL, "127.0.0.1", hostname, 1)
+	}
+
+	configfile.Install()
+	m := configmap.Simple{
+		"type":    "webdav",
+		"url":     ts.URL,
+		"headers": strings.Join(redirectTestHeaders, ","),
+	}
+	ctx := context.Background()
+	f, err := webdav.NewFs(ctx, remoteName, "", m)
+	require.NoError(t, err)
+
+	// PROPFIND with Depth 0 uses rest.PreserveMethodRedirectFn
+	o, err := f.NewObject(ctx, "file.txt")
+	require.NoError(t, err)
+	// GET uses the client's default redirect policy
+	fd, err := o.Open(ctx)
+	require.NoError(t, err)
+	data, err := io.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	assert.Equal(t, "hello", string(data))
+	// PROPFIND with Depth 1 uses the client's default redirect policy
+	_, err = f.List(ctx, "")
+	require.NoError(t, err)
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Len(t, seen, 3, "expected the redirect target to see all three requests")
+	return seen
+}
+
+func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
+	for _, got := range redirectHeaders(t, "") {
+		for i := 0; i < len(redirectTestHeaders); i += 2 {
+			assert.Equal(t, redirectTestHeaders[i+1], got.Get(strings.TrimPrefix(redirectTestHeaders[i], "*")))
+		}
+	}
+}
+
+func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
+	// Redirect to a different host name and port. net/http strips
+	// Authorization on its own here so this checks the custom headers.
+	for _, got := range redirectHeaders(t, "localhost") {
+		for i := 0; i < len(redirectTestHeaders); i += 2 {
+			assert.Empty(t, got.Values(strings.TrimPrefix(redirectTestHeaders[i], "*")), "header %q leaked to redirect target", redirectTestHeaders[i])
+		}
+	}
+}
+
 // TestReservedCharactersInPathAreEscaped verifies that reserved characters
 // like semicolons and equals signs in file paths are percent-encoded in
 // HTTP requests to the WebDAV server (RFC 3986 compliance).
@@ -151,3 +262,119 @@ func TestReservedCharactersInPathAreEscaped(t *testing.T) {
 	assert.Contains(t, capturedPath, "my%3Btest", "semicolons in path should be percent-encoded")
 	assert.NotContains(t, capturedPath, "my;test", "raw semicolons should not appear in path")
 }
+
+// TestListAllRetryDoesNotConcatenate verifies that when a listing PROPFIND
+// fails after having partially decoded and is retried, the final listing is
+// exactly the retried response, with no entries carried over from the
+// failed attempt.
+func TestListAllRetryDoesNotConcatenate(t *testing.T) {
+	entryXML := func(i int) string {
+		name := fmt.Sprintf("file-%03d.bin", i)
+		return "<d:response><d:href>/" + name + "</d:href><d:propstat><d:prop>" +
+			"<d:displayname>" + name + "</d:displayname><d:getcontentlength>1024</d:getcontentlength>" +
+			"</d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response>\n"
+	}
+	head := `<?xml version="1.0" encoding="utf-8"?><d:multistatus xmlns:d="DAV:">`
+	self := head + `<d:response><d:href>/</d:href><d:propstat><d:prop><d:resourcetype><d:collection xmlns:d="DAV:"/></d:resourcetype></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>`
+	writeEntries := func(w http.ResponseWriter, from, to int) {
+		_, err := fmt.Fprint(w, head)
+		require.NoError(t, err)
+		for i := from; i <= to; i++ {
+			_, err := fmt.Fprint(w, entryXML(i))
+			require.NoError(t, err)
+		}
+	}
+
+	var listCalls atomic.Int64
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "PROPFIND", r.Method)
+		w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+		if r.Header.Get("Depth") == "0" {
+			w.WriteHeader(207)
+			_, err := fmt.Fprint(w, self)
+			require.NoError(t, err)
+			return
+		}
+		if listCalls.Add(1) == 1 {
+			// return file-000..file-010 but kill the response before it
+			// completes, so the client has parsed those entries when it
+			// hits an unexpected EOF
+			w.Header().Set("Content-Length", "1000000")
+			w.WriteHeader(207)
+			writeEntries(w, 0, 10)
+			return
+		}
+		// return the disjoint file-011..file-030 as a complete listing
+		w.WriteHeader(207)
+		writeEntries(w, 11, 30)
+		_, err := fmt.Fprint(w, "</d:multistatus>")
+		require.NoError(t, err)
+	}))
+	defer ts.Close()
+
+	configfile.Install()
+	m := configmap.Simple{
+		"type": "webdav",
+		"url":  ts.URL,
+	}
+	f, err := webdav.NewFs(context.Background(), remoteName, "", m)
+	require.NoError(t, err)
+
+	entries, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+	var remotes []string
+	for _, e := range entries {
+		remotes = append(remotes, e.Remote())
+	}
+	var want []string
+	for i := 11; i <= 30; i++ {
+		want = append(want, fmt.Sprintf("file-%03d.bin", i))
+	}
+	assert.ElementsMatch(t, want, remotes)
+}
+
+// TestNotFoundInMultistatus checks that a missing path is detected when the
+// server reports it by appending a Sabre NotFound error to a 207 response it
+// has already started, as ownCloud 10.16 does, rather than by returning 404.
+func TestNotFoundInMultistatus(t *testing.T) {
+	head := `<?xml version="1.0"?>` + "\n" + `<d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns"`
+	root := head + `><d:response><d:href>/</d:href><d:propstat><d:prop><d:resourcetype><d:collection/></d:resourcetype></d:prop><d:status>HTTP/1.1 200 OK</d:status></d:propstat></d:response></d:multistatus>`
+	notFound := head + `<?xml version="1.0" encoding="utf-8"?>
+<d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
+  <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
+  <s:message>File with name missing could not be located</s:message>
+</d:error>
+`
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		require.Equal(t, "PROPFIND", r.Method)
+		w.Header().Set("Content-Type", "application/xml; charset=utf-8")
+		w.WriteHeader(207)
+		body := notFound
+		if r.URL.Path == "/" {
+			body = root
+		}
+		_, err := fmt.Fprint(w, body)
+		require.NoError(t, err)
+	}))
+	defer ts.Close()
+
+	configfile.Install()
+	m := configmap.Simple{
+		"type": "webdav",
+		"url":  ts.URL,
+	}
+	ctx := context.Background()
+
+	// A root which doesn't exist yet
+	_, err := webdav.NewFs(ctx, remoteName, "missing", m)
+	require.NoError(t, err)
+
+	f, err := webdav.NewFs(ctx, remoteName, "", m)
+	require.NoError(t, err)
+
+	_, err = f.NewObject(ctx, "missing")
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
+
+	_, err = f.List(ctx, "missing")
+	assert.Equal(t, fs.ErrorDirNotFound, err)
+}

backend/yandex/yandex.go

@@ -1189,6 +1189,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload.
+	if size := src.Size(); size >= 0 && int64(in1.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in1.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	// Set the modTime of the uploaded file and re-read the metadata
 	// so the object has the md5sum the server computed for the upload.
 	//

bin/test_proxy.py

@@ -4,13 +4,29 @@ A demo proxy for rclone serve sftp/webdav/ftp, etc.
 
 This takes the incoming user/pass and converts it into an sftp backend
 running on localhost.
+
+Logins from outside ALLOWED_NETWORKS are refused.
 """
 
 import sys
 import json
+import ipaddress
+
+ALLOWED_NETWORKS = ["127.0.0.0/8", "::1/128"]
+
+def allowed(ip):
+    """Return True if ip is in one of ALLOWED_NETWORKS."""
+    if ip is None:
+        return False
+    address = ipaddress.ip_address(ip)
+    return any(address in ipaddress.ip_network(network) for network in ALLOWED_NETWORKS)
 
 def main():
     i = json.load(sys.stdin)
+    # Exiting non zero refuses the login - rclone logs whatever we
+    # write on stderr, so say why.
+    if not allowed(i.get("client_ip")):
+        sys.exit("client_ip %s not allowed" % i.get("client_ip"))
     o = {
         "type": "sftp",              # type of backend
         "_root": "",                 # root of the fs

cmd/archive/archive_test.go

@@ -4,6 +4,8 @@ import (
 	"archive/zip"
 	"bytes"
 	"context"
+	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -134,6 +136,47 @@ func testArchiveRemote(t *testing.T, fromLocal bool, subDir string, extension st
 	fstest.CheckListingWithPrecision(t, src, items, nil, fs.ModTimeNotSupported)
 }
 
+// test archiving to stdout when no destination is given
+func TestArchiveStdout(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+
+	f1 := r.WriteObject(ctx, "file1.txt", "content 1", t1)
+	f2 := r.WriteObject(ctx, "dir1/sub1.txt", "sub content 1", t1)
+	fstest.CheckItems(t, r.Fremote, f1, f2)
+
+	// redirect stdout into a file on the local fs
+	archiveName := "stdout.zip"
+	require.NoError(t, r.Flocal.Mkdir(ctx, ""))
+	out, err := os.Create(filepath.Join(r.LocalName, archiveName))
+	require.NoError(t, err)
+	oldStdout := os.Stdout
+	os.Stdout = out
+	err = create.ArchiveCreate(ctx, nil, "", r.Fremote, "zip", "")
+	os.Stdout = oldStdout
+	require.NoError(t, out.Close())
+	require.NoError(t, err)
+
+	// check the archive written to stdout is readable
+	expected := map[string]bool{
+		"file1.txt":     true,
+		"dir1/":         true,
+		"dir1/sub1.txt": true,
+	}
+	listFile := func(ctx context.Context, f archives.FileInfo) error {
+		name := f.NameInArchive
+		if f.IsDir() && !strings.HasSuffix(name, "/") {
+			name += "/"
+		}
+		assert.True(t, expected[name], name)
+		delete(expected, name)
+		return nil
+	}
+	err = list.ArchiveList(ctx, r.Flocal, archiveName, listFile)
+	require.NoError(t, err)
+	assert.Equal(t, 0, len(expected), expected)
+}
+
 func testArchive(t *testing.T) {
 	var extensions = []string{
 		"zip",