rc: fix LogLevel in _config being ignored

Before this change setting LogLevel in the _config parameter of an rc
call had no effect as both the log functions and the log handler only
looked at the global log level.

The context aware log functions now use the log level from the config
in the context, and the log handler lets a log level in the context
which differs from the global one override its own level. This means
an rc call can be made more or less verbose than the rest of rclone
with, for example

    "_config": {"LogLevel": "DEBUG"}

This applies to the logs made with the context aware log functions, so
not to those made by the backends which still use the global log level.

Note that a config copied into a context no longer follows later
changes to the global log level.

.gitattributes

@@ -2,6 +2,9 @@
 go.mod text eol=lf
 go.sum text eol=lf
 
+# Embedded templates must render the same on every OS
+lib/http/templates/** text eol=lf
+
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true

.github/workflows/build.yml

@@ -32,12 +32,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.25']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.26']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '~1.26.5'
+            go: '~1.27.2'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -48,14 +48,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '~1.26.5'
+            go: '~1.27.2'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '~1.26.5'
+            go: '~1.27.2'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -64,14 +64,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '~1.26.5'
+            go: '~1.27.2'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '~1.26.5'
+            go: '~1.27.2'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '~1.26.5'
+            go: '~1.27.2'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.25
+          - job_name: go1.26
             os: ubuntu-latest
-            go: '~1.25.12'
+            go: '~1.26.9'
             quicktest: true
             racequicktest: true
 
@@ -179,6 +179,7 @@ jobs:
           unset HOMEBREW_NO_INSTALL_FROM_API
           brew untap --force homebrew/core
           brew untap --force homebrew/cask
+          brew untap --force aws/tap
           brew update
           brew install --cask macfuse
           brew install git-annex git-annex-remote-rclone
@@ -283,7 +284,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.26.5'
+          go-version: '~1.27.2'
           check-latest: true
           cache: false
           
@@ -398,7 +399,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.26.5'
+          go-version: '~1.27.2'
           # Caching is handled explicitly below to share the module cache
           # with the other jobs - see the build job for the rationale.
           cache: false

.golangci.yml

@@ -46,6 +46,11 @@ linters:
         - -ST1022
         # Disable quickfix checks
         - -QF*
+        # Disabled because its dataflow analysis (new in staticcheck
+        # 0.8.0) makes linting large packages more than 10x slower,
+        # which takes the CI lint job past its timeout, and it
+        # produces false positives (eg on operations.Delete).
+        - -SA4023
     gocritic:
       # With gocritic there are different settings, but since enabled-checks
       # and disabled-checks cannot both be set, for full customization the

AGENTS.md

@@ -20,6 +20,8 @@ The core parts of rclone under `fs` and `vfs` need to work with all backends and
 
 Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
 
+Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
+
 ## Build and Test Commands
 
 ```bash

CONTRIBUTING.md

@@ -630,10 +630,37 @@ remote or an fs.
 - **Remember** we have >50 backends to maintain so keeping them as similar as
   possible to each other is a high priority!
 
+### Managing memory
+
+Use rclone's memory pool for upload buffers rather than allocating them
+yourself. This will enable `--max-buffer-memory` and `--use-mmap` and will make
+rclone's memory usage smaller.
+
+- If the provider supports multipart or parallel chunk uploads, implement
+  [fs.OpenChunkWriter](https://pkg.go.dev/github.com/rclone/rclone/fs#OpenChunkWriter)
+  and drive it with
+  [multipart.UploadMultipart](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#UploadMultipart).
+  This gives you pooled chunk buffers, concurrency, retries and
+  `--multi-thread-streams` support for free. See s3/b2 backends for examples.
+- If you must buffer data yourself take the buffer from the global pool with
+  [multipart.NewRW](https://pkg.go.dev/github.com/rclone/rclone/lib/multipart#NewRW)
+  and fill it with `io.CopyN`. The `pool.RW` is seekable so the same buffer can
+  be rewound for a retry.
+- When using a pooled buffer `Close()` it on all paths so its pages go back to
+  the pool, `Seek(0, io.SeekStart)` inside the retry closure before each
+  attempt, set the request's `ContentLength` explicitly, and if the buffer
+  reaches `http.NewRequest` or an SDK without going through `lib/rest`, wrap it
+  in [readers.NoCloser](https://pkg.go.dev/github.com/rclone/rclone/lib/readers#NoCloser)
+  so the transport can't close it and free its pages before the retry.
+
 ### Unit tests
 
 - Create a config entry called `TestRemote` for the unit tests to use
 - Create a `backend/remote/remote_test.go` - copy and adjust your example remote
+- If your backend does chunked uploads, set `ChunkedUpload` in the
+  `fstests.Opt` and implement `SetUploadChunkSize` (and `SetUploadCutoff`)
+  on your `Fs` in the test file so the `FsPutChunked`, `FsPutShortEOF` and
+  `FsPutRetry` tests can run the chunked path with small chunks.
 - Make sure all tests pass with `go test -v`
 
 ### Integration tests

Dockerfile

@@ -35,17 +35,21 @@ RUN echo "**** Print Version Binary ****" && \
 # Begin final image
 FROM alpine:latest
 
+ARG RCLONE_UID=1009
+ARG RCLONE_GID=1009
+
 RUN echo "**** Install Dependencies ****" && \
     apk add --no-cache \
         ca-certificates \
         fuse3 \
+        mailcap \
         tzdata && \
     echo "Enable user_allow_other in fuse" && \
     echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 
-RUN addgroup -g 1009 rclone && adduser -u 1009 -Ds /bin/sh -G rclone rclone
+RUN addgroup -g "${RCLONE_GID}" rclone && adduser -u "${RCLONE_UID}" -Ds /bin/sh -G rclone rclone
 
 ENTRYPOINT [ "rclone" ]
 

Makefile

@@ -38,6 +38,9 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := beta.rclone.org:
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
+# Update the beta site's directory listings after an upload - add
+# --changed for the uploaded directories
+BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
@@ -93,11 +96,15 @@ test:	rclone test_all
 	@echo "Written logs in test_all.log"
 
 # Quick test
+#
+# The timeout is raised above the go test default of 10m as the
+# cmd/gitannex end to end tests can take longer than that on slow CI
+# runners.
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -timeout 20m ./...
 
 racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race -timeout 20m ./...
 
 compiletest:
 	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
@@ -230,6 +237,7 @@ check_sign:
 upload:
 	rclone -P copy build/ downloads.rclone.org:/$(TAG)
 	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
+	rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -262,6 +270,7 @@ ci_upload:
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
 	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
+	./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 ci_beta:
@@ -271,6 +280,7 @@ ci_beta:
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
 	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
+	rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
 	@echo Beta release ready at $(BETA_URL)
 
 # Fetch the binary builds from GitHub actions
@@ -292,21 +302,26 @@ retag:
 	@echo "Version is $(VERSION)"
 	git tag -f -s -m "Version $(VERSION)" $(VERSION)
 
+# Start development of the next version after a release: the next
+# minor version on master or the next patch version on a stable branch.
 startdev:
-	@echo "Version is $(VERSION)"
-	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
-	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
+	@branch=$$(git rev-parse --abbrev-ref HEAD); \
+	case "$$branch" in \
+	master) next="$(NEXT_VERSION)" ;; \
+	v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
+	*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
+	esac; \
+	if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
+		echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
+	fi; \
+	echo "Version is $(VERSION)"; \
+	echo "Next version is $$next"; \
+	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
+	echo -n "$$next" > docs/layouts/partials/version.html; \
+	echo "$$next" > VERSION; \
+	git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
 
-startstable:
-	@echo "Version is $(VERSION)"
-	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
-	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
+startstable: startdev
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -39,6 +39,7 @@ directories to and from different cloud storage providers.
 - Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
 - DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
 - Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
+- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
 - Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
 - Drime [:page_facing_up:](https://rclone.org/s3/#drime)
 - Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
@@ -110,6 +111,7 @@ directories to and from different cloud storage providers.
 - Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
 - Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
 - RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
+- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
 - rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
 - Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
 - Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)

RELEASE.md

@@ -6,27 +6,37 @@ This file describes how to make the various kinds of releases
 
 - [gh the github cli](https://github.com/cli/cli) for uploading packages
 - pandoc for making the html and man pages
+- hugo for building the website
+- golangci-lint and docker for `make check` (markdownlint runs in docker)
+- a gpg key for signing the tag and the checksums
+- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
+  `www.rclone.org:` and `test-rclone-org:`
+- the sponsor logos - see [Sponsor logos](#sponsor-logos)
 
 ## Making a release
 
+- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
 - git checkout master # see below for stable branch
 - git pull # IMPORTANT
 - make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
 - git status - make sure everything is checked in
 - Check GitHub actions build for master is Green
-- make test # see integration test server or run locally
-- make tag
-- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
-  releases
+- Check the integration tests at <https://integration.rclone.org>
+- cat VERSION # check this is the version being released
+- make tag # signed tag on HEAD + new changelog entry + docs
+- edit docs/content/changelog.md # entries already in point releases are
+  removed automatically
 - make tidy
-- make doc
-- git status - to check for new man pages - git add them
+- make doc # again to pick up the changelog edits
+- git status - to check for new command docs in docs/content/commands - git add them
 - git commit -a -v -m "Version v1.XX.0"
-- make check
-- make retag
+- make check # if this fails, fix, git commit --amend and run it again
+- make retag # moves the tag onto the Version commit
 - git push origin # without --follow-tags so it doesn't push the tag if it fails
 - git push --follow-tags origin
 - \# Wait for the GitHub builds to complete then...
+- \# Don't commit anything until after make upload_github as these steps
+  use the tag on HEAD to find the version
 - make fetch_binaries
 - make tarball
 - make vendorball
@@ -35,9 +45,17 @@ This file describes how to make the various kinds of releases
 - make upload
 - make upload_test_website # check sponsors are correct/unchanged
 - make upload_website
-- make upload_github
-- make startdev # make startstable for stable branch
-- \# announce with forum post, twitter post, and post to annoucements list
+- make upload_github # this triggers the docker plugin and winget workflows
+- Check the docker image, docker plugin and winget GitHub actions succeeded
+- make startdev # works out the next version for master or a stable branch
+- git push
+- Disable the release freeze rule
+- Release any security advisories
+  - request a CVE for each advisory if not done already
+  - set the patched version on each advisory and publish it
+  - when the CVEs are assigned replace `CVE-PENDING` in the changelog on
+    master and the stable branch then `make upload_website`
+- \# announce with forum post, twitter post, and post to announcements list
 
 ## Update dependencies
 
@@ -190,20 +208,53 @@ First make the release branch.  If this is a second point release then
 this will be done already.
 
 - git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
-- make startstable
+- make startdev # sets the version to ${BASE_TAG}.1
+- git push -u origin ${BASE_TAG}-stable
 
 Now
 
-- git co ${BASE_TAG}-stable
-- `git cherry-pick -x` any fixes
-- make startstable
-- Do the steps as above
+- Enable release freeze rule
 - git co master
-- `#` cherry pick the changes to the changelog - check the diff to make sure it
-  is correct
-- git checkout ${BASE_TAG}-stable docs/content/changelog.md
-- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
+- git pull --rebase
+- make fetch-gui-and-commit # update the GUI on master first
 - git push
+- git co ${BASE_TAG}-stable
+- git pull
+- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
+- `git cherry-pick -x` any fixes, including the GUI update if there was one
+  - for security releases consider updating the Go patch version in
+    `.github/workflows/build.yml` and dependencies with CVE fixes
+  - fixes may depend on commits which weren't picked so push to CI
+    and check it is green before tagging
+- Do the steps in "Making a release" above, except
+  - stay on ${BASE_TAG}-stable - don't checkout master or run
+    make fetch-gui-and-commit
+  - check the GitHub actions build for ${BASE_TAG}-stable not master
+  - run the integration tests on ${BASE_TAG}-stable as described below
+  - the release freeze rule is already enabled - leave it until the end
+- git co master
+- git pull --rebase
+- git checkout ${BASE_TAG}-stable docs/content/changelog.md
+- git diff --cached # check only the ${NEW_TAG} entry has changed
+- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
+- git push
+- Disable release freeze rule
+
+Point releases are only made for the latest release. Once v1.(XX+1).0
+is out the ${BASE_TAG}-stable branch is abandoned.
+
+### Running the integration tests on the stable branch
+
+The integration tests at <https://integration.rclone.org> test master.
+To test the stable branch push it to the testing server and run the
+tests there in `screen` as they take a while.
+
+```console
+git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
+ssh rclone@rclone-testing
+screen
+cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
+```
 
 ## Sponsor logos
 

VERSION

@@ -1 +1 @@
-v1.75.0
+v1.76.0

backend/all/all.go

@@ -16,6 +16,7 @@ import (
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
 	_ "github.com/rclone/rclone/backend/doi"
+	_ "github.com/rclone/rclone/backend/dosya"
 	_ "github.com/rclone/rclone/backend/drime"
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/dropbox"

backend/archive/archive.go

@@ -36,6 +36,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/sanitize"
 )
 
 // Register with Fs
@@ -185,8 +186,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 	foundArchive := subArchive(remote)
 	if foundArchive != nil {
 		fs.Debugf(nil, "Found archiver for %q remote %q", foundArchive.archiver.Extension, foundArchive.remote)
-		// Archive path
-		foundArchive.root = strings.Trim(remote[len(foundArchive.remote):], "/")
+		// Archive path, in canonical form so that it compares equal
+		// to the cleaned entry names inside the archive
+		foundArchive.root = strings.Trim(path.Clean(remote[len(foundArchive.remote):]), "/")
+		if foundArchive.root == "." {
+			foundArchive.root = ""
+		}
 		// Path to the archive
 		archiveRemote := remote[:len(foundArchive.remote)]
 		// Remote is archive leaf name
@@ -491,24 +496,50 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return nil, err
 	}
 
-	entries, err = subFs.List(ctx, dir)
+	subEntries, err := subFs.List(ctx, dir)
 	if err != nil {
 		return nil, err
 	}
-	for i, entry := range entries {
+	entries = subEntries[:0]
+	skipped := 0
+	for _, entry := range subEntries {
+		remote := entry.Remote()
+		// Only pass on direct children of dir - anything else
+		// could escape the archive's namespace
+		if !isDirectChild(dir, remote) {
+			fs.Debugf(f, "Skipping entry %q which is not in directory %q", remote, dir)
+			skipped++
+			continue
+		}
 		// Can only unarchive files
 		if o, ok := entry.(fs.Object); ok {
-			remote := o.Remote()
 			archive := f.findArchive(remote)
 			if archive != nil {
 				// Overwrite entry with directory
-				entries[i] = fs.NewDir(remote, o.ModTime(ctx))
+				entry = fs.NewDir(remote, o.ModTime(ctx))
 			}
 		}
+		entries = append(entries, entry)
+	}
+	if skipped > 0 {
+		fs.Logf(f, "Skipped %d entries in %q whose names escape the directory", skipped, dir)
 	}
 	return entries, nil
 }
 
+// isDirectChild reports whether remote names an entry directly in dir
+// ("" being the root), with no ".." or other components in between.
+func isDirectChild(dir, remote string) bool {
+	if dir != "" {
+		var ok bool
+		remote, ok = strings.CutPrefix(remote, dir+"/")
+		if !ok {
+			return false
+		}
+	}
+	return sanitize.Leaf(remote) == nil
+}
+
 // NewObject creates a new remote archive file object
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
@@ -526,6 +557,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
+	// Don't trust the archiver to have returned the object asked for
+	if o.Remote() != remote {
+		fs.Debugf(f, "Ignoring object %q returned for %q", o.Remote(), remote)
+		return nil, fs.ErrorObjectNotFound
+	}
 	return o, nil
 }
 

backend/archive/archive_internal_test.go

@@ -3,6 +3,7 @@
 package archive
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"fmt"
@@ -11,15 +12,18 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
 
+	"github.com/rclone/rclone/backend/archive/archiver"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
@@ -173,6 +177,18 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
 	checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
 
 	// Now check a single file
+	//
+	// The filter below applies to both sides, so check first that the
+	// archive exposes only that file without it
+	Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
+	require.NoError(t, err)
+	entries, err := Fsingle.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	_, err = Fsingle.List(ctx, subDir)
+	assert.Equal(t, fs.ErrorDirNotFound, err)
+	_, err = Fsingle.NewObject(ctx, subDir)
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
 	fiCtx, fi := filter.AddConfig(ctx)
 	require.NoError(t, fi.AddRule("+ "+aFile))
 	require.NoError(t, fi.AddRule("- *"))
@@ -277,3 +293,227 @@ func TestArchiveSquashfsIssue9004(t *testing.T) {
 		assert.True(t, bytes.HasPrefix(data, []byte("<?xml")))
 	})
 }
+
+// TestArchiveUncleanRoot checks that a path into an archive which isn't
+// in canonical form (with "./" or doubled slashes) still finds its
+// directory.
+func TestArchiveUncleanRoot(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create("sub/dir/a.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	zipPath := filepath.Join(t.TempDir(), "test.zip")
+	require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
+
+	for _, root := range []string{"sub/dir", "sub/./dir", "sub//dir", "./sub/dir/", "sub/dir/."} {
+		t.Run(root, func(t *testing.T) {
+			f, err := cache.Get(ctx, ":archive:"+zipPath+"/"+root)
+			require.NoError(t, err)
+			entries, err := f.List(ctx, "")
+			require.NoError(t, err)
+			require.Len(t, entries, 1)
+			assert.Equal(t, "a.txt", entries[0].Remote())
+		})
+	}
+}
+
+// escapingObject is an object whose remote is not where it was asked for.
+type escapingObject struct {
+	fs.Object
+	remote string
+}
+
+func (o *escapingObject) Remote() string { return o.remote }
+func (o *escapingObject) String() string { return o.remote }
+
+// escapingFs stands in for a badly behaved archiver which exposes entry
+// names outside the directory being listed.
+type escapingFs struct {
+	fs.Fs
+	prefix string
+}
+
+func (f *escapingFs) Name() string           { return "escaping" }
+func (f *escapingFs) Root() string           { return "" }
+func (f *escapingFs) String() string         { return "escaping" }
+func (f *escapingFs) Features() *fs.Features { return &fs.Features{} }
+
+func (f *escapingFs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
+	return fs.DirEntries{
+		&escapingObject{remote: path.Join(dir, "good.txt")},
+		fs.NewDir(path.Join(dir, "gooddir"), fstest.Time("2001-02-03T04:05:06.499999999Z")),
+		&escapingObject{remote: path.Join(dir, "../escape.txt")},
+		&escapingObject{remote: "../../escape.txt"},
+		&escapingObject{remote: path.Join(dir, "sub/notachild.txt")},
+		fs.NewDir("../escapedir", fstest.Time("2001-02-03T04:05:06.499999999Z")),
+	}, nil
+}
+
+func (f *escapingFs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	return &escapingObject{remote: "../escape.txt"}, nil
+}
+
+// TestArchiveEscapingArchiver checks that the archive backend does not
+// pass on entries from an archiver which escape the directory being
+// listed, whatever the archiver does.
+func TestArchiveEscapingArchiver(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	archiver.Register(archiver.Archiver{
+		New: func(ctx context.Context, f fs.Fs, remote, prefix, root string) (fs.Fs, error) {
+			return &escapingFs{prefix: prefix}, nil
+		},
+		Extension: ".escaping",
+	})
+
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test.escaping"), []byte("x"), 0600))
+	f, err := cache.Get(ctx, ":archive:"+dir)
+	require.NoError(t, err)
+
+	// Archives are discovered when their parent directory is listed
+	_, err = f.List(ctx, "")
+	require.NoError(t, err)
+
+	entries, err := f.List(ctx, "test.escaping")
+	require.NoError(t, err)
+	var remotes []string
+	for _, entry := range entries {
+		remotes = append(remotes, entry.Remote())
+	}
+	assert.ElementsMatch(t, []string{"test.escaping/good.txt", "test.escaping/gooddir"}, remotes)
+
+	_, err = f.NewObject(ctx, "test.escaping/file.txt")
+	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}
+
+// TestIsDirectChild checks the guard which decides whether an entry
+// returned by an archiver belongs directly in the directory listed.
+func TestIsDirectChild(t *testing.T) {
+	for _, test := range []struct {
+		dir, remote string
+		want        bool
+	}{
+		{"", "a.txt", true},
+		{"", "/a.txt", false},
+		{"", "a.txt/", false},
+		{"", "../a.txt", false},
+		{"", "sub/a.txt", false},
+		{"d", "d/a.txt", true},
+		{"d", "d", false},
+		{"d", "d/", false},
+		{"d", "d//a.txt", false},
+		{"d", "d/../a.txt", false},
+		{"d", "dd/a.txt", false},
+		{"d", "a.txt", false},
+	} {
+		assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
+	}
+}
+
+// A symlink in a squashfs image is skipped unless -l/--links is in use,
+// as it is on the other backends.  With the flag it is exposed under its
+// name with the link suffix and reads back as its target.
+//
+// Note that this uses mksquashfs as an external binary.
+func TestArchiveSquashfsSymlink(t *testing.T) {
+	fstest.Initialise()
+	skipIfNoExe(t, "mksquashfs")
+	ctx := context.Background()
+
+	dir := t.TempDir()
+	input := filepath.Join(dir, "input")
+	require.NoError(t, os.Mkdir(input, 0777))
+	require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
+	require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
+	// a regular file which happens to carry the link suffix must stay
+	// readable under the flag, not be taken for a symlink
+	require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
+	image := filepath.Join(dir, "test.sqfs")
+	run(t, "mksquashfs", input, image)
+
+	remotes := func(t *testing.T, f fs.Fs) []string {
+		entries, err := f.List(ctx, "")
+		require.NoError(t, err)
+		names := make([]string, 0, len(entries))
+		for _, entry := range entries {
+			names = append(names, entry.Remote())
+		}
+		sort.Strings(names)
+		return names
+	}
+
+	read := func(t *testing.T, f fs.Fs, remote string) string {
+		o, err := f.NewObject(ctx, remote)
+		require.NoError(t, err)
+		rc, err := o.Open(ctx)
+		require.NoError(t, err)
+		contents, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		require.NoError(t, rc.Close())
+		assert.Equal(t, o.Size(), int64(len(contents)))
+		return string(contents)
+	}
+
+	f, err := fs.NewFs(ctx, ":archive:"+image)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
+
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = true
+
+	f, err = fs.NewFs(ctx, ":archive:"+image)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
+
+	assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
+	assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
+}
+
+// A hostile archive must not be able to write outside the destination.
+// A symlink pointing above the root with a member inside it is the
+// classic way to try it, and the refusal comes from the local backend,
+// so pin it from end to end here rather than trusting it stays.
+func TestArchiveZipSymlinkEscape(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = true
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	hdr := &zip.FileHeader{Name: "a"}
+	hdr.SetMode(os.ModeSymlink | 0777)
+	w, err := zw.CreateHeader(hdr)
+	require.NoError(t, err)
+	_, err = w.Write([]byte("../outside"))
+	require.NoError(t, err)
+	w, err = zw.Create("a/pwned.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("pwned"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+
+	dir := t.TempDir()
+	zipPath := filepath.Join(dir, "evil.zip")
+	require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
+	dst := filepath.Join(dir, "dst")
+	require.NoError(t, os.Mkdir(dst, 0777))
+
+	fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
+	require.NoError(t, err)
+	fdst, err := fs.NewFs(ctx, dst)
+	require.NoError(t, err)
+
+	assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
+
+	_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
+	assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
+}

backend/archive/archiver/archiver.go

@@ -8,6 +8,11 @@ import (
 )
 
 // Archiver describes an archive package
+//
+// Entry names inside an archive are attacker controlled. An archiver
+// must not expose an entry whose remote escapes the archive's own
+// namespace (for example one with a ".." component), typically by
+// validating names with lib/sanitize.
 type Archiver struct {
 	// New constructs an Fs from the (wrappedFs, remote) with the objects
 	// prefix with prefix and rooted at root

backend/archive/squashfs/squashfs.go

@@ -16,6 +16,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/sanitize"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -41,6 +42,8 @@ type Fs struct {
 	prefix      string   // position for objects
 	prefixSlash string   // position for objects with a slash on
 	root        string   // position to read from within the archive
+	file        string   // remote of the single file root points at, if any
+	links       bool     // read symlinks as .rclonelink files
 }
 
 // recoverParsePanic converts a panic raised while parsing a squashfs image
@@ -97,6 +100,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
 		c:           c,
 		remote:      remote,
 		root:        strings.Trim(root, "/"),
+		links:       fs.GetConfig(ctx).Links,
 		prefix:      prefix,
 		prefixSlash: prefix + "/",
 	}
@@ -114,10 +118,13 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
 			_, err := f.newObjectNative(native)
 			if err == nil {
 				// If it pointed to a file, find the directory above
+				// and remember the file so that only it is listed
 				f.root = path.Dir(f.root)
 				if f.root == "." || f.root == "/" {
 					f.root = ""
 				}
+				f.file = f.prefixSlash + path.Base(native)
+				singleObject = true
 			}
 		}
 	}
@@ -149,7 +156,9 @@ func (f *Fs) Name() string {
 
 // Root of the remote (as passed into NewFs)
 func (f *Fs) Root() string {
-	return f.root
+	// Include the file so the fs cache can tell this Fs apart from
+	// the one for the directory containing it
+	return path.Join(f.root, f.file)
 }
 
 // Features returns the optional features of this Fs
@@ -213,6 +222,30 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
 	}
 }
 
+// Convert a symlink entry into an Object holding its target
+func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
+	return &Object{
+		fs:      f,
+		remote:  f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
+		size:    int64(len(target)),
+		modTime: item.ModTime(),
+		isLink:  true,
+		link:    target,
+	}
+}
+
+// readlink returns the target of the symlink the info describes.
+//
+// A symlink may carry an empty target, so the caller decides what is a
+// symlink from the mode rather than from the target being set.
+func readlink(info os.FileInfo) (string, error) {
+	st, ok := info.Sys().(*squashfs.StatT)
+	if !ok {
+		return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
+	}
+	return st.LinkTarget, nil
+}
+
 // List the objects and directories in dir into entries.  The
 // entries can be returned in any order but should be for a
 // complete directory.
@@ -226,6 +259,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
 	defer recoverParsePanic(&err)
 
+	if f.file != "" && dir != "" {
+		return nil, fs.ErrorDirNotFound
+	}
+
 	nativeDir, err := f.toNative(dir)
 	if err != nil {
 		return nil, err
@@ -237,8 +274,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	entries = make(fs.DirEntries, 0, len(items))
+	skipped := 0
+	skippedLinks := 0
 	for _, item := range items {
 		// fs.Debugf(item.Name(), "entry = %#v", item)
+		// Skip entry names that aren't safe
+		if err := sanitize.Leaf(item.Name()); err != nil {
+			fs.Debugf(f, "Skipping squashfs entry %q which escapes the archive", item.Name())
+			skipped++
+			continue
+		}
 		var entry fs.DirEntry
 		if item.IsDir() {
 			remote := f.fromNative(nativeDir, item.Name())
@@ -253,13 +298,39 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
 			}
 			entry = f.objectFromFileInfo(nativeDir, info)
+		} else if item.Type()&os.ModeSymlink != 0 {
+			if !f.links {
+				skippedLinks++
+				continue
+			}
+			info, err := item.Info()
+			if err != nil {
+				return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
+			}
+			target, err := readlink(info)
+			if err != nil {
+				return nil, err
+			}
+			entry = f.objectFromLink(nativeDir, info, target)
 		} else {
 			fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
 			continue
 		}
+		if f.file != "" && entry.Remote() != f.file {
+			continue
+		}
 		entries = append(entries, entry)
 	}
 
+	// Log once per listing rather than per entry so a crafted image
+	// can't flood the log
+	if skipped > 0 {
+		fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
+	}
+	if skippedLinks > 0 {
+		fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
+	}
+
 	// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
 	return entries, nil
 }
@@ -280,7 +351,17 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 	}
 
 	for _, fi := range fis {
-		if fi.Name() == leaf {
+		// Name the entry as List does, so that the two agree on
+		// which names exist
+		isLink := fi.Type()&os.ModeSymlink != 0
+		if isLink && !f.links {
+			continue
+		}
+		name := fi.Name()
+		if isLink {
+			name += fs.LinkSuffix
+		}
+		if name == leaf {
 			if fi.IsDir() {
 				return nil, fs.ErrorNotAFile
 			}
@@ -288,6 +369,14 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 			if err != nil {
 				return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
 			}
+			if isLink {
+				target, err := readlink(info)
+				if err != nil {
+					return nil, err
+				}
+				o = f.objectFromLink(dir, info, target)
+				break
+			}
 			o = f.objectFromFileInfo(dir, info)
 			break
 		}
@@ -302,6 +391,9 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
 
+	if f.file != "" && remote != f.file {
+		return nil, fs.ErrorObjectNotFound
+	}
 	nativePath, err := f.toNative(remote)
 	if err != nil {
 		return nil, err
@@ -363,6 +455,8 @@ type Object struct {
 	remote  string
 	size    int64
 	modTime time.Time
+	isLink  bool   // whether this is a symlink
+	link    string // target of the symlink
 }
 
 // Fs returns read only access to the Fs that this object is part of
@@ -434,6 +528,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 		}
 	}
 
+	if o.isLink {
+		// Clamp offset into range to avoid panic
+		if offset < 0 {
+			offset = 0
+		}
+		if offset > int64(len(o.link)) {
+			offset = int64(len(o.link))
+		}
+		return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
+	}
+
 	remote, err := o.fs.toNative(o.remote)
 	if err != nil {
 		return nil, err

backend/archive/squashfs/squashfs_internal_test.go

@@ -0,0 +1,32 @@
+package squashfs
+
+import (
+	"context"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	fscache "github.com/rclone/rclone/fs/cache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// A backslash is an ordinary character in a file name on the systems
+// squashfs images are made on, so an entry containing one must be
+// listed and openable like any other file.
+func TestBackslashInName(t *testing.T) {
+	ctx := context.Background()
+	localFs, err := fscache.Get(ctx, "testdata")
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, "backslash.sqfs", "", "")
+	require.NoError(t, err)
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	assert.Equal(t, `back\slash.txt`, entries[0].Remote())
+
+	o, err := f.NewObject(ctx, `back\slash.txt`)
+	require.NoError(t, err)
+	assert.Equal(t, int64(6), o.Size())
+}

backend/archive/zip/zip.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"os"
 	"path"
+	"slices"
 	"strings"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/sanitize"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfscommon"
 )
@@ -41,6 +43,7 @@ type Fs struct {
 	prefix      string          // position for objects
 	prefixSlash string          // position for objects with a slash on
 	root        string          // position to read from within the archive
+	links       bool            // read symlinks as .rclonelink files
 	dt          dirtree.DirTree // read from zipfile
 }
 
@@ -65,6 +68,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
 		node:        node,
 		remote:      remote,
 		root:        root,
+		links:       fs.GetConfig(ctx).Links,
 		prefix:      prefix,
 		prefixSlash: prefix + "/",
 	}
@@ -135,15 +139,37 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 		return singleObject, fmt.Errorf("failed to read zip file: %w", err)
 	}
 	dt := dirtree.New()
+	skipped := 0
+	skippedLinks := 0
 	for _, file := range zr.File {
-		remote := strings.Trim(path.Clean(file.Name), "/")
-		if remote == "." {
-			remote = ""
+		// Skip entries whose name escapes the archive's own namespace
+		remote, err := sanitize.Path(file.Name)
+		if err != nil {
+			skipped++
+			continue
+		}
+		isDir := strings.HasSuffix(file.Name, "/")
+		// A file entry whose last component is "", "." or ".." names
+		// a directory rather than a file, so skip
+		if !isDir && sanitize.Leaf(path.Base(file.Name)) != nil {
+			skipped++
+			continue
+		}
+		if !isDir && file.Mode()&os.ModeSymlink != 0 {
+			if !f.links {
+				skippedLinks++
+				continue
+			}
+			// The member holds the link target, so it reads back
+			// like a .rclonelink file
+			remote += fs.LinkSuffix
 		}
 		remote = path.Join(f.prefix, remote)
 		if f.root != "" {
-			// Ignore all files outside the root
-			if !strings.HasPrefix(remote, f.root) {
+			// Ignore all files outside the root, requiring a path
+			// boundary so that root "foo" does not also match a
+			// sibling entry such as "foobar"
+			if remote != f.root && !strings.HasPrefix(remote, f.root+"/") {
 				continue
 			}
 			if remote == f.root {
@@ -152,11 +178,12 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 				remote = strings.TrimPrefix(remote, f.root+"/")
 			}
 		}
-		if strings.HasSuffix(file.Name, "/") {
+		if isDir {
 			dir := fs.NewDir(remote, file.Modified)
 			dt.AddDir(dir)
 		} else {
 			if remote == "" {
+				// A file at the root is the archive member f.root points at
 				remote = path.Base(f.root)
 				singleObject = true
 				dt = dirtree.New()
@@ -173,6 +200,12 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 			}
 		}
 	}
+	if skipped > 0 {
+		fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
+	}
+	if skippedLinks > 0 {
+		fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
+	}
 	dt.CheckParents("")
 	dt.Sort()
 	f.dt = dt
@@ -200,7 +233,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return nil, fs.ErrorDirNotFound
 	}
 	fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
-	return entries, nil
+	// Return a copy as callers may filter the listing in place
+	return slices.Clone(entries), nil
 }
 
 // NewObject finds the Object at remote.

backend/archive/zip/zip_internal_test.go

@@ -0,0 +1,227 @@
+package zip
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/local"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writeZip builds a zip file with the given entry names (a name ending
+// in "/" is a directory) in dir and returns its leaf name.
+func writeZip(t *testing.T, dir, name string, names ...string) string {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	for _, n := range names {
+		w, err := zw.Create(n)
+		require.NoError(t, err)
+		if !strings.HasSuffix(n, "/") {
+			_, err = w.Write([]byte("data for " + n))
+			require.NoError(t, err)
+		}
+	}
+	require.NoError(t, zw.Close())
+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
+	return name
+}
+
+// writeZipSymlink builds a zip holding file and a symlink entry link
+// pointing at target, and returns its leaf name.
+func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create(file)
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data for " + file))
+	require.NoError(t, err)
+	hdr := &zip.FileHeader{Name: link}
+	hdr.SetMode(os.ModeSymlink | 0777)
+	w, err = zw.CreateHeader(hdr)
+	require.NoError(t, err)
+	_, err = w.Write([]byte(target))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
+	return name
+}
+
+// allRemotes returns every Object remote in the mounted archive's dirtree.
+func allRemotes(t *testing.T, f fs.Fs) []string {
+	zf, ok := f.(*Fs)
+	require.True(t, ok)
+	var remotes []string
+	for _, entries := range zf.dt {
+		for _, entry := range entries {
+			if o, ok := entry.(*Object); ok {
+				remotes = append(remotes, o.Remote())
+			}
+		}
+	}
+	// dt iteration order is nondeterministic, so sort for stable comparison.
+	sort.Strings(remotes)
+	return remotes
+}
+
+// A malicious zip whose entry names escape the archive's own namespace
+// must not be exposed by the zip backend (Zip Slip).
+func TestReadZipSlip(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "evil.zip",
+		"good.txt",
+		"../../etc/cron.d/evil",
+		"../escape.txt",
+		"sub/../../above.txt",
+		`..\..\evil.exe`,
+	)
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+
+	remotes := allRemotes(t, f)
+
+	// The one benign entry must survive.
+	assert.Contains(t, remotes, "good.txt")
+
+	// No escaping entry may be exposed.
+	for _, remote := range remotes {
+		assert.False(t, remote == ".." || strings.HasPrefix(remote, "../"),
+			"escaping remote exposed: %q", remote)
+	}
+	assert.Equal(t, []string{"good.txt"}, remotes)
+}
+
+// Mounting with a non-empty root must only expose entries within that
+// root directory, not sibling directories that merely share a name
+// prefix (root "foo" must not match "foobar").
+func TestReadZipRootBoundary(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "test.zip",
+		"foo/a.txt",
+		"foobar/b.txt",
+	)
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, name, "", "foo")
+	require.NoError(t, err)
+
+	remotes := allRemotes(t, f)
+	assert.Equal(t, []string{"a.txt"}, remotes)
+}
+
+// A file entry whose name refers to a directory (".", "/", "./", "" or
+// "sub/.") must be skipped, not turn the archive or the directory
+// mounted as its root into a single file which hides every other
+// entry, or become an object named for the archive itself when the
+// archive is found by listing its directory.
+func TestReadZipDirNamedFileEntry(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "dot.zip",
+		".",
+		"/",
+		"./",
+		"",
+		"good.txt",
+		"sub/",
+		"sub/.",
+		"sub/good.txt",
+	)
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	for _, test := range []struct {
+		prefix, root string
+		want         []string
+	}{
+		{"", "", []string{"good.txt", "sub/good.txt"}},
+		{"sub/dot.zip", "", []string{"sub/dot.zip/good.txt", "sub/dot.zip/sub/good.txt"}},
+		{"", "sub", []string{"good.txt"}},
+	} {
+		t.Run(fmt.Sprintf("prefix=%q,root=%q", test.prefix, test.root), func(t *testing.T) {
+			f, err := New(ctx, localFs, name, test.prefix, test.root)
+			require.NoError(t, err)
+			assert.Equal(t, test.want, allRemotes(t, f))
+		})
+	}
+}
+
+// Listings are served from a cache which must survive callers
+// filtering the returned slice in place, as fs/list does.
+func TestListCacheNotAliased(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZip(t, dir, "test.zip", "a.txt", "b.txt", "c.txt")
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+	f, err := New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 3)
+	// Compact in place, dropping the first entry
+	copy(entries, entries[1:])
+
+	entries, err = f.List(ctx, "")
+	require.NoError(t, err)
+	var remotes []string
+	for _, entry := range entries {
+		remotes = append(remotes, entry.Remote())
+	}
+	assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
+}
+
+// A symlink stored in the archive is skipped unless -l/--links is in
+// use, as it is on the other backends.  With the flag it is exposed
+// under its name with the link suffix and reads back as its target.
+func TestReadZipSymlink(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
+
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = true
+
+	f, err = New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
+
+	o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len("file.txt")), o.Size())
+
+	rc, err := o.Open(ctx)
+	require.NoError(t, err)
+	target, err := io.ReadAll(rc)
+	require.NoError(t, err)
+	require.NoError(t, rc.Close())
+	assert.Equal(t, "file.txt", string(target))
+}

backend/azureblob/arrow.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !js
 
 // Parallel listing support for azureblob, built on the Apache Arrow
 // listing format.
@@ -18,24 +18,20 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/fs"
 	"golang.org/x/sync/errgroup"
 )
 
+// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
+// response, used to tell whether the service honoured an Arrow request.
+const arrowContentType = "application/vnd.apache.arrow.stream"
+
 // listArrowParallel lists (containerName, directory) by splitting the blob name
 // keyspace into shards and listing them concurrently. opts must already request
 // the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
 // returns the number of raw items seen. directory/prefix are already normalised
 // by the caller.
-func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
-	// Parallel listing needs the arrowlist client (server-side endBefore is
-	// only honoured on the Arrow path) - without it fall back to sequential.
-	if _, err := f.arrowCntSVC(containerName); err != nil {
-		fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
-		return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
-	}
-
+func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
 	// Split the keyspace by a single-case character ladder (see
 	// arrowLadderBoundaries for why single-case matters).
 	points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
@@ -66,7 +62,7 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
 
 	// Schedule every shard; SetLimit bounds concurrency, so more shards than
 	// workers gives free load balancing as workers pick up the next shard.
-	var total int64
+	var total atomic.Int64
 	g, gCtx := errgroup.WithContext(ctx)
 	g.SetLimit(f.opt.ListParallelism)
 	for _, sh := range shards {
@@ -89,12 +85,12 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
 				shardOpts.EndBefore = nil
 			}
 			n, err := f.listBlobsPager(gCtx, containerName, directory, prefix, addContainer, &shardOpts, delimiter, safeFn)
-			atomic.AddInt64(&total, int64(n))
+			total.Add(int64(n))
 			return err
 		})
 	}
 	err := g.Wait()
-	found := int(atomic.LoadInt64(&total))
+	found := int(total.Load())
 	if isEndBeforeUnsupported(err) {
 		// The account can't honour the shards' endBefore bounds, so parallel
 		// listing can't be used on it.
@@ -115,16 +111,13 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
 // isEndBeforeUnsupported reports whether err means the service can't honour
 // the endBefore listing bound, so sharded parallel listing must not be used.
 //
-// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
-// server answered XML with endBefore set), or the service rejecting the
-// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
-// requested operation is not allowed as EndBefore parameter support is
-// missing on this account" - endBefore is only supported on the Arrow
-// listing path of accounts with Blob Listing with Apache Arrow enabled.
+// The service rejects the endBefore parameter with 400
+// OperationNotSupportedWithFeatureMissing "The requested operation is not
+// allowed as EndBefore parameter support is missing on this account" -
+// endBefore is only supported on the Arrow listing path of accounts with
+// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
+// accept an XML page with endBefore set, but the service never sends one.)
 func isEndBeforeUnsupported(err error) bool {
-	if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
-		return true
-	}
 	var respErr *azcore.ResponseError
 	return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
 }

backend/azureblob/arrow_internal_test.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !js
 
 package azureblob
 
@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -212,9 +211,8 @@ func TestArrowLadderBoundaries(t *testing.T) {
 }
 
 func TestIsEndBeforeUnsupported(t *testing.T) {
-	assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
-	assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
 	assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
+	assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
 	assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
 	assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
 	assert.False(t, isEndBeforeUnsupported(nil))

backend/azureblob/arrowlist/arrow.go

@@ -1,421 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
-// public type aliases so it compiles against the released azblob module.
-
-package arrowlist
-
-import (
-	"encoding/base64"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
-	"github.com/apache/arrow-go/v18/arrow"
-	"github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/apache/arrow-go/v18/arrow/ipc"
-	"github.com/apache/arrow-go/v18/arrow/memory"
-)
-
-const (
-	// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
-	ArrowContentType = "application/vnd.apache.arrow.stream"
-
-	// ArrowAcceptHeader is the Accept header value to request Arrow format.
-	ArrowAcceptHeader = ArrowContentType
-
-	// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
-	// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
-	// in Arrow, all rows share the same schema and are distinguished by this field value.
-	resourceTypeBlobPrefix = "blobprefix"
-)
-
-// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
-func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
-	result := container.ListBlobsFlatResponse{}
-
-	// Extract response headers
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
-	// Parse Arrow IPC stream
-	items, nextMarker, err := parseArrowStream(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
-	}
-
-	result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
-		Segment: &container.BlobFlatListSegment{
-			BlobItems: items,
-		},
-		NextMarker: nextMarker,
-	}
-
-	return result, nil
-}
-
-// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
-func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
-	result := container.ListBlobsHierarchyResponse{}
-
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
-	items, nextMarker, err := parseArrowStream(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
-	}
-
-	// Separate blob items from blob prefixes based on ResourceType field.
-	var blobItems []*container.BlobItem
-	var blobPrefixes []*container.BlobPrefix
-	for _, item := range items {
-		if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
-			blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
-				Name:       item.Name,
-				Properties: item.Properties,
-			})
-		} else {
-			blobItems = append(blobItems, item)
-		}
-	}
-
-	result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
-		Segment: &container.BlobHierarchyListSegment{
-			BlobItems:    blobItems,
-			BlobPrefixes: blobPrefixes,
-		},
-		NextMarker: nextMarker,
-	}
-
-	return result, nil
-}
-
-func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
-	if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
-		*clientRequestID = &val
-	}
-	if val := resp.Header.Get("Content-Type"); val != "" {
-		*contentType = &val
-	}
-	if val := resp.Header.Get("Date"); val != "" {
-		if t, err := time.Parse(time.RFC1123, val); err == nil {
-			*date = &t
-		}
-	}
-	if val := resp.Header.Get("x-ms-request-id"); val != "" {
-		*requestID = &val
-	}
-	if val := resp.Header.Get("x-ms-version"); val != "" {
-		*version = &val
-	}
-}
-
-// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
-func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
-	reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
-	}
-	defer reader.Release()
-
-	// Extract NextMarker from schema metadata
-	var nextMarker *string
-	var numRecords int
-	if md := reader.Schema().Metadata(); md.Len() > 0 {
-		if idx := md.FindKey("NextMarker"); idx >= 0 {
-			val := md.Values()[idx]
-			if val != "" {
-				nextMarker = &val
-			}
-		}
-		if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
-			if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
-				numRecords = n
-			}
-		}
-	}
-
-	// Build column index for the schema
-	schema := reader.Schema()
-	colIndex := buildColumnIndex(schema)
-
-	// Pre-allocate if we know the count
-	items := make([]*container.BlobItem, 0, numRecords)
-
-	// Iterate over record batches
-	for reader.Next() {
-		rec := reader.RecordBatch()
-		rows := int(rec.NumRows())
-
-		for row := 0; row < rows; row++ {
-			item := &container.BlobItem{
-				Properties: &container.BlobProperties{},
-			}
-			populateBlobItem(item, rec, row, colIndex)
-			items = append(items, item)
-		}
-	}
-
-	if err := reader.Err(); err != nil {
-		return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
-	}
-
-	return items, nextMarker, nil
-}
-
-// columnIndex maps column names to their indices in the record batch for O(1) lookup.
-type columnIndex map[string]int
-
-func buildColumnIndex(schema *arrow.Schema) columnIndex {
-	idx := make(columnIndex, len(schema.Fields()))
-	for i, f := range schema.Fields() {
-		idx[f.Name] = i
-	}
-	return idx
-}
-
-// populateBlobItem fills a BlobItem from a single row across all columns.
-func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
-	// Top-level BlobItem fields
-	item.Name = getStringField(rec, row, colIdx, "Name")
-	item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
-	item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
-	item.VersionID = getStringField(rec, row, colIdx, "VersionId")
-	item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
-	item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
-
-	// Metadata maps
-	item.Metadata = getMapField(rec, row, colIdx, "Metadata")
-	item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
-
-	// Tags
-	if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
-		var blobTags []*container.BlobTag
-		for k, v := range tags {
-			key := k
-			blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
-		}
-		item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
-	}
-
-	// BlobProperties fields
-	p := item.Properties
-	p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
-	p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
-	p.ETag = getETagField(rec, row, colIdx, "Etag")
-	p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
-	p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
-	p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
-	p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
-	p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
-	p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
-	p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
-	p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
-	p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
-	p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
-	p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
-	p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
-	p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
-	p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
-	p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
-	p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
-	p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
-	p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
-	p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
-	p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
-	p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
-	p.CopyID = getStringField(rec, row, colIdx, "CopyId")
-	p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
-	p.CopySource = getStringField(rec, row, colIdx, "CopySource")
-	p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
-	p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
-	p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
-	p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
-	p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
-	p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
-	p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
-	p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
-	p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
-	p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
-	p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
-	p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
-	p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
-
-	// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
-	// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
-	// It is silently ignored for forward compatibility.
-}
-
-// Field accessor helpers that safely handle missing columns and null values.
-
-func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.String); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Boolean); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Timestamp); ok {
-		val := arr.Value(row)
-		// Arrow timestamps are stored as int64 with a unit; convert to time.Time
-		dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
-		return &dt
-	}
-	return nil
-}
-
-func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Uint64); ok {
-		val := int64(arr.Value(row))
-		return &val
-	}
-	if arr, ok := col.(*array.Int64); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Uint64); ok {
-		val := int32(arr.Value(row))
-		return &val
-	}
-	if arr, ok := col.(*array.Int32); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	etag := azcore.ETag(*s)
-	return &etag
-}
-
-func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	decoded, err := base64.StdEncoding.DecodeString(*s)
-	if err != nil {
-		return nil
-	}
-	return decoded
-}
-
-// getEnumField returns a pointer to a string-typed enum value.
-type stringEnum interface {
-	~string
-}
-
-func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	val := T(*s)
-	return &val
-}
-
-func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	mapArr, ok := col.(*array.Map)
-	if !ok {
-		return nil
-	}
-
-	// Get the offsets for this row's map entries
-	start := int(mapArr.Offsets()[row])
-	end := int(mapArr.Offsets()[row+1])
-	if start == end {
-		return nil
-	}
-
-	keys := mapArr.Keys().(*array.String)
-	values := mapArr.Items().(*array.String)
-
-	result := make(map[string]*string, end-start)
-	for i := start; i < end; i++ {
-		k := keys.Value(i)
-		if values.IsNull(i) {
-			result[k] = nil
-		} else {
-			v := values.Value(i)
-			result[k] = &v
-		}
-	}
-	return result
-}

backend/azureblob/arrowlist/arrow_test.go

@@ -1,374 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost).
-
-package arrowlist
-
-import (
-	"bytes"
-	"io"
-	"net/http"
-	"testing"
-
-	"github.com/apache/arrow-go/v18/arrow"
-	arrowArray "github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/apache/arrow-go/v18/arrow/ipc"
-	"github.com/apache/arrow-go/v18/arrow/memory"
-	"github.com/stretchr/testify/require"
-)
-
-// coreSchema returns a basic Arrow schema for testing with common blob fields.
-func coreSchema(md *arrow.Metadata) *arrow.Schema {
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
-		{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
-		{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
-		{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	return arrow.NewSchema(fields, md)
-}
-
-// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
-func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
-	t.Helper()
-	alloc := memory.DefaultAllocator
-
-	var buf bytes.Buffer
-	w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
-	builder := arrowArray.NewRecordBuilder(alloc, schema)
-	defer builder.Release()
-
-	populate(builder)
-
-	rec := builder.NewRecordBatch()
-	defer rec.Release()
-
-	require.NoError(t, w.Write(rec))
-	require.NoError(t, w.Close())
-
-	return buf.Bytes()
-}
-
-func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
-	h := http.Header{}
-	h.Set("Content-Type", ArrowContentType)
-	for k, v := range headers {
-		h.Set(k, v)
-	}
-	return &http.Response{
-		StatusCode: http.StatusOK,
-		Header:     h,
-		Body:       io.NopCloser(bytes.NewReader(data)),
-	}
-}
-
-func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
-		b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
-		b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
-		b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
-		b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
-		b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
-		b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
-		b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
-		b.Field(9).(*arrowArray.StringBuilder).Append("available")
-		b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
-		b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
-		b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
-		b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
-		b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
-		b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
-		b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
-		b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
-		b.Field(9).(*arrowArray.StringBuilder).Append("available")
-		b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	require.NotNil(t, result.NextMarker)
-	require.Equal(t, "marker123", *result.NextMarker)
-	require.Len(t, result.Segment.BlobItems, 2)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "blob1.txt", *item.Name)
-	require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
-	require.Equal(t, int64(1024), *item.Properties.ContentLength)
-	require.Equal(t, "Hot", string(*item.Properties.AccessTier))
-}
-
-func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		// No rows added
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Nil(t, result.NextMarker)
-	require.Empty(t, result.Segment.BlobItems)
-}
-
-func TestHandleFlatListResponse_NullFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
-		b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
-		b.Field(3).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(4).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
-		b.Field(6).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
-		b.Field(8).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(9).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(10).(*arrowArray.StringBuilder).AppendNull()
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "blob.txt", *item.Name)
-	require.Nil(t, item.Properties.CreationTime)
-	require.Nil(t, item.Properties.BlobType)
-	require.Nil(t, item.Properties.ETag)
-	require.Nil(t, item.Properties.ContentLength)
-	require.Nil(t, item.Properties.ContentType)
-	require.Nil(t, item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-	require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
-}
-
-func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
-	alloc := memory.DefaultAllocator
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
-	schema := coreSchema(&md)
-
-	var buf bytes.Buffer
-	w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
-	// First batch: 2 rows
-	builder := arrowArray.NewRecordBuilder(alloc, schema)
-	for i := 0; i < 2; i++ {
-		builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
-		for j := 1; j < 11; j++ {
-			builder.Field(j).AppendNull()
-		}
-	}
-	rec := builder.NewRecordBatch()
-	require.NoError(t, w.Write(rec))
-	rec.Release()
-	builder.Release()
-
-	// Second batch: 1 row
-	builder = arrowArray.NewRecordBuilder(alloc, schema)
-	builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
-	for j := 1; j < 11; j++ {
-		builder.Field(j).AppendNull()
-	}
-	rec = builder.NewRecordBatch()
-	require.NoError(t, w.Write(rec))
-	rec.Release()
-	builder.Release()
-
-	require.NoError(t, w.Close())
-
-	resp := makeHTTPResponse(buf.Bytes(), nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 3)
-}
-
-func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		// A regular blob
-		b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
-		b.Field(1).(*arrowArray.StringBuilder).AppendNull()
-
-		// A blob prefix (virtual directory)
-		b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
-		b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleHierarchyListResponse(resp)
-	require.NoError(t, err)
-
-	require.Len(t, result.Segment.BlobItems, 1)
-	require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
-
-	require.Len(t, result.Segment.BlobPrefixes, 1)
-	require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
-}
-
-func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-	})
-
-	resp := makeHTTPResponse(data, map[string]string{
-		"x-ms-request-id":        "req-123",
-		"x-ms-version":           "2026-10-06",
-		"x-ms-client-request-id": "client-req-456",
-		"Date":                   "Mon, 01 Jan 2024 00:00:00 GMT",
-	})
-
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Equal(t, "req-123", *result.RequestID)
-	require.Equal(t, "2026-10-06", *result.Version)
-	require.Equal(t, "client-req-456", *result.ClientRequestID)
-	require.NotNil(t, result.Date)
-}
-
-func TestHandleFlatListResponse_MapFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
-		{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-
-		metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
-		metaBuilder.Append(true)
-		metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
-		metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
-
-		tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
-		tagBuilder.Append(true)
-		tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
-		tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-
-	item := result.Segment.BlobItems[0]
-	require.NotNil(t, item.Metadata)
-	require.Equal(t, "value1", *item.Metadata["key1"])
-
-	require.NotNil(t, item.BlobTags)
-	require.Len(t, item.BlobTags.BlobTagSet, 1)
-}
-
-func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
-		b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
-		b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	item := result.Segment.BlobItems[0]
-	require.True(t, *item.Deleted)
-	require.False(t, *item.IsCurrentVersion)
-	require.True(t, *item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
-		b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
-		b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
-		b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
-	require.Equal(t, "snap-123", *item.Snapshot)
-	require.True(t, *item.IsCurrentVersion)
-}

backend/azureblob/arrowlist/arrowlist.go

@@ -1,259 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Package arrowlist implements the experimental "Blob Listing with Apache
-// Arrow" feature on top of the released Azure azblob SDK.
-//
-// The Azure SDK for Go has support for Arrow listing on the unreleased
-// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
-// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
-// response types. Until that ships in a tagged azblob release this package
-// fills the gap: it exposes the same options and pager interface as the
-// experimental SDK, built only on the public API of the released SDK plus
-// copies of two small internal auth policies (see sharedkey.go and
-// challenge_policy.go).
-//
-// This package is temporary. When Arrow listing ships in a released azblob
-// the whole package should be deleted and its callers pointed back at
-// container.Client.NewListBlobsHierarchyPager - the option and response
-// types here are deliberately source compatible with the SDK's to make that
-// a mechanical change.
-package arrowlist
-
-import (
-	"context"
-	"errors"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-)
-
-const (
-	// moduleName and moduleVersion identify this package in the azcore
-	// telemetry policy's User-Agent fragment.
-	moduleName    = "github.com/rclone/rclone/backend/azureblob/arrowlist"
-	moduleVersion = "v0.0.1"
-
-	// serviceVersion is the x-ms-version sent with listing requests. Arrow
-	// responses and the startFrom/endBefore parameters need at least
-	// 2026-06-06.
-	serviceVersion = "2026-06-06"
-
-	// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
-	// own clients.
-	tokenScope = "https://storage.azure.com/.default"
-)
-
-// ErrEndBeforeXMLFallback is returned by the pager when the server answered
-// with XML instead of Arrow while EndBefore was set. The service only honours
-// endBefore on the Arrow listing path, so an XML page could silently contain
-// blobs outside the requested range and cannot be used.
-var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
-
-// ListBlobsHierarchyOptions provides the configuration for
-// Client.NewListBlobsHierarchyPager.
-//
-// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
-// the released SDK's options are embedded (providing Include, Marker,
-// MaxResults, Prefix and StartFrom by promotion) with the two fields the
-// released SDK lacks added alongside.
-type ListBlobsHierarchyOptions struct {
-	container.ListBlobsHierarchyOptions
-	// EndBefore limits listing to blobs whose full container path is
-	// lexically before it (exclusive). Only honoured on Arrow responses.
-	EndBefore *string
-	// UseArrowFormat requests the response as an Apache Arrow IPC stream.
-	UseArrowFormat *bool
-}
-
-// ClientOptions contains the optional parameters for client creation.
-type ClientOptions struct {
-	azcore.ClientOptions
-}
-
-// Client issues Arrow listing requests to a single container.
-type Client struct {
-	endpoint string
-	azClient *azcore.Client
-}
-
-// newClient creates a Client for the container at containerURL sending
-// requests through a pipeline with the given auth policy (which may be nil
-// for anonymous or SAS access).
-func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
-	if options == nil {
-		options = &ClientOptions{}
-	}
-	plOpts := runtime.PipelineOptions{}
-	if authPolicy != nil {
-		plOpts.PerRetry = []policy.Policy{authPolicy}
-	}
-	azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
-	if err != nil {
-		return nil, err
-	}
-	return &Client{
-		endpoint: containerURL,
-		azClient: azClient,
-	}, nil
-}
-
-// NewClient creates a Client authenticating with a token credential.
-func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
-}
-
-// NewClientWithNoCredential creates a Client with no authentication, for
-// anonymous access or a containerURL carrying a SAS token in its query.
-func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, nil, options)
-}
-
-// NewClientWithSharedKeyCredential creates a Client signing requests with the
-// account's shared key.
-func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
-}
-
-// NewListBlobsHierarchyPager returns a pager over the container's blobs,
-// requesting Arrow responses when o.UseArrowFormat is set and returning the
-// same response type as container.Client.NewListBlobsHierarchyPager.
-//
-// If the service does not support Arrow listing (the feature not enabled on
-// the account, or an HNS account) it answers with XML, which is decoded
-// transparently - unless EndBefore is set, in which case the fetcher returns
-// ErrEndBeforeXMLFallback (see there).
-func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
-	// Copy the options so advancing the Marker doesn't mutate the caller's struct.
-	opts := ListBlobsHierarchyOptions{}
-	if o != nil {
-		opts = *o
-	}
-	useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
-	return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
-		More: func(page container.ListBlobsHierarchyResponse) bool {
-			return page.NextMarker != nil && len(*page.NextMarker) > 0
-		},
-		Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
-			if page != nil {
-				opts.Marker = page.NextMarker
-			}
-			req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
-			if err != nil {
-				return container.ListBlobsHierarchyResponse{}, err
-			}
-			resp, err := c.azClient.Pipeline().Do(req)
-			if err != nil {
-				return container.ListBlobsHierarchyResponse{}, err
-			}
-			if !runtime.HasStatusCode(resp, http.StatusOK) {
-				return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
-			}
-			if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
-				return HandleHierarchyListResponse(resp)
-			}
-			if opts.EndBefore != nil {
-				return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
-			}
-			return handleXMLResponse(resp)
-		},
-	})
-}
-
-// listCreateRequest builds the ListBlobs request, mirroring the generated
-// request builders in the SDK.
-func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
-	req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
-	if err != nil {
-		return nil, err
-	}
-	// Reading the existing query first preserves any SAS token in the endpoint.
-	reqQP := req.Raw().URL.Query()
-	reqQP.Set("comp", "list")
-	reqQP.Set("restype", "container")
-	reqQP.Set("delimiter", delimiter)
-	if include := formatInclude(opts.Include); include != "" {
-		reqQP.Set("include", include)
-	}
-	if opts.Marker != nil {
-		reqQP.Set("marker", *opts.Marker)
-	}
-	if opts.MaxResults != nil {
-		reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
-	}
-	if opts.Prefix != nil {
-		reqQP.Set("prefix", *opts.Prefix)
-	}
-	if opts.StartFrom != nil {
-		reqQP.Set("startFrom", *opts.StartFrom)
-	}
-	if useArrow && opts.EndBefore != nil {
-		reqQP.Set("endBefore", *opts.EndBefore)
-	}
-	req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
-	if useArrow {
-		// Stream the Arrow body rather than buffering it in the pipeline.
-		runtime.SkipBodyDownload(req)
-		req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
-	} else {
-		req.Raw().Header["Accept"] = []string{"application/xml"}
-	}
-	req.Raw().Header["x-ms-version"] = []string{serviceVersion}
-	return req, nil
-}
-
-// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
-// generated ListBlobHierarchySegmentHandleResponse.
-func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
-	result := container.ListBlobsHierarchyResponse{}
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-	if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
-		return container.ListBlobsHierarchyResponse{}, err
-	}
-	return result, nil
-}
-
-// formatInclude renders the include datasets as the comma separated query
-// parameter value, in the same order as the SDK's unexported
-// ListBlobsInclude.format.
-func formatInclude(l container.ListBlobsInclude) string {
-	var include []string
-	if l.Copy {
-		include = append(include, "copy")
-	}
-	if l.Deleted {
-		include = append(include, "deleted")
-	}
-	if l.DeletedWithVersions {
-		include = append(include, "deletedwithversions")
-	}
-	if l.ImmutabilityPolicy {
-		include = append(include, "immutabilitypolicy")
-	}
-	if l.LegalHold {
-		include = append(include, "legalhold")
-	}
-	if l.Metadata {
-		include = append(include, "metadata")
-	}
-	if l.Snapshots {
-		include = append(include, "snapshots")
-	}
-	if l.Tags {
-		include = append(include, "tags")
-	}
-	if l.UncommittedBlobs {
-		include = append(include, "uncommittedblobs")
-	}
-	if l.Versions {
-		include = append(include, "versions")
-	}
-	if l.Permissions {
-		include = append(include, "permissions")
-	}
-	return strings.Join(include, ",")
-}

backend/azureblob/arrowlist/arrowlist_test.go

@@ -1,199 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-package arrowlist
-
-import (
-	"context"
-	"encoding/base64"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/apache/arrow-go/v18/arrow"
-	arrowArray "github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// arrowPage builds a one row Arrow IPC listing page with the given blob name
-// and NextMarker.
-func arrowPage(t *testing.T, name, nextMarker string) []byte {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-	return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append(name)
-		b.Field(1).(*arrowArray.Uint64Builder).Append(42)
-	})
-}
-
-func TestPagerArrow(t *testing.T) {
-	var requests []*http.Request
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		requests = append(requests, r.Clone(context.Background()))
-		w.Header().Set("Content-Type", ArrowContentType)
-		if r.URL.Query().Get("marker") == "" {
-			_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
-		} else {
-			_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
-		}
-	}))
-	defer srv.Close()
-
-	// SAS style query on the endpoint must be preserved in requests.
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
-	require.NoError(t, err)
-
-	opts := &ListBlobsHierarchyOptions{
-		ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
-			Include:    container.ListBlobsInclude{Metadata: true, Tags: true},
-			Prefix:     to.Ptr("dir/"),
-			MaxResults: to.Ptr(int32(1000)),
-			StartFrom:  to.Ptr("testcontainer/dir/a"),
-		},
-		EndBefore:      to.Ptr("testcontainer/dir/n"),
-		UseArrowFormat: to.Ptr(true),
-	}
-	pager := client.NewListBlobsHierarchyPager("/", opts)
-
-	var names []string
-	for pager.More() {
-		page, err := pager.NextPage(context.Background())
-		require.NoError(t, err)
-		assert.Equal(t, ArrowContentType, *page.ContentType)
-		for _, item := range page.Segment.BlobItems {
-			names = append(names, *item.Name)
-			assert.Equal(t, int64(42), *item.Properties.ContentLength)
-		}
-	}
-	assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
-
-	require.Len(t, requests, 2)
-	q := requests[0].URL.Query()
-	assert.Equal(t, "list", q.Get("comp"))
-	assert.Equal(t, "container", q.Get("restype"))
-	assert.Equal(t, "/", q.Get("delimiter"))
-	assert.Equal(t, "metadata,tags", q.Get("include"))
-	assert.Equal(t, "dir/", q.Get("prefix"))
-	assert.Equal(t, "1000", q.Get("maxresults"))
-	assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
-	assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
-	assert.Equal(t, "fakesas", q.Get("sv"))
-	assert.Empty(t, q.Get("marker"))
-	assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
-	assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
-	assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
-
-	// The caller's options must not have been mutated by paging.
-	assert.Nil(t, opts.Marker)
-}
-
-const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
-<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
-  <Blobs>
-    <Blob>
-      <Name>file.txt</Name>
-      <Properties>
-        <Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
-        <Etag>0x8D1</Etag>
-        <Content-Length>7</Content-Length>
-        <BlobType>BlockBlob</BlobType>
-      </Properties>
-    </Blob>
-    <BlobPrefix>
-      <Name>subdir/</Name>
-    </BlobPrefix>
-  </Blobs>
-  <NextMarker />
-</EnumerationResults>`
-
-func TestPagerXMLFallback(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/xml")
-		_, _ = w.Write([]byte(xmlListResponse))
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: to.Ptr(true),
-	})
-	page, err := pager.NextPage(context.Background())
-	require.NoError(t, err)
-	assert.False(t, pager.More())
-	assert.Equal(t, "application/xml", *page.ContentType)
-	require.Len(t, page.Segment.BlobItems, 1)
-	assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
-	assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
-	require.Len(t, page.Segment.BlobPrefixes, 1)
-	assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
-}
-
-func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/xml")
-		_, _ = w.Write([]byte(xmlListResponse))
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		EndBefore:      to.Ptr("testcontainer/n"),
-		UseArrowFormat: to.Ptr(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
-}
-
-func TestPagerSharedKeyAuth(t *testing.T) {
-	var gotAuth, gotDate string
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gotAuth = r.Header.Get("Authorization")
-		gotDate = r.Header.Get("x-ms-date")
-		w.Header().Set("Content-Type", ArrowContentType)
-		_, _ = w.Write(arrowPage(t, "blob.txt", ""))
-	}))
-	defer srv.Close()
-
-	cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
-	require.NoError(t, err)
-	client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: to.Ptr(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.NoError(t, err)
-	assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
-	assert.NotEmpty(t, gotDate)
-}
-
-func TestPagerResponseError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: to.Ptr(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.Error(t, err)
-	assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
-}

backend/azureblob/arrowlist/challenge_policy.go

@@ -1,121 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
-// released SDK does not export its storage bearer-challenge policy, which is
-// needed for token authentication with the same semantics as the SDK's own
-// clients.
-
-package arrowlist
-
-import (
-	"errors"
-	"net/http"
-	"strings"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-)
-
-type storageAuthorizer struct {
-	scopes   []string
-	tenantID string
-}
-
-// NewStorageChallengePolicy returns a bearer token policy for cred which
-// handles the storage service's authentication challenges.
-func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
-	s := storageAuthorizer{scopes: []string{audience}}
-	return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
-		AuthorizationHandler: policy.AuthorizationHandler{
-			OnRequest:   s.onRequest,
-			OnChallenge: s.onChallenge,
-		},
-		InsecureAllowCredentialWithHTTP: allowHTTP,
-	})
-}
-
-func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
-	return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
-	// parse the challenge
-	err := s.parseChallenge(resp)
-	if err != nil {
-		return err
-	}
-	// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
-	return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-type challengePolicyError struct {
-	err error
-}
-
-func (c *challengePolicyError) Error() string {
-	return c.err.Error()
-}
-
-func (*challengePolicyError) NonRetriable() {
-	// marker method
-}
-
-func (c *challengePolicyError) Unwrap() error {
-	return c.err
-}
-
-// parses Tenant ID from auth challenge
-// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
-func parseTenant(url string) string {
-	if url == "" {
-		return ""
-	}
-	parts := strings.Split(url, "/")
-	if len(parts) >= 3 {
-		tenant := parts[3]
-		tenant = strings.ReplaceAll(tenant, ",", "")
-		return tenant
-	}
-	return ""
-}
-
-func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
-	authHeader := resp.Header.Get("WWW-Authenticate")
-	if authHeader == "" {
-		return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
-	}
-
-	// Strip down to auth and resource
-	// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
-	authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
-
-	parts := strings.Split(authHeader, " ")
-
-	vals := map[string]string{}
-	for _, part := range parts {
-		subParts := strings.Split(part, "=")
-		if len(subParts) == 2 {
-			stripped := strings.ReplaceAll(subParts[1], "\"", "")
-			stripped = strings.TrimSuffix(stripped, ",")
-			vals[subParts[0]] = stripped
-		}
-	}
-
-	s.tenantID = parseTenant(vals["authorization_uri"])
-
-	scope := vals["resource_id"]
-	if scope == "" {
-		return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
-	}
-
-	if !strings.HasSuffix(scope, "/.default") {
-		scope += "/.default"
-	}
-	s.scopes = []string{scope}
-	return nil
-}

backend/azureblob/arrowlist/sharedkey.go

@@ -1,318 +0,0 @@
-//go:build !plan9 && !solaris && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
-// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
-// with the internal header constants inlined. The released SDK does not
-// export its shared key signing policy, so a copy is needed to sign the
-// Arrow listing requests this package sends through its own pipeline.
-
-package arrowlist
-
-import (
-	"bytes"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/url"
-	"sort"
-	"strings"
-	"sync/atomic"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/rclone/rclone/fs"
-)
-
-// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
-// storage account's name and either its primary or secondary key.
-func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
-	c := SharedKeyCredential{accountName: accountName}
-	if err := c.SetAccountKey(accountKey); err != nil {
-		return nil, err
-	}
-	return &c, nil
-}
-
-// SharedKeyCredential contains an account's name and its primary or secondary key.
-type SharedKeyCredential struct {
-	// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
-	accountName string
-	accountKey  atomic.Value // []byte
-}
-
-// AccountName returns the Storage account's name.
-func (c *SharedKeyCredential) AccountName() string {
-	return c.accountName
-}
-
-// SetAccountKey replaces the existing account key with the specified account key.
-func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
-	_bytes, err := base64.StdEncoding.DecodeString(accountKey)
-	if err != nil {
-		return fmt.Errorf("decode account key: %w", err)
-	}
-	c.accountKey.Store(_bytes)
-	return nil
-}
-
-// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
-func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
-	h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
-	_, err := h.Write([]byte(message))
-	return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
-}
-
-func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
-	// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
-	headers := req.Header
-	contentLength := getHeader("Content-Length", headers)
-	if contentLength == "0" {
-		contentLength = ""
-	}
-
-	canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
-	if err != nil {
-		return "", err
-	}
-
-	stringToSign := strings.Join([]string{
-		req.Method,
-		getHeader("Content-Encoding", headers),
-		getHeader("Content-Language", headers),
-		contentLength,
-		getHeader("Content-MD5", headers),
-		getHeader("Content-Type", headers),
-		"", // Empty date because x-ms-date is expected (as per web page above)
-		getHeader("If-Modified-Since", headers),
-		getHeader("If-Match", headers),
-		getHeader("If-None-Match", headers),
-		getHeader("If-Unmodified-Since", headers),
-		getHeader("Range", headers),
-		c.buildCanonicalizedHeader(headers),
-		canonicalizedResource,
-	}, "\n")
-	return stringToSign, nil
-}
-
-func getHeader(key string, headers map[string][]string) string {
-	if headers == nil {
-		return ""
-	}
-	if v, ok := headers[key]; ok {
-		if len(v) > 0 {
-			return v[0]
-		}
-	}
-
-	return ""
-}
-
-func getWeightTables() [][]int {
-	tableLv0 := [...]int{
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
-		0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
-		0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
-		0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
-		0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
-		0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
-		0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
-	}
-	tableLv2 := [...]int{
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-	}
-	tables := [][]int{tableLv0[:], tableLv2[:]}
-	return tables
-}
-
-// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
-func compareHeaders(lhs, rhs string, tables [][]int) int {
-	currLevel, i, j := 0, 0, 0
-	n := len(tables)
-	lhsLen := len(lhs)
-	rhsLen := len(rhs)
-
-	for currLevel < n {
-		if currLevel == (n-1) && i != j {
-			if i > j {
-				return -1
-			}
-			if i < j {
-				return 1
-			}
-			return 0
-		}
-
-		var w1, w2 int
-
-		// Check bounds before accessing lhs[i]
-		if i < lhsLen {
-			w1 = tables[currLevel][lhs[i]]
-		} else {
-			w1 = 0x1
-		}
-
-		// Check bounds before accessing rhs[j]
-		if j < rhsLen {
-			w2 = tables[currLevel][rhs[j]]
-		} else {
-			w2 = 0x1
-		}
-
-		if w1 == 0x1 && w2 == 0x1 {
-			i = 0
-			j = 0
-			currLevel++
-		} else if w1 == w2 {
-			i++
-			j++
-		} else if w1 == 0 {
-			i++
-		} else if w2 == 0 {
-			j++
-		} else {
-			if w1 < w2 {
-				return -1
-			}
-			if w1 > w2 {
-				return 1
-			}
-			return 0
-		}
-	}
-	return 0
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
-	cm := map[string][]string{}
-	for k, v := range headers {
-		headerName := strings.TrimSpace(strings.ToLower(k))
-		if strings.HasPrefix(headerName, "x-ms-") {
-			cm[headerName] = v // NOTE: the value must not have any whitespace around it.
-		}
-	}
-	if len(cm) == 0 {
-		return ""
-	}
-
-	keys := make([]string, 0, len(cm))
-	for key := range cm {
-		keys = append(keys, key)
-	}
-	tables := getWeightTables()
-	// Sort the keys using the custom comparator
-	sort.Slice(keys, func(i, j int) bool {
-		return compareHeaders(keys[i], keys[j], tables) < 0
-	})
-	ch := bytes.NewBufferString("")
-	for i, key := range keys {
-		if i > 0 {
-			ch.WriteRune('\n')
-		}
-		ch.WriteString(key)
-		ch.WriteRune(':')
-		ch.WriteString(strings.Join(cm[key], ","))
-	}
-	return ch.String()
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
-	// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
-	cr := bytes.NewBufferString("/")
-	cr.WriteString(c.accountName)
-
-	if len(u.Path) > 0 {
-		// Any portion of the CanonicalizedResource string that is derived from
-		// the resource's URI should be encoded exactly as it is in the URI.
-		// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
-		cr.WriteString(u.EscapedPath())
-	} else {
-		// a slash is required to indicate the root path
-		cr.WriteString("/")
-	}
-
-	// params is a map[string][]string; param name is key; params values is []string
-	params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
-	if err != nil {
-		return "", fmt.Errorf("failed to parse query params: %w", err)
-	}
-
-	if len(params) > 0 { // There is at least 1 query parameter
-		var paramNames []string // We use this to sort the parameter key names
-		for paramName := range params {
-			paramNames = append(paramNames, paramName) // paramNames must be lowercase
-		}
-		sort.Strings(paramNames)
-
-		for _, paramName := range paramNames {
-			paramValues := params[paramName]
-			sort.Strings(paramValues)
-
-			// Join the sorted key values separated by ','
-			// Then prepend "keyName:"; then add this string to the buffer
-			cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
-		}
-	}
-	return cr.String(), nil
-}
-
-// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
-func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
-	return cred.computeHMACSHA256(message)
-}
-
-// SharedKeyCredPolicy is a pipeline policy that signs each request with the
-// account's shared key.
-type SharedKeyCredPolicy struct {
-	cred *SharedKeyCredential
-}
-
-// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
-func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
-	return &SharedKeyCredPolicy{cred: cred}
-}
-
-// Do signs the request and sends it via the next policy in the pipeline.
-func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
-	// skip adding the authorization header if no SharedKeyCredential was provided.
-	// this prevents a panic that might be hard to diagnose and allows testing
-	// against http endpoints that don't require authentication.
-	if s.cred == nil {
-		return req.Next()
-	}
-
-	if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
-		req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
-	}
-	stringToSign, err := s.cred.buildStringToSign(req.Raw())
-	if err != nil {
-		return nil, err
-	}
-	signature, err := s.cred.computeHMACSHA256(stringToSign)
-	if err != nil {
-		return nil, err
-	}
-	authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
-	req.Raw().Header.Set("Authorization", authHeader)
-
-	response, err := req.Next()
-	if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
-		// Service failed to authenticate request, log it
-		fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
-	}
-	return response, err
-}

backend/azureblob/azureblob.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
@@ -27,15 +27,12 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/backend/azureblob/auth"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/chunksize"
@@ -52,6 +49,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/rclone/rclone/lib/transferaccounter"
 	"golang.org/x/sync/errgroup"
 )
@@ -70,6 +68,7 @@ const (
 	defaultChunkSize      = 4 * fs.Mebi
 	defaultAccessTier     = blob.AccessTier("") // FIXME AccessTierNone
 	sasCopyValidity       = time.Hour           // how long SAS should last when doing server side copy
+	sasCopyStartSkew      = 15 * time.Minute    // how far back SAS should start to allow for clock skew
 )
 
 // setAcceptEncodingGzip is a per-call policy that sets Accept-Encoding: gzip
@@ -241,40 +240,38 @@ avoid the time out.`,
 			Advanced: true,
 		}, {
 			Name: "use_arrow_list",
-			Help: `Use the experimental Apache Arrow listing format.
+			Help: `Use the Apache Arrow listing format.
 
 If set, directory listings are fetched using the ListBlobs Apache
-Arrow response format instead of XML. This can be faster for very
-large containers.
+Arrow response format instead of XML. Arrow responses are smaller and
+much cheaper to parse, making listings of large containers several
+times faster. Combine with "list_parallelism" for the biggest gains.
 
-This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
-preview feature to be enabled on the storage account. It is NOT
-supported on accounts with a hierarchical namespace (ADLS Gen2) -
-those return a 409 error. If the feature is not enabled the server
-returns XML and the listing transparently falls back to the normal XML
-path (logged at debug level).
-
-Not supported with connection_string auth - falls back to normal
-listing.`,
+"Blob Listing with Apache Arrow" is in public preview at Microsoft and
+is only supported on flat namespace accounts. On accounts with a
+hierarchical namespace (ADLS Gen2), or where the feature is otherwise
+unavailable, the server returns XML and the listing transparently
+falls back to the normal XML path (logged at debug level).`,
 			Default:  false,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
 		}, {
 			Name: "list_parallelism",
 			Help: `Number of parallel shards to list a directory with.
 
-EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
-directory is split into this many ranges which are listed concurrently
-using the Arrow startFrom/endBefore range parameters. This can
-dramatically speed up listing containers with millions of objects, for
-both recursive (ListR) and single directory listings.
+If set greater than 1, the blob name keyspace of each directory is
+split into ranges which are listed concurrently using the Arrow
+startFrom/endBefore range parameters. This can dramatically speed up
+listing containers with millions of objects, for both recursive
+(ListR) and single directory listings. Speed keeps improving up to a
+parallelism of around 30.
 
 This has no effect unless "use_arrow_list" is also set, as Arrow is the
-only listing path that supports server-side name ranges. The default of
-0 (or 1) lists sequentially.`,
+only listing path that supports server-side name ranges. If the
+account does not support range listing (e.g. it has a hierarchical
+namespace) the listing falls back to sequential. The default of 0 (or
+1) lists sequentially.`,
 			Default:  0,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
 		}, {
 			Name: "access_tier",
 			Help: `Access tier of blob: hot, cool, cold or archive.
@@ -368,7 +365,22 @@ creates an empty object ending with "/", to persist the folder.
 
 This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
 the Microsoft standard.
+
+This is not needed on storage accounts with a hierarchical namespace
+(see the hns option) as these always support empty directories.
  `,
+		}, {
+			Name:     "hns",
+			Default:  fs.Tristate{},
+			Advanced: true,
+			Help: `Set if the storage account has a hierarchical namespace.
+
+Storage accounts with hierarchical namespace enabled (Azure Data Lake
+Storage Gen2) have real directories which can be empty.
+
+If this is unset then rclone will read it from the storage account
+and save it in the config file.
+`,
 		}, {
 			Name: "no_check_container",
 			Help: `If set, don't attempt to check the container exists or create it.
@@ -436,6 +448,7 @@ type Options struct {
 	Enc                  encoder.MultiEncoder `config:"encoding"`
 	PublicAccess         string               `config:"public_access"`
 	DirectoryMarkers     bool                 `config:"directory_markers"`
+	HNS                  fs.Tristate          `config:"hns"`
 	NoCheckContainer     bool                 `config:"no_check_container"`
 	NoHeadObject         bool                 `config:"no_head_object"`
 	DeleteSnapshots      string               `config:"delete_snapshots"`
@@ -449,10 +462,8 @@ type Fs struct {
 	opt                Options                      // parsed config options
 	ci                 *fs.ConfigInfo               // global config
 	features           *fs.Features                 // optional features
-	cntSVCcacheMu      sync.Mutex                   // mutex to protect cntSVCcache and arrowCntSVCcache
+	cntSVCcacheMu      sync.Mutex                   // mutex to protect cntSVCcache
 	cntSVCcache        map[string]*container.Client // reference to containerClient per container
-	arrowCntSVCcache   map[string]*arrowlist.Client // reference to arrowlist client per container
-	arrowClientOpts    *arrowlist.ClientOptions     // client options for the arrowlist clients
 	arrowXMLFallback   atomic.Bool                  // set once the server answers XML so parallel listing is not retried
 	svc                *service.Client              // client to access azblob
 	cred               azcore.TokenCredential       // how to generate tokens (may be nil)
@@ -461,6 +472,7 @@ type Fs struct {
 	rootContainer      string                       // container part of root (if any)
 	rootDirectory      string                       // directory part of root (if any)
 	isLimited          bool                         // if limited to one container
+	hns                bool                         // if the storage account has a hierarchical namespace
 	cache              *bucket.Cache                // cache for container creation status
 	pacer              *fs.Pacer                    // To pace and retry the API calls
 	uploadToken        *pacer.TokenDispenser        // control concurrency
@@ -527,7 +539,7 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
 	containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
-	if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
+	if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
 		containerPath = containerPath[:len(containerPath)-1]
 	}
 	return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
@@ -575,8 +587,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	var storageErr *azcore.ResponseError
-	if errors.As(err, &storageErr) {
+	if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
 		// General errors from:
 		// https://learn.microsoft.com/en-us/rest/api/storageservices/common-rest-api-error-codes
 		// Blob specific errors from:
@@ -671,15 +682,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:             name,
-		opt:              *opt,
-		ci:               ci,
-		pacer:            fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		uploadToken:      pacer.NewTokenDispenser(ci.Transfers),
-		copyToken:        pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
-		cache:            bucket.NewCache(),
-		cntSVCcache:      make(map[string]*container.Client, 1),
-		arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
+		name:        name,
+		opt:         *opt,
+		ci:          ci,
+		pacer:       fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+		copyToken:   pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
+		cache:       bucket.NewCache(),
+		cntSVCcache: make(map[string]*container.Client, 1),
 	}
 	f.publicAccess = container.PublicAccessType(opt.PublicAccess)
 	f.setRoot(root)
@@ -696,10 +706,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ServerSideAcrossConfigs: true,
 		DoubleSlash:             true,
 	}).Fill(ctx, f)
-	if opt.DirectoryMarkers {
-		f.features.CanHaveEmptyDirectories = true
-		fs.Debugf(f, "Using directory markers")
-	}
 
 	conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
 		DefaultBaseURL:                   storageDefaultBaseURL,
@@ -726,21 +732,34 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.usingSharedKeyCred = res.UsingSharedKeyCred
 	f.anonymous = res.Anonymous
 
-	// Client options for the arrowlist clients, using the same transport and
-	// gzip policy as the SDK clients built above.
-	f.arrowClientOpts = &arrowlist.ClientOptions{
-		ClientOptions: azcore.ClientOptions{
-			Transport:       auth.Transporter(ctx),
-			PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
-		},
-	}
-
 	// if using Container level SAS put the container client into the cache
 	if opt.SASURL != "" && res.Container != "" {
 		_ = f.cntSVC(res.Container)
 		f.isLimited = true
 	}
 
+	if opt.HNS.Valid {
+		f.hns = opt.HNS.Value
+	} else if !f.anonymous {
+		containerName := f.rootContainer
+		if containerName == "" {
+			containerName = res.Container
+		}
+		f.hns, err = f.readHNS(ctx, containerName)
+		if err != nil {
+			fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
+		} else {
+			m.Set("hns", strconv.FormatBool(f.hns))
+		}
+	}
+	if f.hns {
+		fs.Debugf(f, "Storage account has a hierarchical namespace")
+	}
+	if f.dirMarkers() {
+		f.features.CanHaveEmptyDirectories = true
+		fs.Debugf(f, "Using directory markers")
+	}
+
 	if f.rootContainer != "" && f.rootDirectory != "" {
 		// Check to see if the (container,directory) is actually an existing file
 		oldRoot := f.root
@@ -761,6 +780,39 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
+// readHNS reads whether the storage account has a hierarchical namespace
+//
+// If containerName is set then the container client is used which works
+// with credentials limited to that container.
+func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
+	var isHNS *bool
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		if containerName != "" {
+			var resp container.GetAccountInfoResponse
+			resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
+			isHNS = resp.IsHierarchicalNamespaceEnabled
+		} else {
+			var resp service.GetAccountInfoResponse
+			resp, err = f.svc.GetAccountInfo(ctx, nil)
+			isHNS = resp.IsHierarchicalNamespaceEnabled
+		}
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return false, err
+	}
+	if isHNS == nil {
+		return false, errors.New("not returned by server")
+	}
+	return *isHNS, nil
+}
+
+// dirMarkers returns true if directories are persisted with directory markers
+func (f *Fs) dirMarkers() bool {
+	return f.opt.DirectoryMarkers || f.hns
+}
+
 // return the container client for the container passed in
 func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
 	f.cntSVCcacheMu.Lock()
@@ -773,44 +825,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
 	return containerClient
 }
 
-// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
-// credentials can't be used for Arrow listing.
-var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
-
-// return the arrowlist client for the container passed in
-//
-// Returns errArrowAuthUnsupported if the configured credentials can't be
-// reused for the arrowlist client's pipeline.
-func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
-	// The container URL includes any SAS token in its query
-	url := f.cntSVC(containerName).URL()
-	f.cntSVCcacheMu.Lock()
-	defer f.cntSVCcacheMu.Unlock()
-	if client, ok := f.arrowCntSVCcache[containerName]; ok {
-		return client, nil
-	}
-	switch {
-	case f.usingSharedKeyCred:
-		// Covers account+key and the emulator (auth fills in Account/Key)
-		var cred *arrowlist.SharedKeyCredential
-		cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
-		if err == nil {
-			client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
-		}
-	case f.cred != nil:
-		client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
-	case f.anonymous || f.opt.SASURL != "":
-		client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
-	default:
-		return nil, errArrowAuthUnsupported
-	}
-	if err != nil {
-		return nil, err
-	}
-	f.arrowCntSVCcache[containerName] = client
-	return client, nil
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -836,7 +850,12 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(ctx, remote, nil)
+	o, err := f.newObjectWithInfo(ctx, remote, nil)
+	if err == fs.ErrorNotAFile {
+		// The blob is a directory
+		return nil, fs.ErrorObjectNotFound
+	}
+	return o, err
 }
 
 // getBlobSVC creates a blob client
@@ -905,15 +924,15 @@ func mapMetadataToAzure(meta map[string]string, logf func(string, ...any)) (head
 		lowerKey := strings.ToLower(k)
 		switch lowerKey {
 		case "cache-control":
-			headers.BlobCacheControl = pString(v)
+			headers.BlobCacheControl = new(v)
 		case "content-disposition":
-			headers.BlobContentDisposition = pString(v)
+			headers.BlobContentDisposition = new(v)
 		case "content-encoding":
-			headers.BlobContentEncoding = pString(v)
+			headers.BlobContentEncoding = new(v)
 		case "content-language":
-			headers.BlobContentLanguage = pString(v)
+			headers.BlobContentLanguage = new(v)
 		case "content-type":
-			headers.BlobContentType = pString(v)
+			headers.BlobContentType = new(v)
 		case "x-ms-tags":
 			parsed, perr := parseXMsTags(v)
 			if perr != nil {
@@ -1150,18 +1169,33 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
 		if endsWithSlash || remote == "" {
 			return true
 		}
-		// Note that metadata with hdi_isfolder = true seems to be a
-		// defacto standard for marking blobs as directories.
-		// Note also that the metadata hasn't been normalised to lower case yet
-		for k, v := range metadata {
-			if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
-				return true
-			}
+		return hasDirMetadata(metadata)
+	}
+	return false
+}
+
+// Returns whether the metadata marks the blob as a directory
+func hasDirMetadata(metadata map[string]*string) bool {
+	// Note that metadata with hdi_isfolder = true seems to be a
+	// defacto standard for marking blobs as directories.
+	// Note also that the metadata hasn't been normalised to lower case yet
+	for k, v := range metadata {
+		if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
+			return true
 		}
 	}
 	return false
 }
 
+// Returns whether props are for a directory on a storage account with
+// a hierarchical namespace
+//
+// These ignore a trailing "/" on the name so a file can be returned
+// when looking for a directory.
+func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
+	return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
+}
+
 // listFn is called from list to handle an object
 type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
 
@@ -1187,27 +1221,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 		delimiter = "/"
 	}
 
-	opts := &arrowlist.ListBlobsHierarchyOptions{
-		ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
-			// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
-			Include: container.ListBlobsInclude{
-				Copy:             false,
-				Metadata:         true,
-				Snapshots:        false,
-				UncommittedBlobs: false,
-				Deleted:          false,
-			},
-			Prefix:     &directory,
-			MaxResults: &maxResults,
+	opts := &container.ListBlobsHierarchyOptions{
+		// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
+		Include: container.ListBlobsInclude{
+			Copy:             false,
+			Metadata:         true,
+			Snapshots:        false,
+			UncommittedBlobs: false,
+			Deleted:          false,
 		},
+		Prefix:     &directory,
+		MaxResults: &maxResults,
 	}
-	// Experimental: request the Apache Arrow listing format. The arrowlist
-	// pager requests an Arrow IPC stream and decodes it, falling back to XML
-	// if the account doesn't have Arrow listing enabled. Skip the
-	// maxResults==1 probe (isEmpty) which doesn't benefit.
+	// Request the Apache Arrow listing format. The SDK pager requests an
+	// Arrow IPC stream and decodes it, falling back to XML if the account
+	// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
+	// (isEmpty) which doesn't benefit.
 	useArrow := f.opt.UseArrowList && maxResults != 1
 	if useArrow {
-		opts.UseArrowFormat = to.Ptr(true)
+		opts.ResponseFormat = container.StorageResponseFormatArrow
 	}
 
 	var foundItems int
@@ -1222,15 +1254,18 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 	if err != nil {
 		return err
 	}
-	if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
+	if f.dirMarkers() && foundItems == 0 && directory != "" {
 		// Determine whether the directory exists or not by whether it has a marker
-		_, err := f.readMetaData(ctx, containerName, directory)
+		props, err := f.readMetaData(ctx, containerName, directory)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				return fs.ErrorDirNotFound
 			}
 			return err
 		}
+		if f.hns && !isHNSDirectory(props) {
+			return fs.ErrorDirNotFound
+		}
 	}
 	return nil
 }
@@ -1238,25 +1273,11 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 // listBlobsPager runs the hierarchy listing described by opts, calling fn for
 // each blob and subdirectory, and returns the number of raw items seen.
 // delimiter selects flat (recurse) vs hierarchical listing. If opts requests
-// the Apache Arrow format the listing goes through the arrowlist pager
-// (falling back to the SDK pager if the credentials don't support it); if the
-// service then answers with XML (Arrow listing not enabled) a debug message
-// is logged.
-func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
-	useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
-	var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
-	if useArrow {
-		arrowSVC, err := f.arrowCntSVC(containerName)
-		if err != nil {
-			fs.Debugf(f, "Not using Arrow listing: %v", err)
-			useArrow = false
-		} else {
-			pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
-		}
-	}
-	if pager == nil {
-		pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
-	}
+// the Apache Arrow format and the service answers with XML (Arrow listing
+// not enabled) a debug message is logged.
+func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
+	useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
+	pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
 	checkedArrow := false
 	for pager.More() {
 		var response container.ListBlobsHierarchyResponse
@@ -1272,7 +1293,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
 		// but not accelerated.
 		if useArrow && !checkedArrow && err == nil {
 			checkedArrow = true
-			if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
+			if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
 				fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
 			}
 		}
@@ -1297,6 +1318,13 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
 				fs.Debugf(f, "Nil name received")
 				continue
 			}
+			// Storage accounts with a hierarchical namespace return
+			// the file "dir" when listing "dir/"
+			if !strings.HasPrefix(*file.Name, directory) {
+				fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
+				foundItems--
+				continue
+			}
 			remote := f.opt.Enc.ToStandardPath(*file.Name)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Debugf(f, "Odd name received %q", remote)
@@ -1558,7 +1586,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
 func (f *Fs) listContainersToFn(fn listContainerFn) error {
 	max := int32(f.opt.ListChunkSize)
 	pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
-		Include:    service.ListContainersInclude{Metadata: true, Deleted: true},
+		Include:    service.ListContainersInclude{Metadata: true},
 		MaxResults: &max,
 	})
 	ctx := context.Background()
@@ -1608,7 +1636,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // Create directory marker file and parents
 func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
-	if !f.opt.DirectoryMarkers || container == "" {
+	if !f.dirMarkers() || container == "" {
 		return nil
 	}
 
@@ -1665,6 +1693,11 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
+	if f.hns {
+		// Parent directories are created by the upload
+		container, _ := f.split(remote)
+		return f.makeContainer(ctx, container)
+	}
 	remote, _ = strings.CutSuffix(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
@@ -1770,6 +1803,9 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	container, directory := f.split(dir)
+	if f.hns && container != "" && directory != "" {
+		return f.rmdirHNS(ctx, container, directory)
+	}
 	// Remove directory marker file
 	if f.opt.DirectoryMarkers && container != "" && directory != "" {
 		o := &Object{
@@ -1792,6 +1828,39 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return f.deleteContainer(ctx, container)
 }
 
+// rmdirHNS removes the directory on a storage account with a
+// hierarchical namespace
+//
+// The directory must be deleted without a trailing "/" and the server
+// refuses to delete it if it isn't empty.
+func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
+	directory, _ = strings.CutSuffix(directory, "/")
+	// Check it is a directory as deleting a file would succeed
+	props, err := f.readMetaData(ctx, container, directory)
+	if err == fs.ErrorObjectNotFound {
+		return fs.ErrorDirNotFound
+	} else if err != nil {
+		return err
+	}
+	if !isHNSDirectory(props) {
+		return fs.ErrorIsFile
+	}
+	blb := f.getBlobSVC(container, directory)
+	err = f.pacer.Call(func() (bool, error) {
+		_, err := blb.Delete(ctx, nil)
+		return f.shouldRetry(ctx, err)
+	})
+	if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
+		if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
+			return fs.ErrorDirectoryNotEmpty
+		}
+		if storageErr.StatusCode == http.StatusNotFound {
+			return fs.ErrorDirNotFound
+		}
+	}
+	return err
+}
+
 // Precision of the remote
 func (f *Fs) Precision() time.Duration {
 	return time.Nanosecond
@@ -1827,7 +1896,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
 	}
 
 	// Validity window
-	start := time.Now().UTC()
+	start := time.Now().UTC().Add(-sasCopyStartSkew)
 	expiry := start.Add(2 * sasCopyValidity)
 	startStr := start.Format(time.RFC3339)
 	expiryStr := expiry.Format(time.RFC3339)
@@ -1876,7 +1945,7 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
 		// Build the SAS values
 		perms := sas.BlobPermissions{Read: true}
 		container, containerPath := o.split()
-		start := time.Now().UTC()
+		start := time.Now().UTC().Add(-sasCopyStartSkew)
 		expiry := start.Add(sasCopyValidity)
 		vals := sas.BlobSignatureValues{
 			StartTime:     start,
@@ -2346,6 +2415,16 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
 	} else {
 		size = *info.ContentLength
 	}
+	// On a range request Content-Length is the length of the range, not the object, so take the
+	// object's size from the total in Content-Range instead.
+	if info.ContentRange != nil {
+		contentRange, err := rest.ParseContentRange(*info.ContentRange)
+		if err != nil {
+			fs.Debugf(o, "Failed to parse Content-Range %q: %v", *info.ContentRange, err)
+		} else if contentRange.Size >= 0 {
+			size = contentRange.Size
+		}
+	}
 	if isDirectoryMarker(size, metadata, o.remote) {
 		return fs.ErrorNotAFile
 	}
@@ -2379,22 +2458,6 @@ func (o *Object) decodeMetaDataFromDownloadResponse(info *blob.DownloadStreamRes
 	// 	o.accessTier = blob.AccessTier(*info.AccessTier)
 	// }
 	o.setMetadata(metadata)
-
-	// If it was a Range request, the size is wrong, so correct it
-	if info.ContentRange != nil {
-		contentRange := *info.ContentRange
-		slash := strings.IndexRune(contentRange, '/')
-		if slash >= 0 {
-			i, err := strconv.ParseInt(contentRange[slash+1:], 10, 64)
-			if err == nil {
-				o.size = i
-			} else {
-				fs.Debugf(o, "Failed to find parse integer from in %q: %v", contentRange, err)
-			}
-		} else {
-			fs.Debugf(o, "Failed to find length in %q", contentRange)
-		}
-	}
 	o.contentEncoding = info.ContentEncoding
 
 	// If decompressing then size and md5sum are unknown
@@ -2636,11 +2699,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return downloadResponse.Body, nil
 }
 
-// Converts a string into a pointer to a string
-func pString(s string) *string {
-	return &s
-}
-
 // readSeekCloser joins an io.Reader and an io.Seeker and provides a no-op io.Closer
 type readSeekCloser struct {
 	io.Reader
@@ -2795,8 +2853,7 @@ func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectIn
 // isInvalidBlockOrBlob looks for the InvalidBlockOrBlob error in err
 // returning true if it is found
 func isInvalidBlockOrBlob(err error) bool {
-	var storageErr *azcore.ResponseError
-	if errors.As(err, &storageErr) {
+	if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
 		return storageErr.ErrorCode == string(bloberror.InvalidBlobOrBlock)
 	}
 	return false
@@ -3168,7 +3225,7 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 
 	// Start with default content-type based on source
 	ui.httpHeaders = blob.HTTPHeaders{
-		BlobContentType: pString(fs.MimeType(ctx, src)),
+		BlobContentType: new(fs.MimeType(ctx, src)),
 	}
 
 	// Apply mapped metadata/headers/tags if requested
@@ -3214,15 +3271,15 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 				o.tags[parts[0]] = parts[1]
 			}
 		case "cache-control":
-			ui.httpHeaders.BlobCacheControl = pString(value)
+			ui.httpHeaders.BlobCacheControl = new(value)
 		case "content-disposition":
-			ui.httpHeaders.BlobContentDisposition = pString(value)
+			ui.httpHeaders.BlobContentDisposition = new(value)
 		case "content-encoding":
-			ui.httpHeaders.BlobContentEncoding = pString(value)
+			ui.httpHeaders.BlobContentEncoding = new(value)
 		case "content-language":
-			ui.httpHeaders.BlobContentLanguage = pString(value)
+			ui.httpHeaders.BlobContentLanguage = new(value)
 		case "content-type":
-			ui.httpHeaders.BlobContentType = pString(value)
+			ui.httpHeaders.BlobContentType = new(value)
 		}
 	}
 

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !js
 
 package azureblob
 
@@ -54,6 +54,58 @@ func TestBlockIDCreator(t *testing.T) {
 	assert.ErrorContains(t, bic2.checkID(chunkNumber, got), "random bytes")
 }
 
+func TestDecodeMetaDataFromDownloadResponse(t *testing.T) {
+	pInt64 := func(i int64) *int64 { return &i }
+	pString := func(s string) *string { return &s }
+	newTestObject := func() *Object {
+		return &Object{
+			fs:     &Fs{},
+			remote: "test.bin",
+			size:   -2, // sentinel to check it gets set
+		}
+	}
+
+	t.Run("WholeBlob", func(t *testing.T) {
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		info.ContentLength = pInt64(12345)
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(12345), o.size)
+	})
+
+	t.Run("RangeRequest", func(t *testing.T) {
+		// On a ranged download Content-Length is the chunk length so the
+		// size must come from the total in Content-Range
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		info.ContentLength = pInt64(67108864)
+		info.ContentRange = pString("bytes 67108864-134217727/169721004032")
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(169721004032), o.size)
+	})
+
+	t.Run("BadContentRange", func(t *testing.T) {
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		info.ContentLength = pInt64(67108864)
+		info.ContentRange = pString("potato")
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(67108864), o.size)
+	})
+
+	t.Run("UnknownLength", func(t *testing.T) {
+		o := newTestObject()
+		info := blob.DownloadStreamResponse{}
+		require.NoError(t, o.decodeMetaDataFromDownloadResponse(&info))
+		assert.Equal(t, int64(-1), o.size)
+	})
+}
+
+func TestCopySASTimingConstants(t *testing.T) {
+	require.Greater(t, sasCopyStartSkew, time.Duration(0))
+	require.Greater(t, sasCopyValidity, sasCopyStartSkew)
+}
+
 func (f *Fs) testFeatures(t *testing.T) {
 	// Check first feature flags are set on this remote
 	enabled := f.Features().SetTier
@@ -273,6 +325,14 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
 }
 
 // helper to read blob tags for an object
+// skipIfHNS skips the test if the storage account has a hierarchical
+// namespace as these don't support blob tags
+func skipIfHNS(t *testing.T, f *Fs) {
+	if f.hns {
+		t.Skip("blob tags not supported with hierarchical namespace")
+	}
+}
+
 func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
 	ao := o.(*Object)
 	blb := ao.getBlobSVC()
@@ -358,6 +418,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 		// Tags: Singlepart upload
 		t.Run("PutSinglepartTags", func(t *testing.T) {
+			skipIfHNS(t, f)
 			contents := random.String(int(f.opt.ChunkSize / 2))
 			item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 			meta := fs.Metadata{
@@ -373,6 +434,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 		// Tags: Multipart upload
 		t.Run("PutMultipartTags", func(t *testing.T) {
+			skipIfHNS(t, f)
 			contents := random.String(int(f.opt.ChunkSize + 2048))
 			item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 			meta := fs.Metadata{
@@ -560,6 +622,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 	// Tags: Singlepart copy
 	t.Run("CopySinglepartTags", func(t *testing.T) {
+		skipIfHNS(t, f)
 		// create small source
 		contents := random.String(int(f.opt.ChunkSize / 2))
 		srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
@@ -585,6 +648,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 	// Tags: Multipart copy
 	t.Run("CopyMultipartTags", func(t *testing.T) {
+		skipIfHNS(t, f)
 		// create large source to force multipart
 		contents := random.String(int(f.opt.CopyCutoff + 4096))
 		srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js
+//go:build !plan9 && !js
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,7 +1,7 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js
+//go:build plan9 || js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob

backend/azurefiles/azurefiles.go

@@ -650,11 +650,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 	return resp.Body, nil
 }
 
-// Returns a pointer to t - useful for returning pointers to constants
-func ptr[T any](t T) *T {
-	return &t
-}
-
 var warnStreamUpload sync.Once
 
 // Update the object with the contents of the io.Reader, modTime, size and MD5 hash
@@ -696,11 +691,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	// Measure the size if it is unknown
-	if sizeUnknown {
-		counter = readers.NewCountingReader(in)
-		in = counter
-	}
+	// Count the bytes read so short reads from the source can be detected
+	counter = readers.NewCountingReader(in)
+	in = counter
 
 	// Check we have a source MD5 hash...
 	if hashStr, err := src.Hash(ctx, hash.MD5); err == nil && hashStr != "" {
@@ -735,6 +728,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if sizeUnknown {
 		// Read the uploaded size - the file will be truncated to that size by updateSizeHashModTime
 		size = int64(counter.BytesRead())
+	} else if bytesRead := int64(counter.BytesRead()); bytesRead != size {
+		// The file was created at the declared size so a short source
+		// leaves the remainder padded with zeroes
+		if isNewlyCreated {
+			if _, delErr := fc.Delete(ctx, nil); delErr != nil {
+				fs.Errorf(o, "failed to delete partially uploaded file: %v", delErr)
+			}
+		} else if bytesRead < size {
+			// Truncate the file to the bytes actually received so it
+			// is not left padded to the declared size with zeroes
+			if _, resizeErr := fc.Resize(ctx, bytesRead, nil); resizeErr != nil {
+				fs.Errorf(o, "failed to truncate partially updated file: %v", resizeErr)
+			}
+		}
+		return fmt.Errorf("update: expected %d bytes in input, but got %d: %w", size, bytesRead, io.ErrUnexpectedEOF)
 	}
 	if hashUnknown {
 		md5Hash = hasher.Sum(nil)
@@ -805,8 +813,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fmt.Errorf("Move: mkParentDir failed: %w", err)
 	}
 	opt := file.RenameOptions{
-		IgnoreReadOnly:  ptr(true),
-		ReplaceIfExists: ptr(true),
+		IgnoreReadOnly:  new(true),
+		ReplaceIfExists: new(true),
 	}
 	dstAbsPath := f.absPath(remote)
 	fc := srcObj.fileClient()
@@ -851,8 +859,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 
 	opt := directory.RenameOptions{
-		IgnoreReadOnly:  ptr(false),
-		ReplaceIfExists: ptr(false),
+		IgnoreReadOnly:  new(false),
+		ReplaceIfExists: new(false),
 	}
 	dstAbsPath := dstFs.absPath(dstRemote)
 	dirClient := srcFs.dirClient(srcRemote)
@@ -891,8 +899,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			ChangeTime:         file.SourceCopyFileChangeTime{},
 			CreationTime:       file.SourceCopyFileCreationTime{},
 			LastWriteTime:      file.SourceCopyFileLastWriteTime{},
-			PermissionCopyMode: ptr(file.PermissionCopyModeTypeSource),
-			IgnoreReadOnly:     ptr(true),
+			PermissionCopyMode: new(file.PermissionCopyModeTypeSource),
+			IgnoreReadOnly:     new(true),
 		},
 	}
 	srcURL := srcObj.fileClient().URL()

backend/b2/b2.go

@@ -191,6 +191,11 @@ If the custom endpoint rewrites the requests for authentication,
 e.g., in Cloudflare Workers, this header needs to be handled properly.
 Leave blank if you want to use the endpoint provided by Backblaze.
 
+Rclone will not follow a redirect from this endpoint which downgrades
+HTTPS to HTTP, and will not send the "Authorization" header or the
+SSE-C key headers once a redirect has left the host and port of this
+endpoint.
+
 The URL provided here SHOULD have the protocol and SHOULD NOT have
 a trailing slash or specify the /file/bucket subpath as rclone will
 request files with "{download_url}/file/{bucket_name}/{path}".
@@ -529,6 +534,25 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
+// redirectSecretHeaders are the request headers carrying secrets that
+// must not be forwarded when a redirect leaves the original host.
+var redirectSecretHeaders = []string{
+	"Authorization",
+	sseAlgorithmHeader,
+	sseKeyHeader,
+	sseMd5Header,
+}
+
+// newClient makes the http client used for all B2 requests.
+//
+// The download_url option sends the requests to a host the user does
+// not control so it strips headers on cross host redirects.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
+	return client
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -581,7 +605,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:        name,
 		opt:         *opt,
 		ci:          ci,
-		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
+		srv:         rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
 		cache:       bucket.NewCache(),
 		_bucketID:   make(map[string]string, 1),
 		_bucketType: make(map[string]string, 1),

backend/b2/b2_internal_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"crypto/sha1"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"path"
 	"sort"
 	"strings"
@@ -617,3 +619,60 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// redirectTestHeaders is a deliberately literal copy of
+// redirectSecretHeaders so that a header dropped from the production
+// list fails TestRedirectSecretHeaders rather than silently losing
+// test coverage.
+var redirectTestHeaders = map[string]string{
+	"Authorization": "secret-token",
+	"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
+	"X-Bz-Server-Side-Encryption-Customer-Key":       "secret-key",
+	"X-Bz-Server-Side-Encryption-Customer-Key-Md5":   "secret-md5",
+}
+
+func TestRedirectSecretHeaders(t *testing.T) {
+	want := make([]string, 0, len(redirectTestHeaders))
+	for header := range redirectTestHeaders {
+		want = append(want, header)
+	}
+	sort.Strings(want)
+	got := append([]string(nil), redirectSecretHeaders...)
+	sort.Strings(got)
+	assert.Equal(t, want, got)
+}
+
+// redirectTestRequest makes a GET request to rawURL carrying the
+// secret headers plus a harmless marker header.
+func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
+	require.NoError(t, err)
+	for header, value := range redirectTestHeaders {
+		req.Header.Set(header, value)
+	}
+	req.Header.Set("X-Bz-Test-Mode", "marker")
+	return req
+}
+
+func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
+	client := newClient(context.Background())
+
+	redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for header := range redirectTestHeaders {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+		assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer redirectServer.Close()
+
+	initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
+	}))
+	defer initialServer.Close()
+
+	resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+}

backend/box/box.go

@@ -45,6 +45,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/youmark/pkcs8"
 )
@@ -1460,7 +1461,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 					// Item in the cache has the same or newer SequenceID than
 					// this event. Ignore this event, it must be old.
 					f.itemMetaCacheMu.Unlock()
-					fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
+					fs.Debugf(f, "%s ignored due to old SequenceID (%d)", eventDetails, itemMeta.SequenceID)
 					continue
 				}
 
@@ -1691,7 +1692,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // upload does a single non-multipart upload
 //
 // This is recommended for less than 50 MiB of content
-func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
+func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
 	upload := api.UploadFile{
 		Name:              o.fs.opt.Enc.FromStandardName(leaf),
 		ContentModifiedAt: api.Time(modTime),
@@ -1703,9 +1704,10 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 
 	var resp *http.Response
 	var result api.FolderItems
+	counter := readers.NewCountingReader(in)
 	opts := rest.Opts{
 		Method:                "POST",
-		Body:                  in,
+		Body:                  counter,
 		MultipartMetadataName: "attributes",
 		MultipartContentName:  "contents",
 		MultipartFileName:     upload.Name,
@@ -1725,6 +1727,11 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 	if err != nil {
 		return err
 	}
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload.
+	if size >= 0 && int64(counter.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
 	if result.TotalCount != 1 || len(result.Entries) != 1 {
 		return fmt.Errorf("failed to upload %v - not sure why", o)
 	}
@@ -1754,7 +1761,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Upload with simple or multipart
 	if size <= int64(o.fs.opt.UploadCutoff) {
-		err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
+		err = o.upload(ctx, in, leaf, directoryID, size, modTime, options...)
 	} else {
 		err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
 	}

backend/box/upload.go

@@ -3,7 +3,6 @@
 package box
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha1"
 	"encoding/base64"
@@ -20,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -55,9 +55,15 @@ func sha1Digest(digest []byte) string {
 }
 
 // uploadPart uploads a part in an upload session
-func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
-	chunkSize := int64(len(chunk))
-	sha1sum := sha1.Sum(chunk)
+//
+// chunk must contain exactly the part's data and is read from the start
+// on each attempt.
+func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk io.ReadSeeker, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
+	hasher := sha1.New()
+	chunkSize, err := io.Copy(hasher, chunk)
+	if err != nil {
+		return nil, err
+	}
 	opts := rest.Opts{
 		Method:        "PUT",
 		Path:          "/files/upload_sessions/" + SessionID,
@@ -67,12 +73,16 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
 		Options:       options,
 		ExtraHeaders: map[string]string{
-			"Digest": sha1Digest(sha1sum[:]),
+			"Digest": sha1Digest(hasher.Sum(nil)),
 		},
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		opts.Body = wrap(bytes.NewReader(chunk))
+		_, err = chunk.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		opts.Body = wrap(chunk)
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
 		return shouldRetry(ctx, resp, err)
 	})
@@ -213,27 +223,25 @@ outer:
 
 		reqSize := min(remaining, chunkSize)
 
-		// Make a block of memory
-		buf := make([]byte, reqSize)
-
-		// Read the chunk
-		_, err = io.ReadFull(in, buf)
+		// Read the chunk into memory from the global pool, hashing
+		// it for the whole file hash (must be done sequentially)
+		rw := multipart.NewRW().Reserve(reqSize)
+		_, err = io.CopyN(rw, io.TeeReader(in, hash), reqSize)
 		if err != nil {
+			_ = rw.Close()
 			err = fmt.Errorf("multipart upload failed to read source: %w", err)
 			break outer
 		}
 
-		// Make the global hash (must be done sequentially)
-		_, _ = hash.Write(buf)
-
 		// Transfer the chunk
 		wg.Add(1)
 		o.fs.uploadToken.Get()
 		go func(part int, position int64) {
 			defer wg.Done()
 			defer o.fs.uploadToken.Put()
+			defer func() { _ = rw.Close() }()
 			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
-			partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
+			partResponse, err := o.uploadPart(ctx, session.ID, position, size, rw, wrap, options...)
 			if err != nil {
 				err = fmt.Errorf("multipart upload failed to upload part: %w", err)
 				select {

backend/chunker/chunker.go

@@ -801,9 +801,12 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 				fs.Debugf(f, "skip orphan data chunk %q", remote)
 				break
 			}
-			if mainObject == nil && !f.useMeta {
+			if mainObject == nil && !f.useMeta && !isSpecial {
 				// this is the "nometa" case
 				// create dummy chunked object without metadata
+				//
+				// not for temporary or control chunks as they may be all
+				// that is left of an interrupted upload
 				mainObject = f.newObject(mainRemote, nil, nil)
 				byRemote[mainRemote] = mainObject
 				if !badEntry[mainRemote] {

backend/chunker/chunker_internal_test.go

@@ -43,7 +43,7 @@ func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
 type settings map[string]any
 
 func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, path string, opts settings) fs.Fs {
-	fsName := strings.Split(f.Name(), "{")[0] // strip off hash
+	fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
 	configMap := configmap.Simple{}
 	for key, val := range opts {
 		configMap[key] = fmt.Sprintf("%v", val)
@@ -627,6 +627,22 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	_ = file.Remove(ctx)
 }
 
+// testOrphanTempChunks checks that temporary chunks left behind by an
+// interrupted upload are not listed and don't break the listing
+func testOrphanTempChunks(t *testing.T, f *Fs) {
+	const dir = "orphan"
+	ctx := context.Background()
+	defer func() {
+		_ = operations.Purge(ctx, f.base, dir)
+	}()
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
+	fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
+
+	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
+}
+
 func testMetadataInput(t *testing.T, f *Fs) {
 	const minChunkForTest = 50
 	if f.opt.ChunkSize < minChunkForTest {
@@ -928,6 +944,9 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("BackwardsCompatibility", func(t *testing.T) {
 		testBackwardsCompatibility(t, f)
 	})
+	t.Run("OrphanTempChunks", func(t *testing.T) {
+		testOrphanTempChunks(t, f)
+	})
 	t.Run("ChunkerServerSideMove", func(t *testing.T) {
 		testChunkerServerSideMove(t, f)
 	})

backend/cloudinary/cloudinary.go

@@ -472,8 +472,8 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		if updateOptions, ok := option.(*api.UpdateOptions); ok {
 			if updateOptions.PublicID != "" {
 				updateObject = true
-				params.Overwrite = SDKApi.Bool(true)
-				params.Invalidate = SDKApi.Bool(true)
+				params.Overwrite = new(true)
+				params.Invalidate = new(true)
 				params.PublicID = updateOptions.PublicID
 				params.ResourceType = updateOptions.ResourceType
 				params.Type = SDKApi.DeliveryType(updateOptions.DeliveryType)

backend/compress/compress.go

@@ -31,6 +31,7 @@ import (
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/lib/multipart"
 )
 
 // Globals
@@ -563,15 +564,19 @@ type compressionResult[T sgzip.GzipMetadata | SzstdMetadata] struct {
 func (f *Fs) rcat(ctx context.Context, dstFileName string, in io.ReadCloser, modTime time.Time, options []fs.OpenOption) (o fs.Object, err error) {
 
 	// cache small files in memory and do normal upload
-	buf := make([]byte, f.opt.RAMCacheLimit)
-	if n, err := io.ReadFull(in, buf); err == io.EOF || err == io.ErrUnexpectedEOF {
-		src := object.NewStaticObjectInfo(dstFileName, modTime, int64(len(buf[:n])), false, nil, f.Fs)
-		return f.Fs.Put(ctx, bytes.NewBuffer(buf[:n]), src, options...)
+	rw := multipart.NewRW()
+	defer fs.CheckClose(rw, &err)
+	_, err = io.CopyN(rw, in, int64(f.opt.RAMCacheLimit))
+	if err == io.EOF {
+		src := object.NewStaticObjectInfo(dstFileName, modTime, rw.Size(), false, nil, f.Fs)
+		return f.Fs.Put(ctx, rw, src, options...)
+	} else if err != nil {
+		return nil, err
 	}
 
 	// Need to include what we already read
 	in = &ReadCloserWrapper{
-		Reader: io.MultiReader(bytes.NewReader(buf), in),
+		Reader: io.MultiReader(rw, in),
 		Closer: in,
 	}
 
@@ -698,6 +703,16 @@ func (f *Fs) putWithCustomFunctions(ctx context.Context, in io.Reader, src fs.Ob
 		return nil, err
 	}
 
+	// Check the source supplied the number of bytes it declared -
+	// compressed data is stored under a name containing the
+	// original size so a mismatch makes the object unreadable.
+	if size := src.Size(); size >= 0 && meta.Size != size {
+		if removeErr := dataObject.Remove(ctx); removeErr != nil {
+			fs.Errorf(dataObject, "Failed to remove partially transferred object: %v", removeErr)
+		}
+		return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, meta.Size, io.ErrUnexpectedEOF)
+	}
+
 	mo, err := f.putMetadata(ctx, meta, src, options, putMeta)
 
 	// meta data upload may fail. in this case we try to remove the original object
@@ -1070,6 +1085,35 @@ type ObjectMetadata struct {
 	CompressionMetadataZstd *SzstdMetadata      // Metadata for Zstd compression
 }
 
+// validate checks metadata read from the wrapped remote before it is used to
+// construct a compressed reader.
+func (meta *ObjectMetadata) validate() error {
+	if meta.Mode != Gzip {
+		return nil
+	}
+	if meta.CompressionMetadataGzip == nil {
+		return errors.New("missing gzip metadata")
+	}
+	if meta.CompressionMetadataGzip.BlockSize <= 0 {
+		return fmt.Errorf("invalid gzip block size %d", meta.CompressionMetadataGzip.BlockSize)
+	}
+	if meta.Size != meta.CompressionMetadataGzip.Size {
+		return errors.New("gzip metadata size does not match object size")
+	}
+	if meta.Size < 0 {
+		return errors.New("invalid gzip object size")
+	}
+	blockSize := int64(meta.CompressionMetadataGzip.BlockSize)
+	blocks := meta.Size / blockSize
+	if meta.Size%blockSize != 0 {
+		blocks++
+	}
+	if int64(len(meta.CompressionMetadataGzip.BlockData)) < blocks {
+		return errors.New("gzip block data is incomplete")
+	}
+	return nil
+}
+
 // Object with external metadata
 type Object struct {
 	fs.Object                 // Wraps around data object for this object
@@ -1093,6 +1137,9 @@ func readMetadata(ctx context.Context, mo fs.Object) (meta *ObjectMetadata, err
 	if err = jr.Decode(meta); err != nil {
 		return nil, err
 	}
+	if err = meta.validate(); err != nil {
+		return nil, err
+	}
 	return meta, nil
 }
 
@@ -1110,6 +1157,15 @@ func (o *Object) Remove(ctx context.Context) error {
 	return objErr
 }
 
+// countingDiscard is an io.Writer which discards its input and counts
+// the bytes written
+type countingDiscard int64
+
+func (c *countingDiscard) Write(p []byte) (int, error) {
+	*c += countingDiscard(len(p))
+	return len(p), nil
+}
+
 // ReadCloserWrapper combines a Reader and a Closer to a ReadCloser
 type ReadCloserWrapper struct {
 	io.Reader

backend/compress/compress_test.go

@@ -6,6 +6,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/buengese/sgzip"
+
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/local"
 	_ "github.com/rclone/rclone/backend/s3"
@@ -14,6 +16,93 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
+func TestObjectMetadataValidate(t *testing.T) {
+	tests := []struct {
+		name string
+		meta ObjectMetadata
+		want bool
+	}{
+		{
+			name: "nil gzip metadata",
+			meta: ObjectMetadata{Mode: Gzip},
+			want: true,
+		},
+		{
+			name: "zero block size",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1024,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 0,
+					Size:      1024,
+					BlockData: []uint32{0},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "negative block size",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1024,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: -1,
+					Size:      1024,
+					BlockData: []uint32{0},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "mismatched sizes",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1024,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 512,
+					Size:      2048,
+					BlockData: []uint32{0, 1},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "short block data",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1025,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 512,
+					Size:      1025,
+					BlockData: []uint32{0, 1},
+				},
+			},
+			want: true,
+		},
+		{
+			name: "valid",
+			meta: ObjectMetadata{
+				Mode: Gzip,
+				Size: 1025,
+				CompressionMetadataGzip: &sgzip.GzipMetadata{
+					BlockSize: 512,
+					Size:      1025,
+					BlockData: []uint32{0, 1, 2},
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			err := test.meta.validate()
+			if (err != nil) != test.want {
+				t.Fatalf("validate() error = %v, want error: %v", err, test.want)
+			}
+		})
+	}
+}
+
 var defaultOpt = fstests.Opt{
 	RemoteName: "TestCompress:",
 	NilObject:  (*Object)(nil),

backend/compress/gzip_handler.go

@@ -2,7 +2,6 @@ package compress
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
@@ -23,7 +22,7 @@ type gzipModeHandler struct{}
 // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
 // the configured threshold
 func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
-	var b bytes.Buffer
+	var b countingDiscard
 	var n int64
 	w, err := sgzip.NewWriterLevel(&b, sgzip.DefaultCompression)
 	if err != nil {
@@ -37,7 +36,7 @@ func (g *gzipModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
 	if err != nil {
 		return false, err
 	}
-	ratio := float64(n) / float64(b.Len())
+	ratio := float64(n) / float64(b)
 	return ratio > minCompressionRatio, nil
 }
 

backend/compress/rcat_internal_test.go

@@ -0,0 +1,137 @@
+package compress
+
+import (
+	"bytes"
+	"context"
+	"crypto/rand"
+	"io"
+	"testing"
+	"time"
+
+	_ "github.com/rclone/rclone/backend/memory"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// putRecorder wraps an fs.Fs and records what rcat hands to Put and
+// PutStream. It reads each body twice when it is seekable to model a
+// backend retrying an upload.
+type putRecorder struct {
+	fs.Fs
+	t        *testing.T
+	stream   bool // expose PutStream from the wrapped Fs
+	size     int64
+	seekable bool
+	body     []byte
+	streamed bool
+}
+
+func (r *putRecorder) Features() *fs.Features {
+	features := *r.Fs.Features()
+	features.PutStream = nil
+	if r.stream {
+		features.PutStream = r.PutStream
+	}
+	return &features
+}
+
+func (r *putRecorder) record(in io.Reader, src fs.ObjectInfo) []byte {
+	r.size = src.Size()
+	body, err := io.ReadAll(in)
+	require.NoError(r.t, err)
+	if seeker, ok := in.(io.Seeker); ok {
+		r.seekable = true
+		_, err = seeker.Seek(0, io.SeekStart)
+		require.NoError(r.t, err)
+		again, err := io.ReadAll(in)
+		require.NoError(r.t, err)
+		assert.Equal(r.t, body, again, "retried body should be identical")
+	}
+	r.body = body
+	return body
+}
+
+func (r *putRecorder) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	body := r.record(in, src)
+	return r.Fs.Put(ctx, bytes.NewReader(body), object.NewStaticObjectInfo(src.Remote(), src.ModTime(ctx), int64(len(body)), true, nil, r.Fs), options...)
+}
+
+func (r *putRecorder) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	r.streamed = true
+	return r.Put(ctx, in, src, options...)
+}
+
+// TestRcat checks the three upload paths rcat takes: small files buffered
+// in memory and uploaded with a known size and a seekable body, large
+// files streamed when the wrapped remote supports it, and large files
+// spooled to a temporary file when it doesn't. Pool pages taken for the
+// buffer must be returned on every path.
+func TestRcat(t *testing.T) {
+	ctx := context.Background()
+	const limit = 1024
+	for _, test := range []struct {
+		name     string
+		size     int
+		stream   bool
+		wantSize int64
+	}{
+		{name: "Small", size: limit / 2, stream: true, wantSize: limit / 2},
+		{name: "Empty", size: 0, stream: true, wantSize: 0},
+		{name: "ExactLimit", size: limit, stream: true, wantSize: -1},
+		{name: "LargeStream", size: 3 * limit, stream: true, wantSize: -1},
+		{name: "LargeSpool", size: 3 * limit, stream: false, wantSize: 3 * limit},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			remote, err := fs.NewFs(ctx, ":memory:rcat-"+test.name)
+			require.NoError(t, err)
+			rec := &putRecorder{Fs: remote, t: t, stream: test.stream}
+			f := &Fs{Fs: rec}
+			f.opt.RAMCacheLimit = limit
+			content := []byte(random.String(test.size))
+			inUse := pool.Global().InUse()
+
+			o, err := f.rcat(ctx, "file", io.NopCloser(bytes.NewReader(content)), time.Now(), nil)
+			require.NoError(t, err)
+			assert.Equal(t, inUse, pool.Global().InUse(), "pool pages not returned")
+
+			assert.Equal(t, test.wantSize, rec.size)
+			assert.Equal(t, content, rec.body)
+			assert.Equal(t, test.stream && test.size >= limit, rec.streamed)
+			if test.size < limit {
+				assert.True(t, rec.seekable, "small file body should be seekable")
+			}
+			rc, err := o.Open(ctx)
+			require.NoError(t, err)
+			got, err := io.ReadAll(rc)
+			require.NoError(t, err)
+			require.NoError(t, rc.Close())
+			assert.Equal(t, content, got)
+		})
+	}
+}
+
+// TestIsCompressible checks the compressibility heuristic on data at
+// both ends of the scale for every compression mode which has one.
+func TestIsCompressible(t *testing.T) {
+	compressible := bytes.Repeat([]byte("compress me "), 4096)
+	incompressible := make([]byte, len(compressible))
+	_, err := io.ReadFull(rand.Reader, incompressible)
+	require.NoError(t, err)
+	for name, handler := range map[string]compressionModeHandler{
+		"gzip": &gzipModeHandler{},
+		"zstd": &zstdModeHandler{},
+	} {
+		t.Run(name, func(t *testing.T) {
+			ok, err := handler.isCompressible(bytes.NewReader(compressible), 0)
+			require.NoError(t, err)
+			assert.True(t, ok)
+			ok, err = handler.isCompressible(bytes.NewReader(incompressible), 0)
+			require.NoError(t, err)
+			assert.False(t, ok)
+		})
+	}
+}

backend/compress/zstd_handler.go

@@ -2,7 +2,6 @@ package compress
 
 import (
 	"bufio"
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/hex"
@@ -23,7 +22,7 @@ type zstdModeHandler struct{}
 // isCompressible checks the compression ratio of the provided data and returns true if the ratio exceeds
 // the configured threshold
 func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool, error) {
-	var b bytes.Buffer
+	var b countingDiscard
 	var n int64
 	w, err := NewWriterSzstd(&b, zstd.WithEncoderLevel(zstd.SpeedDefault))
 	if err != nil {
@@ -37,7 +36,7 @@ func (z *zstdModeHandler) isCompressible(r io.Reader, compressionMode int) (bool
 	if err != nil {
 		return false, err
 	}
-	ratio := float64(n) / float64(b.Len())
+	ratio := float64(n) / float64(b)
 	return ratio > minCompressionRatio, nil
 }
 

backend/crypt/cipher.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 	"strconv"
 	"strings"
 	"sync"
@@ -117,8 +118,8 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-// fileNameEncoding are the encoding methods dealing with encrypted file names
-type fileNameEncoding interface {
+// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
+type FileNameEncoding interface {
 	EncodeToString(src []byte) string
 	DecodeString(s string) ([]byte, error)
 }
@@ -153,7 +154,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
 }
 
 // NewNameEncoding creates a NameEncoding from a string
-func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
+func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
 	s = strings.ToLower(s)
 	switch s {
 	case "base32":
@@ -175,7 +176,7 @@ type Cipher struct {
 	nameTweak       [nameCipherBlockSize]byte // used to tweak the name crypto
 	block           gocipher.Block
 	mode            NameEncryptionMode
-	fileNameEnc     fileNameEncoding
+	fileNameEnc     FileNameEncoding
 	buffers         sync.Pool // encrypt/decrypt buffers
 	cryptoRand      io.Reader // read crypto random numbers from here
 	dirNameEncrypt  bool
@@ -184,7 +185,7 @@ type Cipher struct {
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
 	c := &Cipher{
 		mode:            mode,
 		fileNameEnc:     enc,
@@ -486,7 +487,12 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 }
 
 // encryptFileName encrypts a file path
-func (c *Cipher) encryptFileName(in string) string {
+//
+// If stripVersion is set, a version string (as used by --b2-versions)
+// on the last segment is kept in plain text. Set it for file names
+// only: directory names must encrypt identically whether passed alone
+// or as the parent of a file name.
+func (c *Cipher) encryptFileName(in string, stripVersion bool) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		// Skip directory name encryption if the user chose to
@@ -499,7 +505,7 @@ func (c *Cipher) encryptFileName(in string) string {
 		// of the file name gets encrypted/obfuscated
 		hasVersion := false
 		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
+		if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
 			var s string
 			t, s = version.Remove(segments[i])
 			// version.Remove can fail, in which case it returns segments[i]
@@ -529,19 +535,24 @@ func (c *Cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
 		return in + c.encryptedSuffix
 	}
-	return c.encryptFileName(in)
+	return c.encryptFileName(in, true)
 }
 
 // EncryptDirName encrypts a directory path
+//
+// Unlike EncryptFileName, a version string on the last segment is
+// encrypted along with the rest of the name.
 func (c *Cipher) EncryptDirName(in string) string {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in
 	}
-	return c.encryptFileName(in)
+	return c.encryptFileName(in, false)
 }
 
 // decryptFileName decrypts a file path
-func (c *Cipher) decryptFileName(in string) (string, error) {
+//
+// stripVersion is as for encryptFileName.
+func (c *Cipher) decryptFileName(in string, stripVersion bool) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -555,7 +566,7 @@ func (c *Cipher) decryptFileName(in string) (string, error) {
 		// of the file name gets decrypted/deobfuscated
 		hasVersion := false
 		var t time.Time
-		if i == (len(segments)-1) && version.Match(segments[i]) {
+		if stripVersion && i == (len(segments)-1) && version.Match(segments[i]) {
 			var s string
 			t, s = version.Remove(segments[i])
 			// version.Remove can fail, in which case it returns segments[i]
@@ -601,15 +612,31 @@ func (c *Cipher) DecryptFileName(in string) (string, error) {
 		// Leave the version string on, if it was there
 		return decrypted, nil
 	}
-	return c.decryptFileName(in)
+	return c.decryptFileName(in, true)
 }
 
 // DecryptDirName decrypts a directory path
+//
+// Unlike DecryptFileName, a version string on the last segment is
+// expected to be part of the encrypted name. Directory names created
+// by rclone before v1.76 had a version-like suffix left in plain
+// text; these are still decrypted so that they appear in listings,
+// but they can't be opened or removed until renamed on the
+// underlying remote.
 func (c *Cipher) DecryptDirName(in string) (string, error) {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in, nil
 	}
-	return c.decryptFileName(in)
+	out, err := c.decryptFileName(in, false)
+	if err != nil && version.Match(path.Base(in)) {
+		var legacyErr error
+		out, legacyErr = c.decryptFileName(in, true)
+		if legacyErr == nil {
+			fs.Logf(nil, "crypt: directory %q has a legacy encrypted name - rename %q on the underlying remote to %q to make it accessible", out, in, c.encryptFileName(out, false))
+			return out, nil
+		}
+	}
+	return out, err
 }
 
 // NameEncryptionMode returns the encryption mode in use for names

backend/crypt/cipher_test.go

@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 	"strings"
 	"testing"
 
@@ -682,6 +683,43 @@ func TestNonStandardDecryptDirName(t *testing.T) {
 	}
 }
 
+// Test directories whose name looks like it has a version string -
+// these encrypt verbatim, so that EncryptDirName agrees with the same
+// directory encrypted as the parent of a file name, whereas file names
+// keep the version string in plain text
+func TestVersionedDirName(t *testing.T) {
+	const dir = "dir-v2001-02-03-040506-123"
+	enc, err := NewNameEncoding("base32")
+	require.NoError(t, err)
+	for _, mode := range []NameEncryptionMode{NameEncryptionStandard, NameEncryptionObfuscated} {
+		c, err := newCipher(mode, "", "", true, enc)
+		require.NoError(t, err)
+		what := fmt.Sprintf("Testing mode=%v", mode)
+		// Check EncryptDirName matches the parent of an encrypted file name
+		encryptedDir := c.EncryptDirName(dir)
+		encryptedFile := c.EncryptFileName(dir + "/file.txt")
+		assert.Equal(t, encryptedDir, path.Dir(encryptedFile), what)
+		// Check the encrypted directory name round trips OK
+		decryptedDir, err := c.DecryptDirName(encryptedDir)
+		assert.NoError(t, err, what)
+		assert.Equal(t, dir, decryptedDir, what)
+		// Check a file with the same name keeps its version string in plain text
+		encryptedFile = c.EncryptFileName(dir)
+		assert.NotEqual(t, encryptedDir, encryptedFile, what)
+		assert.Contains(t, encryptedFile, "-v2001-02-03-040506-123", what)
+		decryptedFile, err := c.DecryptFileName(encryptedFile)
+		assert.NoError(t, err, what)
+		assert.Equal(t, dir, decryptedFile, what)
+		// Check a directory with a legacy encrypted name (the file form) still decrypts
+		decryptedDir, err = c.DecryptDirName(encryptedFile)
+		assert.NoError(t, err, what)
+		assert.Equal(t, dir, decryptedDir, what)
+		// but a name which is bad in both forms is still an error
+		_, err = c.DecryptDirName("!!!-v2001-02-03-040506-123")
+		assert.Error(t, err, what)
+	}
+}
+
 func TestEncryptedSize(t *testing.T) {
 	c, _ := newCipher(NameEncryptionStandard, "", "", true, nil)
 	for _, test := range []struct {

backend/crypt/crypt.go

@@ -1179,6 +1179,11 @@ func (o *ObjectInfo) Size() int64 {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
+	// If the data is unchanged then the hash of the source is the
+	// hash of the object which will be uploaded
+	if o.f.opt.NoDataEncryption {
+		return o.ObjectInfo.Hash(ctx, hash)
+	}
 	var srcObj fs.Object
 	var ok bool
 	// Get the underlying object if there is one

backend/crypt/crypt_internal_test.go

@@ -84,6 +84,10 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 
 	// Test ObjectInfo.Hash
 	wantHash := md5.Sum(outBuf.Bytes())
+	if f.opt.NoDataEncryption {
+		// If the data isn't encrypted, the hash should be that of the plaintext
+		wantHash = md5.Sum([]byte(contents))
+	}
 	gotHash, err := src.Hash(ctx, hash.MD5)
 	require.NoError(t, err)
 	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)

backend/dosya/api/types.go

@@ -0,0 +1,259 @@
+// Package api has type definitions for dosya.dev
+package api
+
+import "fmt"
+
+// Response is the base response wrapper
+type Response struct {
+	OK    bool   `json:"ok"`
+	Error string `json:"error,omitempty"`
+}
+
+// Error is an error response from the API.
+//
+// StatusCode and Status are filled in from the HTTP response. The API
+// sets ErrorCode on the errors a client is expected to act on rather
+// than just report, e.g. "concurrent_upload_limit".
+type Error struct {
+	StatusCode           int    `json:"-"`
+	Status               string `json:"-"`
+	OK                   bool   `json:"ok"`
+	Message              string `json:"error"`
+	ErrorCode            string `json:"error_code,omitempty"`
+	MaxConcurrentUploads int    `json:"max_concurrent_uploads,omitempty"`
+}
+
+// Error satisfies the error interface
+func (e *Error) Error() string {
+	out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	if e.ErrorCode != "" {
+		out += " (" + e.ErrorCode + ")"
+	}
+	return out
+}
+
+// FileItem represents a file from the dosya API
+type FileItem struct {
+	ID        string  `json:"id"`
+	Name      string  `json:"name"`
+	SizeBytes int64   `json:"size_bytes"`
+	MimeType  string  `json:"mime_type"`
+	Extension string  `json:"extension"`
+	Region    string  `json:"region"`
+	CreatedAt float64 `json:"created_at"`
+	UpdatedAt float64 `json:"updated_at"`
+	FolderID  *string `json:"folder_id"`
+	IsSynced  int     `json:"is_synced"`
+	// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
+	// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
+	// are never re-read to hash the whole object - the same limitation S3 has).
+	ContentHash string `json:"content_hash"`
+}
+
+// FolderItem represents a folder from the dosya API
+type FolderItem struct {
+	ID        string  `json:"id"`
+	Name      string  `json:"name"`
+	ParentID  *string `json:"parent_id"`
+	CreatedAt float64 `json:"created_at"`
+	FileCount int     `json:"file_count"`
+	IsSynced  int     `json:"is_synced"`
+}
+
+// ListResponse is the response from GET /api/files
+type ListResponse struct {
+	Response
+	Files      []FileItem   `json:"files"`
+	Folders    []FolderItem `json:"folders"`
+	Pagination Pagination   `json:"pagination"`
+}
+
+// Pagination describes which page of files a ListResponse holds
+type Pagination struct {
+	Page       int `json:"page"`
+	PerPage    int `json:"per_page"`
+	TotalFiles int `json:"total_files"`
+	TotalPages int `json:"total_pages"`
+}
+
+// CreateFolderRequest is the request body for POST /api/folders
+type CreateFolderRequest struct {
+	WorkspaceID string  `json:"workspace_id"`
+	ParentID    *string `json:"parent_id"`
+	Name        string  `json:"name"`
+}
+
+// CreateFolderResponse is the response from POST /api/folders
+type CreateFolderResponse struct {
+	Response
+	Folder struct {
+		ID        string  `json:"id"`
+		Name      string  `json:"name"`
+		ParentID  *string `json:"parent_id"`
+		Workspace string  `json:"workspace_id"`
+	} `json:"folder"`
+}
+
+// DeleteFolderResponse is the response from DELETE /api/folders/:id
+type DeleteFolderResponse struct {
+	Response
+	Permanent      bool `json:"permanent"`
+	Complete       bool `json:"complete"`
+	Remaining      int  `json:"remaining"`
+	FilesAffected  int  `json:"files_affected"`
+	FoldersRemoved int  `json:"folders_removed"`
+}
+
+// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
+type RenameFolderRequest struct {
+	Name string `json:"name"`
+}
+
+// MoveFolderRequest is the request body for PUT /api/folders/:id/move
+type MoveFolderRequest struct {
+	ParentID *string `json:"parent_id"`
+	Name     string  `json:"name,omitempty"`
+}
+
+// MoveFolderResponse is the response from PUT /api/folders/:id/move
+type MoveFolderResponse struct {
+	Response
+	Name string `json:"name"`
+}
+
+// UploadInitRequest is the request body for POST /api/upload/init
+type UploadInitRequest struct {
+	WorkspaceID string  `json:"workspace_id"`
+	FileName    string  `json:"file_name"`
+	FileSize    int64   `json:"file_size"`
+	MimeType    string  `json:"mime_type"`
+	Region      string  `json:"region,omitempty"`
+	FolderID    *string `json:"folder_id"`
+	FileID      *string `json:"file_id,omitempty"`
+}
+
+// UploadInitResponse is the response from POST /api/upload/init
+type UploadInitResponse struct {
+	Response
+	SessionID   string `json:"session_id"`
+	UploadURL   string `json:"upload_url"`
+	WorkspaceID string `json:"workspace_id"`
+	FileName    string `json:"file_name"`
+	FileSize    int64  `json:"file_size"`
+	MimeType    string `json:"mime_type"`
+	Extension   string `json:"extension"`
+	Region      string `json:"region"`
+	Resumable   *struct {
+		PartSize      int64  `json:"part_size"`
+		TotalParts    int    `json:"total_parts"`
+		PartUploadURL string `json:"part_upload_url"`
+		CompleteURL   string `json:"complete_url"`
+		StatusURL     string `json:"status_url"`
+	} `json:"resumable"`
+}
+
+// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
+type UploadCompleteResponse struct {
+	Response
+	File struct {
+		ID        string  `json:"id"`
+		Name      string  `json:"name"`
+		SizeBytes int64   `json:"size_bytes"`
+		MimeType  string  `json:"mime_type"`
+		Extension string  `json:"extension"`
+		Region    string  `json:"region"`
+		Version   int     `json:"version"`
+		CreatedAt float64 `json:"created_at"`
+		// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
+		// "" for a multipart upload, which the server does not hash.
+		ContentHash string `json:"content_hash"`
+	} `json:"file"`
+}
+
+// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
+type UploadPartResponse struct {
+	Response
+	PartNumber     int    `json:"part_number"`
+	ETag           string `json:"etag"`
+	BytesUploaded  int64  `json:"bytes_uploaded"`
+	PartsCompleted int    `json:"parts_completed"`
+	TotalParts     int    `json:"total_parts"`
+}
+
+// DeleteFileResponse is the response from DELETE /api/files/:id
+type DeleteFileResponse struct {
+	Response
+	Permanent bool `json:"permanent"`
+}
+
+// RenameFileRequest is the request body for PUT /api/files/:id/rename
+type RenameFileRequest struct {
+	Name string `json:"name"`
+}
+
+// RenameFileResponse is the response from PUT /api/files/:id/rename
+type RenameFileResponse struct {
+	Response
+	Name string `json:"name"`
+}
+
+// MoveFileRequest is the request body for PUT /api/files/:id/move
+type MoveFileRequest struct {
+	FolderID *string `json:"folder_id"`
+	Name     *string `json:"name,omitempty"`
+}
+
+// CopyFileRequest is the request body for POST /api/files/:id/copy
+type CopyFileRequest struct {
+	FolderID *string `json:"folder_id"`
+	Name     string  `json:"name,omitempty"`
+}
+
+// CopyFileResponse is the response from POST /api/files/:id/copy
+type CopyFileResponse struct {
+	Response
+	FileID string `json:"file_id"`
+	Name   string `json:"name"`
+}
+
+// ShareLinkRequest is the request body for POST /api/files/:id/share
+type ShareLinkRequest struct {
+	ExpiresInDays *int `json:"expires_in_days,omitempty"`
+}
+
+// ShareLinkResponse is the response from POST /api/files/:id/share
+type ShareLinkResponse struct {
+	Response
+	Link struct {
+		ID        string  `json:"id"`
+		Token     string  `json:"token"`
+		URL       string  `json:"url"`
+		ExpiresAt *int64  `json:"expires_at"`
+		CreatedAt float64 `json:"created_at"`
+	} `json:"link"`
+}
+
+// Workspace is one entry of the workspace list
+type Workspace struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+// WorkspaceListResponse is the response from GET /api/workspaces
+type WorkspaceListResponse struct {
+	Response
+	Workspaces []Workspace `json:"workspaces"`
+}
+
+// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
+type WorkspaceInfoResponse struct {
+	Response
+	Storage struct {
+		Used  int64 `json:"used"`
+		Total int64 `json:"total"`
+		Free  int64 `json:"free"`
+	} `json:"storage"`
+}

backend/dosya/dosya.go

@@ -0,0 +1,772 @@
+// Package dosya provides an interface to the dosya.dev cloud storage system.
+package dosya
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+const (
+	rootID         = "root"
+	defaultBaseURL = "https://api.dosya.dev"
+	minSleep       = 100 * time.Millisecond
+	maxSleep       = 2 * time.Second
+	decayConstant  = 2
+	attackConstant = 0
+)
+
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "dosya",
+		Description: "dosya.dev",
+		NewFs:       NewFs,
+		Options: []fs.Option{{
+			Help:      "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
+			Name:      "api_key",
+			Sensitive: true,
+			Required:  true,
+		}, {
+			Name: "workspace_id",
+			Help: `Your workspace ID.
+
+Leave this blank to use the one workspace the API key can reach, which
+is what a key restricted to a single workspace does. If the key can
+reach several, rclone will refuse and list their IDs for you to choose
+from.`,
+		}, {
+			Help:     "API base URL.",
+			Name:     "api_url",
+			Default:  defaultBaseURL,
+			Advanced: true,
+		}, {
+			Name: "upload_concurrency",
+			Help: `Concurrency for multipart uploads.
+
+This is the number of parts of the same file that are uploaded
+concurrently. The first part is always uploaded on its own, because
+the server creates the upload when it receives it.
+
+Each part in flight is held in memory, so memory use grows with
+this value times the part size the server chooses.
+
+If you are uploading large files over high-speed links and these
+uploads do not fully utilize your bandwidth, then increasing this
+may help to speed up the transfers.`,
+			Default:  4,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeSingleQuote |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeLtGt |
+				encoder.EncodeCtl |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
+		}},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	APIKey            string               `config:"api_key"`
+	WorkspaceID       string               `config:"workspace_id"`
+	APIURL            string               `config:"api_url"`
+	UploadConcurrency int                  `config:"upload_concurrency"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents a remote dosya.dev filesystem
+type Fs struct {
+	name     string
+	root     string
+	opt      Options
+	features *fs.Features
+	rest     *rest.Client
+	pacer    *fs.Pacer
+	dirCache *dircache.DirCache
+}
+
+// Object represents a dosya.dev file
+type Object struct {
+	fs     *Fs
+	remote string
+	file   api.FileItem
+}
+
+// findWorkspace fills in the workspace ID when the config leaves it blank.
+//
+// The ID is not shown anywhere in the dosya.dev web interface, so requiring
+// it meant asking people to find it in a developer console. A key restricted
+// to one workspace can only mean that one, and an account with a single
+// workspace has only one possible answer; anything else genuinely has to be
+// chosen, so list the choices in the error.
+func (f *Fs) findWorkspace(ctx context.Context) error {
+	workspaces, err := f.listWorkspaces(ctx)
+	if err != nil {
+		return err
+	}
+	switch len(workspaces) {
+	case 0:
+		return errors.New("no workspaces found for this API key")
+	case 1:
+		f.opt.WorkspaceID = workspaces[0].ID
+		fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
+		return nil
+	}
+	var choices strings.Builder
+	for _, w := range workspaces {
+		_, _ = fmt.Fprintf(&choices, "\n    %s (%s)", w.ID, w.Name)
+	}
+	return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
+		len(workspaces), choices.String())
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+	listing, err := f.listFilesAndFolders(ctx, pathID)
+	if err != nil {
+		return "", false, err
+	}
+	for _, folder := range listing.Folders {
+		if f.opt.Enc.ToStandardName(folder.Name) == leaf {
+			return folder.ID, true, nil
+		}
+	}
+	return "", false, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+	name := f.opt.Enc.FromStandardName(leaf)
+	resp, err := f.createFolder(ctx, name, pathID)
+	if err != nil {
+		return "", err
+	}
+	return resp.Folder.ID, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("dosya.dev root '%s'", f.root)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	// Modification times are stored and returned to the second (the API
+	// keeps them as Unix seconds, sent on upload via the source-mtime header).
+	return time.Second
+}
+
+// Hashes returns the supported hash types of the filesystem
+func (f *Fs) Hashes() hash.Set {
+	// The server computes and stores the SHA-256 of a single-shot upload's
+	// bytes; multipart uploads have none (see Object.Hash).
+	return hash.NewHashSet(hash.SHA256)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// NewFs makes a new Fs object from the path
+func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
+	opt := new(Options)
+	err := configstruct.Set(config, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	root = strings.Trim(root, "/")
+
+	f := &Fs{
+		name:  name,
+		root:  root,
+		opt:   *opt,
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
+	}
+
+	// No DuplicateFiles: an upload that names no existing file but whose
+	// (workspace, folder, name) is already taken becomes a new VERSION of
+	// that file rather than a second row, so a name is unique within a
+	// folder and the same upload twice returns the same file id.
+	f.features = (&fs.Features{
+		CaseInsensitive:         false,
+		CanHaveEmptyDirectories: true,
+		ReadMimeType:            true,
+	}).Fill(ctx, f)
+
+	client := fshttp.NewClient(ctx)
+	f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
+	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
+
+	if f.opt.WorkspaceID == "" {
+		err = f.findWorkspace(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	f.dirCache = dircache.New(root, rootID, f)
+
+	// Find the current root
+	err = f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		// Assume it is a file
+		newRoot, remote := dircache.SplitPath(root)
+		tempF := *f
+		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
+		tempF.root = newRoot
+		// Make new Fs which is the parent
+		err = tempF.dirCache.FindRoot(ctx, false)
+		if err != nil {
+			// No root so return old f
+			return f, nil
+		}
+		_, err := tempF.NewObject(ctx, remote)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound {
+				// File doesn't exist so return old f
+				return f, nil
+			}
+			return nil, err
+		}
+		f.features.Fill(ctx, &tempF)
+		f.dirCache = tempF.dirCache
+		f.root = tempF.root
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// List the objects and directories in dir into entries
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return nil, err
+	}
+
+	listing, err := f.listFilesAndFolders(ctx, directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, folder := range listing.Folders {
+		folderName := f.opt.Enc.ToStandardName(folder.Name)
+		fullPath := getRemote(dir, folderName)
+		modTime := time.Unix(int64(folder.CreatedAt), 0)
+		d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
+		entries = append(entries, d)
+		f.dirCache.Put(fullPath, folder.ID)
+	}
+
+	for _, file := range listing.Files {
+		fileName := f.opt.Enc.ToStandardName(file.Name)
+		remote := getRemote(dir, fileName)
+		o := &Object{
+			fs:     f,
+			remote: remote,
+			file:   file,
+		}
+		entries = append(entries, o)
+	}
+
+	return entries, nil
+}
+
+// NewObject finds the Object at remote
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
+	if err != nil {
+		if err == fs.ErrorDirNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
+	}
+
+	listing, err := f.listFilesAndFolders(ctx, directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range listing.Files {
+		if f.opt.Enc.ToStandardName(file.Name) == leaf {
+			return &Object{
+				fs:     f,
+				remote: remote,
+				file:   file,
+			}, nil
+		}
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Put uploads a new file
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	existingObj, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return existingObj, existingObj.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		return f.put(ctx, in, src, options...)
+	default:
+		return nil, err
+	}
+}
+
+// put uploads src as a new file
+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	remote := src.Remote()
+
+	_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	modTime := src.ModTime(ctx)
+	resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file:   fileFromUpload(resp, modTime),
+	}, nil
+}
+
+// fileFromUpload builds the FileItem for the object an upload just created.
+// The single-shot upload response omits updated_at, so the modification time
+// is taken from what was sent (modTime) rather than defaulting to the create
+// time - it is the time the server stored via the source-mtime header. The
+// SHA-256 comes straight from the response (empty for a multipart upload).
+func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
+	updatedAt := resp.File.CreatedAt
+	if !modTime.IsZero() {
+		updatedAt = float64(modTime.Unix())
+	}
+	return api.FileItem{
+		ID:          resp.File.ID,
+		Name:        resp.File.Name,
+		SizeBytes:   resp.File.SizeBytes,
+		MimeType:    resp.File.MimeType,
+		Extension:   resp.File.Extension,
+		Region:      resp.File.Region,
+		CreatedAt:   resp.File.CreatedAt,
+		UpdatedAt:   updatedAt,
+		ContentHash: resp.File.ContentHash,
+	}
+}
+
+// Mkdir makes the directory (container, bucket)
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	_, err := f.dirCache.FindDir(ctx, dir, true)
+	return err
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+
+	// The server checks emptiness itself (?if_empty=1) and answers
+	// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
+	err = f.removeFolder(ctx, directoryID, true)
+	if err != nil {
+		return err
+	}
+
+	f.dirCache.FlushDir(dir)
+	return nil
+}
+
+// Move src to this remote using server-side move operations
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
+	_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Find destination directory
+	dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	needsMove := srcDirID != dstDirID
+	needsRename := srcLeaf != dstLeaf
+
+	// Moving and renaming in the same request (rather than a moveFile call
+	// followed by a separate renameFile call) matters when both are needed:
+	// a two-step move-then-rename can transiently place the file's CURRENT
+	// name into a destination folder that already holds an unrelated file
+	// with that name, even though the FINAL name is free - the server only
+	// allows one active name per folder, so that intermediate state 409s.
+	if needsMove || needsRename {
+		newName := ""
+		if needsRename {
+			newName = f.opt.Enc.FromStandardName(dstLeaf)
+		}
+		err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Return new object. file.Name must reflect the destination leaf, not
+	// the source's - a caller that reuses this object as the src of a
+	// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
+	// recomputes srcLeaf from file.Name; a stale name there makes that
+	// second move see srcLeaf == dstLeaf and skip the rename entirely,
+	// silently leaving the object under the wrong name.
+	newFile := srcObj.file
+	newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file:   newFile,
+	}, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+	if err != nil {
+		return err
+	}
+
+	srcLeaf := srcRemote
+	if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
+		srcLeaf = srcRemote[idx+1:]
+	}
+	needsRename := srcLeaf != dstLeaf
+	newName := ""
+	if needsRename {
+		newName = f.opt.Enc.FromStandardName(dstLeaf)
+	}
+
+	// Move folder to new parent only if parent changed. The new name goes
+	// in the same request, as for files in Move: a move followed by a
+	// separate rename can collide with an unrelated folder sharing the
+	// CURRENT name in the destination even though the final name is free.
+	if srcDirectoryID != dstDirectoryID {
+		renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
+		if err != nil {
+			return fmt.Errorf("couldn't move directory: %w", err)
+		}
+		if renamed {
+			needsRename = false
+		}
+	}
+
+	if needsRename {
+		err = f.renameFolder(ctx, srcID, newName)
+		if err != nil {
+			return fmt.Errorf("couldn't rename directory: %w", err)
+		}
+	}
+
+	srcFs.dirCache.FlushDir(srcRemote)
+	return nil
+}
+
+// Copy src to this remote using server-side copy operations
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+
+	dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	// The copy API creates a new row named after the source file when
+	// copying across folders, which collides with one already at the
+	// destination. Copy under a temporary name, then delete the old
+	// destination, then move the copy onto the name - so a copy that fails
+	// leaves the destination as it was. Deleting it up front lost it
+	// (prior art: backend/box).
+	existingObj, err := f.NewObject(ctx, remote)
+	switch err {
+	case nil:
+		tempRemote := remote + "-rclone-copy-" + random.String(8)
+		fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
+		tempObj, err := f.Copy(ctx, src, tempRemote)
+		if err != nil {
+			return nil, err
+		}
+		fs.Debugf(tempRemote, "moving to real name %v", remote)
+		if err := existingObj.Remove(ctx); err != nil {
+			return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
+		}
+		return f.Move(ctx, tempObj, remote)
+	case fs.ErrorObjectNotFound:
+		// nothing at the destination, so copy straight onto the name
+	default:
+		return nil, err
+	}
+
+	dstName := f.opt.Enc.FromStandardName(dstLeaf)
+
+	resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't copy file: %w", err)
+	}
+
+	// Servers that predate the name field ignore it and name the copy after
+	// the source, or "Copy of <original>" in the same folder. Rename to the
+	// requested destination name if it differs.
+	if resp.Name != dstName {
+		err = f.renameFile(ctx, resp.FileID, dstName)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't rename copied file: %w", err)
+		}
+		resp.Name = dstName
+	}
+
+	// A server-side copy is byte-identical to the source and the API
+	// preserves the source's modification time and content hash onto the
+	// copy, so the returned object must report the SOURCE's modtime and hash,
+	// not the copy's wall-clock time. Reading "now" here made the object
+	// rclone holds in memory disagree with the one read back from the remote
+	// (its Fingerprint check compares the two exactly).
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file: api.FileItem{
+			ID:          resp.FileID,
+			Name:        resp.Name,
+			SizeBytes:   srcObj.file.SizeBytes,
+			MimeType:    srcObj.file.MimeType,
+			Extension:   srcObj.file.Extension,
+			Region:      srcObj.file.Region,
+			CreatedAt:   srcObj.file.CreatedAt,
+			UpdatedAt:   srcObj.file.UpdatedAt,
+			ContentHash: srcObj.file.ContentHash,
+		},
+	}, nil
+}
+
+// Purge deletes all the files and the container
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+
+	// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
+	err = f.removeFolder(ctx, directoryID, false)
+	if err != nil {
+		return err
+	}
+
+	f.dirCache.FlushDir(dir)
+	return nil
+}
+
+// PublicLink generates a public link to the remote path
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	o, err := f.NewObject(ctx, remote)
+	if err != nil {
+		return "", err
+	}
+	obj := o.(*Object)
+
+	resp, err := f.createShareLink(ctx, obj.file.ID, expire)
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Link.URL, nil
+}
+
+// DirCacheFlush resets the directory cache - used in testing as an
+// optional interface
+func (f *Fs) DirCacheFlush() {
+	f.dirCache.ResetRoot()
+}
+
+// About gets quota information from the Fs
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+	info, err := f.getWorkspaceInfo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	usage := &fs.Usage{
+		Used:  fs.NewUsageValue(info.Storage.Used),
+		Total: fs.NewUsageValue(info.Storage.Total),
+		Free:  fs.NewUsageValue(info.Storage.Free),
+	}
+	return usage, nil
+}
+
+// ------------------------------------------------------------
+// Object methods
+// ------------------------------------------------------------
+
+// String returns a description of the Object
+func (o *Object) String() string {
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification date of the file
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	if o.file.UpdatedAt > 0 {
+		return time.Unix(int64(o.file.UpdatedAt), 0)
+	}
+	return time.Unix(int64(o.file.CreatedAt), 0)
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return o.file.SizeBytes
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns the selected checksum of the file
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.SHA256 {
+		return "", hash.ErrUnsupported
+	}
+	// May be "" - a multipart-uploaded object carries no server-side hash,
+	// which rclone treats as "hash unavailable for this object" (as it does
+	// for S3 multipart objects).
+	return o.file.ContentHash, nil
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+// Open opens the file for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	fs.FixRangeOption(options, o.file.SizeBytes)
+	return o.fs.downloadFile(ctx, o.file.ID, options)
+}
+
+// Update replaces the file content
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	// Find the directory ID for this file
+	_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
+	if err != nil {
+		return err
+	}
+
+	// Upload as new version of existing file
+	fileID := o.file.ID
+	modTime := src.ModTime(ctx)
+	resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
+	if err != nil {
+		return err
+	}
+
+	// Update object metadata
+	o.file = fileFromUpload(resp, modTime)
+
+	return nil
+}
+
+// Remove removes this object
+func (o *Object) Remove(ctx context.Context) error {
+	return o.fs.deleteFile(ctx, o.file.ID)
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.file.MimeType
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.file.ID
+}
+
+func getRemote(dir, fileName string) string {
+	if dir == "" {
+		return fileName
+	}
+	return dir + "/" + fileName
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs              = (*Fs)(nil)
+	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Purger          = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ dircache.DirCacher = (*Fs)(nil)
+	_ fs.Object          = (*Object)(nil)
+	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
+)

backend/dosya/dosya_api.go

@@ -0,0 +1,880 @@
+package dosya
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
+)
+
+// sourceMTimeHeader is the request header the upload endpoints read to set a
+// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
+const sourceMTimeHeader = "x-dosya-source-mtime"
+
+// modTimeHeaders returns the ExtraHeaders that carry a source modification time
+// on an upload, or nil when there is nothing plausible to send. The server
+// clamps implausible values itself, but a zero time is never worth a header.
+func modTimeHeaders(modTime time.Time) map[string]string {
+	if modTime.IsZero() {
+		return nil
+	}
+	return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
+}
+
+// workspaceHintHeader names the workspace a by-id request is about. The API
+// maps an id to its workspace through an index the workspace fills about a
+// second after a write, so an id this client was handed moments ago can still
+// miss; the hint lets the server look for it in the named workspace instead.
+// It only locates: every route still checks membership against the workspace
+// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
+const workspaceHintHeader = "X-Dosya-Workspace"
+
+// byIDHeaders returns the ExtraHeaders every /api/files/:id and
+// /api/folders/:id call carries, or nil when no workspace is configured.
+func (f *Fs) byIDHeaders() map[string]string {
+	if f.opt.WorkspaceID == "" {
+		return nil
+	}
+	return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	429, // Too Many Requests
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
+	if retry {
+		if d, ok := retryAfter(resp); ok {
+			// The API answers 503 + Retry-After while a workspace is being
+			// moved between storage homes, and Cloudflare sends it with 429.
+			// The pacer's own backoff tops out at maxSleep, far short of that.
+			fs.Debugf(nil, "Server asked to retry after %v", d)
+			err = pacer.RetryAfterError(err, d)
+		}
+	}
+	return retry, err
+}
+
+// maxRetryAfter caps how long a single Retry-After is honoured for
+const maxRetryAfter = 5 * time.Minute
+
+// retryAfter reads a 429 or 503 response's Retry-After header, which the API
+// sends as whole seconds
+func retryAfter(resp *http.Response) (time.Duration, bool) {
+	if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
+		return 0, false
+	}
+	seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
+	if err != nil || seconds <= 0 {
+		return 0, false
+	}
+	return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
+}
+
+// errorHandler decodes a non-2xx response into an *api.Error so callers can
+// act on the machine-readable error_code rather than the message text.
+func errorHandler(resp *http.Response) error {
+	body, err := rest.ReadBody(resp)
+	if err != nil {
+		return fmt.Errorf("error reading error out of body: %w", err)
+	}
+	apiErr := &api.Error{
+		StatusCode: resp.StatusCode,
+		Status:     resp.Status,
+	}
+	if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
+		// Not the API's JSON shape (e.g. an edge response that never
+		// reached the API) - keep the body text so it isn't lost.
+		apiErr.ErrorCode = ""
+		apiErr.Message = strings.TrimSpace(string(body))
+	}
+	return apiErr
+}
+
+// The workspace caps the number of in-flight uploads per user. The API
+// refuses an upload/init past that cap with a 400 carrying
+// error_code "concurrent_upload_limit", and documents it as an error to
+// wait on rather than give up on: slots free as uploads finish, and the
+// server marks sessions idle for 10 minutes as failed, so waiting up to
+// that long guarantees progress unless the workspace really is saturated.
+// A 400 is deliberately outside the pacer's retry codes, so the wait
+// lives in initUpload instead of shouldRetry.
+var (
+	concurrentUploadRetryDelay = 5 * time.Second
+	concurrentUploadWaitMax    = 10 * time.Minute
+)
+
+// abortUploadTimeout bounds the abort of a session the client is giving up on,
+// which runs outside the transfer's context so a cancelled transfer still
+// sends it
+const abortUploadTimeout = 10 * time.Second
+
+// isConcurrentUploadLimit reports whether err is the API refusing an
+// upload because the workspace's concurrent upload limit is reached
+func isConcurrentUploadLimit(err error) bool {
+	var apiErr *api.Error
+	return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
+}
+
+// listPageSize is the largest page GET /api/files will serve
+const listPageSize = 500
+
+// listFilesAndFolders lists files and folders in a directory
+//
+// The API pages files (at most listPageSize per page) but returns every
+// folder on each page, so the files from all pages are gathered and the
+// folders are taken from the first page only. Pages are ordered by name,
+// which the API tie-breaks on id, so offsets are stable between requests.
+func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
+	var listing *api.ListResponse
+	for page := 1; ; page++ {
+		opts := rest.Opts{
+			Method: "GET",
+			Path:   "/api/files",
+			Parameters: map[string][]string{
+				"workspace_id": {f.opt.WorkspaceID},
+				"per_page":     {strconv.Itoa(listPageSize)},
+				"page":         {strconv.Itoa(page)},
+				"sort":         {"name"},
+				"dir":          {"asc"},
+			},
+		}
+		if folderID != "" && folderID != rootID {
+			opts.Parameters["folder_id"] = []string{folderID}
+		}
+
+		var result api.ListResponse
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("couldn't list files: %w", err)
+		}
+		if !result.OK {
+			return nil, fmt.Errorf("API error: %s", result.Error)
+		}
+		if listing == nil {
+			listing = &result
+		} else {
+			listing.Files = append(listing.Files, result.Files...)
+		}
+		if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
+			return listing, nil
+		}
+	}
+}
+
+// createFolder creates a folder
+func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
+	request := api.CreateFolderRequest{
+		WorkspaceID: f.opt.WorkspaceID,
+		Name:        name,
+	}
+	if parentID != "" && parentID != rootID {
+		request.ParentID = &parentID
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/api/folders",
+	}
+
+	var result api.CreateFolderResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create folder: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// removeFolder deletes a folder
+//
+// ?permanent=1 trashes and purges a live folder in one request on today's
+// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
+// HTTP 409 "Folder is not empty" instead of purging a folder that still
+// holds a live file or a live child folder; that 409 is turned into
+// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
+// error, and it is never retried (409 is not one of retryErrorCodes). A big
+// subtree is purged in bounded passes (202, complete=false), so keep
+// calling until the purge is complete. An older server that ignores
+// permanent=1 only trashes on the first call (answering permanent=false),
+// so the loop's second pass sends the plain DELETE that purges it.
+func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
+	parameters := map[string][]string{"permanent": {"1"}}
+	if ifEmpty {
+		parameters["if_empty"] = []string{"1"}
+	}
+	opts := rest.Opts{
+		Method:       "DELETE",
+		Path:         "/api/folders/" + folderID,
+		Parameters:   parameters,
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	for pass := 0; ; pass++ {
+		var result api.DeleteFolderResponse
+		var statusCode int
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+			if resp != nil {
+				statusCode = resp.StatusCode
+			}
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			if ifEmpty && statusCode == http.StatusConflict {
+				return fs.ErrorDirectoryNotEmpty
+			}
+			if pass == 0 {
+				return fmt.Errorf("couldn't remove folder: %w", err)
+			}
+			return fmt.Errorf("couldn't permanently remove folder: %w", err)
+		}
+		if !result.OK {
+			return fmt.Errorf("API error: %s", result.Error)
+		}
+		if result.Permanent && result.Complete {
+			return nil
+		}
+		if pass >= maxFolderPurgePasses {
+			return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
+		}
+	}
+}
+
+// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
+const maxFolderPurgePasses = 1000
+
+// renameFolder renames a folder
+func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
+	request := api.RenameFolderRequest{Name: newName}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/folders/" + folderID + "/rename",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.Response
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't rename folder: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+	return nil
+}
+
+// moveFolder moves a folder to a new parent
+func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
+	request := api.MoveFolderRequest{Name: newName}
+	if newParentID != "" && newParentID != rootID {
+		request.ParentID = &newParentID
+	}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/folders/" + folderID + "/move",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.MoveFolderResponse
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return false, fmt.Errorf("couldn't move folder: %w", err)
+	}
+	if !result.OK {
+		return false, fmt.Errorf("API error: %s", result.Error)
+	}
+	// Servers that predate the name field ignore it and answer without a
+	// name, leaving the caller to rename separately
+	return newName != "" && result.Name == newName, nil
+}
+
+// deleteFile deletes a file
+//
+// ?permanent=1 trashes and purges it in one request on today's API. An
+// older server that ignores the flag only soft-deletes it, answering
+// permanent=false, so a second DELETE on the now soft-deleted file purges
+// it from the database and R2, as it always has.
+func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
+	opts := rest.Opts{
+		Method:       "DELETE",
+		Path:         "/api/files/" + fileID,
+		Parameters:   map[string][]string{"permanent": {"1"}},
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.DeleteFileResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't delete file: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+
+	// Answered into its own value so a field the second response omits
+	// can't be read off the first.
+	if !result.Permanent {
+		var permanent api.DeleteFileResponse
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return fmt.Errorf("couldn't permanently delete file: %w", err)
+		}
+		if !permanent.OK {
+			return fmt.Errorf("API error: %s", permanent.Error)
+		}
+	}
+	return nil
+}
+
+// renameFile renames a file
+func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
+	request := api.RenameFileRequest{Name: newName}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/files/" + fileID + "/rename",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.RenameFileResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't rename file: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+	return nil
+}
+
+// moveFile moves a file to a new folder and, if newName is non-empty,
+// renames it in the same request
+func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
+	request := api.MoveFileRequest{}
+	if folderID != "" && folderID != rootID {
+		request.FolderID = &folderID
+	}
+	if newName != "" {
+		request.Name = &newName
+	}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/files/" + fileID + "/move",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.Response
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't move file: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+	return nil
+}
+
+// copyFile copies a file to a folder
+func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
+	request := api.CopyFileRequest{Name: name}
+	if folderID != "" && folderID != rootID {
+		request.FolderID = &folderID
+	}
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         "/api/files/" + fileID + "/copy",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.CopyFileResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't copy file: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// downloadFile downloads a file by its ID
+//
+// The download endpoint answers 302 with a presigned R2 URL in Location, so
+// this is two calls. Both go through the rest client, which is built on
+// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
+// --user-agent, --bind, --dump and the proxy settings apply to downloads as
+// they do to every other call, and errorHandler and shouldRetry decide the
+// errors and the retries uniformly.
+func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
+	// NoRedirect stops the client following the 302 with the API's
+	// Authorization header attached. A 302 is not a 2xx, so Call reports it
+	// as an error (having read and closed the body); the Location header
+	// is what actually matters, and finding it clears the error, as in
+	// backend/linkbox.
+	opts := rest.Opts{
+		Method:       "GET",
+		Path:         "/api/files/" + fileID + "/download",
+		ExtraHeaders: f.byIDHeaders(),
+		NoRedirect:   true,
+	}
+	var downloadURL string
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.Call(ctx, &opts)
+		if resp != nil {
+			if location := resp.Header.Get("Location"); location != "" {
+				downloadURL, err = location, nil
+			}
+		}
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get download URL: %w", err)
+	}
+	if downloadURL == "" {
+		return nil, errors.New("no download URL in redirect response")
+	}
+
+	// Fetch the object from the presigned URL. It carries its own
+	// credentials and R2 refuses a request that also has an Authorization
+	// header, so the client-level bearer token is removed by setting the
+	// header empty: the rest client skips a header with an empty value.
+	opts = rest.Opts{
+		Method:       "GET",
+		RootURL:      downloadURL,
+		Options:      options,
+		ExtraHeaders: map[string]string{"Authorization": ""},
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		resp, err = f.rest.Call(ctx, &opts)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't download file: %w", err)
+	}
+	return resp.Body, nil
+}
+
+// uploadSmallFile uploads a file smaller than 10MB in a single request
+func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
+	opts := rest.Opts{
+		Method:        "PUT",
+		Path:          "/api/upload/" + sessionID,
+		Body:          in,
+		ContentLength: &size,
+		ContentType:   "application/octet-stream",
+		ExtraHeaders:  modTimeHeaders(modTime),
+	}
+
+	// The body is a stream which can't be rewound, so a low level retry
+	// would resend whatever is left of it - nothing - and the server
+	// would store that as an empty file. Don't retry here; rclone's high
+	// level retries redo the whole transfer instead.
+	var result api.UploadCompleteResponse
+	err := f.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't upload file: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// uploadPart uploads a single part of a multipart upload
+func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
+	size := int64(len(part))
+	opts := rest.Opts{
+		Method:        "PUT",
+		Path:          "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
+		ContentLength: &size,
+		ContentType:   "application/octet-stream",
+	}
+
+	var result api.UploadPartResponse
+	err := f.pacer.Call(func() (bool, error) {
+		// A fresh reader per attempt so a retry resends the whole part
+		opts.Body = bytes.NewReader(part)
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// completeUpload finalizes a multipart upload
+func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         "/api/upload/" + sessionID + "/complete",
+		ExtraHeaders: modTimeHeaders(modTime),
+	}
+
+	var result api.UploadCompleteResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't complete upload: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// initUpload initializes a new upload session
+func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
+	request := api.UploadInitRequest{
+		WorkspaceID: f.opt.WorkspaceID,
+		FileName:    fileName,
+		FileSize:    fileSize,
+		MimeType:    mimeType,
+		FileID:      fileID,
+	}
+	if folderID != "" && folderID != rootID {
+		request.FolderID = &folderID
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/api/upload/init",
+	}
+
+	var result api.UploadInitResponse
+	deadline := time.Now().Add(concurrentUploadWaitMax)
+	for {
+		result = api.UploadInitResponse{}
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err == nil {
+			break
+		}
+		if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
+			return nil, fmt.Errorf("couldn't init upload: %w", err)
+		}
+		fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
+		case <-time.After(concurrentUploadRetryDelay):
+		}
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// uploadParts reads the source part by part and uploads the parts.
+//
+// Part 1 goes on its own: the server creates the R2 multipart upload (and a
+// new file's id and key) when it receives the first part, so parts sent
+// before that has finished would each start an upload of their own. The
+// rest go out opt.UploadConcurrency at a time. A part is only read from the
+// source once a buffer is free, so at most that many parts are held in
+// memory, and a short source is still found by the reader before the part
+// it could not fill is sent.
+func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
+	readPart := func(buf []byte, partNum int) ([]byte, error) {
+		currentPartSize := partSize
+		if partNum == totalParts {
+			currentPartSize = size - partSize*int64(totalParts-1)
+		}
+		part := buf[:currentPartSize]
+		if _, err := io.ReadFull(in, part); err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
+			}
+			return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
+		}
+		return part, nil
+	}
+
+	concurrency := max(f.opt.UploadConcurrency, 1)
+	buffers := make(chan []byte, concurrency)
+	buffers <- make([]byte, partSize)
+	allocated := 1
+
+	first := <-buffers
+	part, err := readPart(first, 1)
+	if err != nil {
+		return err
+	}
+	if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
+		return err
+	}
+	buffers <- first
+
+	// A failed read has to take the parts already in flight down with it,
+	// or they upload to completion while the error waits on Wait.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	g, gCtx := errgroup.WithContext(ctx)
+	var readErr error
+	for partNum := 2; partNum <= totalParts; partNum++ {
+		// Take a free buffer, allocating one while under the limit.
+		var buf []byte
+		select {
+		case buf = <-buffers:
+		default:
+			if allocated < concurrency {
+				buf = make([]byte, partSize)
+				allocated++
+			} else {
+				select {
+				case buf = <-buffers:
+				case <-gCtx.Done():
+				}
+			}
+		}
+		if buf == nil {
+			break // a part failed; its error comes back from Wait
+		}
+		part, err := readPart(buf, partNum)
+		if err != nil {
+			readErr = err
+			cancel()
+			break
+		}
+		g.Go(func() error {
+			defer func() { buffers <- buf }()
+			_, err := f.uploadPart(gCtx, part, sessionID, partNum)
+			return err
+		})
+	}
+	err = g.Wait()
+	if readErr != nil {
+		return readErr
+	}
+	return err
+}
+
+// abortUpload retires an upload session the client is giving up on.
+//
+// Best effort: the server reclaims an abandoned session itself, so a failure
+// here - including an older server with no such route - costs the slot the
+// wait it would have cost anyway and is not worth failing an upload over.
+// Sent on a context of its own so a cancelled transfer still gives its slot
+// back rather than leaving it to the sweep.
+func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
+	defer cancel()
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/api/upload/" + sessionID,
+		NoResponse: true,
+	}
+	err := f.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := f.rest.Call(ctx, &opts)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
+	}
+}
+
+// uploadFile handles the full upload flow (small or multipart)
+func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
+	// Both refusals live here so a new file and a new version of an existing
+	// one answer the same way: upload/init needs a byte count up front, and
+	// the server stores no zero-length object.
+	if size < 0 {
+		return nil, errors.New("can't upload files with unknown size")
+	}
+	if size == 0 {
+		return nil, fs.ErrorCantUploadEmptyFiles
+	}
+
+	leaf := f.opt.Enc.FromStandardName(path.Base(remote))
+
+	initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
+	if err != nil {
+		return nil, err
+	}
+	// A session abandoned here holds one of the workspace's upload slots
+	// until the server's sweep retires it ten minutes later, and rclone's
+	// high-level retry opens a fresh session rather than resuming this one -
+	// so without this, a few failed attempts use up every slot.
+	defer func() {
+		if err != nil {
+			f.abortUpload(ctx, initResp.SessionID)
+		}
+	}()
+
+	// Count what the source actually supplies, so a source which ends
+	// early is reported as an error rather than stored as a truncated
+	// file.
+	counter := readers.NewCountingReader(in)
+
+	// Small file upload (single PUT, no resumable)
+	if initResp.Resumable == nil {
+		resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
+		if err != nil {
+			return nil, err
+		}
+		if got := int64(counter.BytesRead()); got != size {
+			return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
+		}
+		return resp, nil
+	}
+
+	// Multipart upload. Each part is read fully into memory before it is
+	// sent, so a short source is caught before the part goes out and a
+	// part which has to be retried is resent with all of its bytes.
+	partSize := initResp.Resumable.PartSize
+	totalParts := initResp.Resumable.TotalParts
+	// Both bounds matter. partSize*(totalParts-1) >= size means a part would
+	// have nothing to carry; size > partSize*totalParts means the last part
+	// is bigger than partSize, and readPart would slice past its buffer and
+	// panic.
+	if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
+		return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
+	}
+	if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
+		return nil, err
+	}
+
+	return f.completeUpload(ctx, initResp.SessionID, modTime)
+}
+
+// createShareLink creates a public share link for a file
+func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
+	request := api.ShareLinkRequest{}
+	if expire < fs.DurationOff {
+		days := int(time.Duration(expire).Hours() / 24)
+		if days < 1 {
+			days = 1
+		}
+		request.ExpiresInDays = &days
+	}
+
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         "/api/files/" + fileID + "/share",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.ShareLinkResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create share link: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// getWorkspaceInfo fetches workspace storage info
+// listWorkspaces lists the workspaces this credential can reach. An API key
+// pinned to a workspace sees exactly one: its own.
+func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/api/workspaces",
+	}
+
+	var result api.WorkspaceListResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't list workspaces: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return result.Workspaces, nil
+}
+
+func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/api/workspaces/" + f.opt.WorkspaceID,
+	}
+
+	var result api.WorkspaceInfoResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get workspace info: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}

backend/dosya/dosya_compat_test.go

@@ -0,0 +1,235 @@
+package dosya
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShouldRetryHonoursRetryAfter(t *testing.T) {
+	ctx := context.Background()
+	for _, tc := range []struct {
+		name   string
+		status int
+		header string
+		want   time.Duration
+		ok     bool
+	}{
+		{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
+		{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
+		{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
+		{"missing header", http.StatusServiceUnavailable, "", 0, false},
+		{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
+		{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
+			if tc.header != "" {
+				resp.Header.Set("Retry-After", tc.header)
+			}
+			retry, err := shouldRetry(ctx, resp, errorHandler(resp))
+			assert.True(t, retry)
+			d, ok := pacer.IsRetryAfter(err)
+			assert.Equal(t, tc.ok, ok)
+			if tc.ok {
+				assert.Equal(t, tc.want, d)
+			}
+		})
+	}
+}
+
+func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
+	resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
+	resp.Header.Set("Retry-After", "30")
+	retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
+	assert.False(t, retry)
+	_, ok := pacer.IsRetryAfter(err)
+	assert.False(t, ok)
+}
+
+func TestRemoveFolderPurgesPermanently(t *testing.T) {
+	// a big subtree purges in bounded passes: 202 complete=false repeats
+	// the same request until complete=true
+	answers := []struct {
+		status int
+		body   string
+	}{
+		{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
+		{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
+	}
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		require.Equal(t, "DELETE", r.Method)
+		require.Equal(t, "/api/folders/fld_1", r.URL.Path)
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "", r.URL.Query().Get("if_empty"))
+		require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
+		a := answers[calls]
+		calls++
+		return jsonResp(a.status, a.body), nil
+	}))
+
+	require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
+	assert.Equal(t, len(answers), calls)
+}
+
+func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
+	}))
+
+	err := f.removeFolder(context.Background(), "fld_1", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "locked file")
+	assert.Equal(t, 1, calls)
+}
+
+func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+		return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
+	}))
+
+	err := f.removeFolder(context.Background(), "fld_1", true)
+	assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
+	assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
+}
+
+func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+		return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+	}))
+
+	require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
+	assert.Equal(t, 1, calls)
+}
+
+func TestRmdirMakesOneRequest(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		assert.Equal(t, "DELETE", r.Method)
+		assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+		return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+	}))
+	f.dirCache = dircache.New("", rootID, f)
+	require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
+	f.dirCache.Put("sub", "fld_1")
+
+	require.NoError(t, f.Rmdir(context.Background(), "sub"))
+	assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
+}
+
+func TestRmdirMapsNonEmptyConflict(t *testing.T) {
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
+	}))
+	f.dirCache = dircache.New("", rootID, f)
+	require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
+	f.dirCache.Put("sub", "fld_1")
+
+	err := f.Rmdir(context.Background(), "sub")
+	assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
+}
+
+func TestDeleteFileSendsPermanentFlag(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		permanent bool
+		wantCalls int
+	}{
+		{"today's API purges in one call", true, 1},
+		{"an older server ignores the flag and still gets a second DELETE", false, 2},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			calls := 0
+			f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+				calls++
+				assert.Equal(t, "DELETE", r.Method)
+				assert.Equal(t, "/api/files/fil_1", r.URL.Path)
+				assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+				return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
+			}))
+
+			require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
+			assert.Equal(t, tc.wantCalls, calls)
+		})
+	}
+}
+
+// decodeBody reads a JSON request body into a map
+func decodeBody(t *testing.T, r *http.Request) map[string]any {
+	b, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(b, &m))
+	return m
+}
+
+func TestMoveFolderSendsName(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		newName     string
+		respBody    string
+		wantRenamed bool
+	}{
+		{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
+		{"older server ignores name", "dst", `{"ok":true}`, false},
+		{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+				assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
+				body := decodeBody(t, r)
+				assert.Equal(t, "fld_parent", body["parent_id"])
+				if tc.newName == "" {
+					assert.NotContains(t, body, "name")
+				} else {
+					assert.Equal(t, tc.newName, body["name"])
+				}
+				return jsonResp(http.StatusOK, tc.respBody), nil
+			}))
+
+			renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
+			require.NoError(t, err)
+			assert.Equal(t, tc.wantRenamed, renamed)
+		})
+	}
+}
+
+func TestCopyFileSendsName(t *testing.T) {
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
+		body := decodeBody(t, r)
+		assert.Equal(t, "fld_dst", body["folder_id"])
+		assert.Equal(t, "final.txt", body["name"])
+		return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
+	}))
+
+	resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
+	require.NoError(t, err)
+	assert.Equal(t, "fil_2", resp.FileID)
+	assert.Equal(t, "final.txt", resp.Name)
+}

backend/dosya/dosya_hash_modtime_test.go

@@ -0,0 +1,171 @@
+package dosya
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
+	f := &Fs{}
+	assert.Equal(t, time.Second, f.Precision())
+	assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
+	// Exactly SHA-256, nothing else.
+	assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
+}
+
+func TestObjectHashReturnsContentHash(t *testing.T) {
+	o := &Object{file: api.FileItem{ContentHash: "abc123"}}
+
+	got, err := o.Hash(context.Background(), hash.SHA256)
+	require.NoError(t, err)
+	assert.Equal(t, "abc123", got)
+
+	// A multipart object with no server-side hash yields "" and no error,
+	// exactly as an S3 multipart object does.
+	empty := &Object{file: api.FileItem{ContentHash: ""}}
+	got, err = empty.Hash(context.Background(), hash.SHA256)
+	require.NoError(t, err)
+	assert.Equal(t, "", got)
+
+	// Any other algorithm is unsupported.
+	_, err = o.Hash(context.Background(), hash.MD5)
+	assert.ErrorIs(t, err, hash.ErrUnsupported)
+}
+
+func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+	o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
+	assert.True(t, mt.Equal(o.ModTime(context.Background())))
+}
+
+func TestModTimeHeaders(t *testing.T) {
+	assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
+
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
+	h := modTimeHeaders(mt)
+	require.NotNil(t, h)
+	assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
+}
+
+func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
+	resp := &api.UploadCompleteResponse{}
+	resp.File.ID = "fil_1"
+	resp.File.Name = "x.txt"
+	resp.File.SizeBytes = 10
+	resp.File.CreatedAt = 1_000
+	resp.File.ContentHash = "deadbeef"
+
+	mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
+	item := fileFromUpload(resp, mt)
+	assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
+	assert.Equal(t, "deadbeef", item.ContentHash)
+	assert.Equal(t, float64(1_000), item.CreatedAt)
+
+	// A zero modtime falls back to the create time.
+	item = fileFromUpload(resp, time.Time{})
+	assert.Equal(t, float64(1_000), item.UpdatedAt)
+}
+
+// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
+// fake transport and asserts the source-mtime header actually rides on the PUT.
+func TestUploadSendsSourceMTimeHeader(t *testing.T) {
+	var putMTime string
+	rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+		if r.Body != nil {
+			_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
+		}
+		switch {
+		case r.Method == "POST" && r.URL.Path == "/api/upload/init":
+			return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
+		case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
+			putMTime = r.Header.Get(sourceMTimeHeader)
+			return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
+		}
+		t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		return nil, nil
+	})
+	f := newFakeFs(rt)
+
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+	resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "h", resp.File.ContentHash)
+	assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
+}
+
+// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
+// the completion request for a multipart upload (the server stores the modtime
+// there, not on the individual parts).
+func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
+	var completeMTime string
+	rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+		if r.Body != nil {
+			_, _ = io.ReadAll(r.Body)
+		}
+		switch {
+		case r.URL.Path == "/api/upload/init":
+			return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
+		case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
+			return jsonResp(http.StatusOK, `{"ok":true}`), nil
+		case r.URL.Path == "/api/upload/upl_1/complete":
+			completeMTime = r.Header.Get(sourceMTimeHeader)
+			return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
+		}
+		t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		return nil, nil
+	})
+	f := newFakeFs(rt)
+
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+	_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
+}
+
+// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
+// server-side Copy returns: it must report the SOURCE's modtime and hash (the
+// copy is byte-identical and the API preserves both), not the copy's wall
+// clock. Reading "now" here made rclone's in-memory object disagree with the
+// one read back from the remote and failed FsCopy's fingerprint check.
+func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
+	rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+		switch {
+		case r.Method == "GET" && r.URL.Path == "/api/files":
+			// NewObject's existence probe on the destination: nothing there.
+			return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
+		case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
+			return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
+		}
+		t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		return nil, nil
+	})
+	f := newFakeFs(rt)
+	f.dirCache = dircache.New("", rootID, f)
+
+	mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
+	ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+	src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
+		ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
+		CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
+	}}
+
+	dst, err := f.Copy(context.Background(), src, "f-copy.txt")
+	require.NoError(t, err)
+
+	assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
+	assert.Equal(t, int64(100), dst.Size())
+	got, err := dst.Hash(context.Background(), hash.SHA256)
+	require.NoError(t, err)
+	assert.Equal(t, "srchash", got, "copy keeps the source hash")
+}

backend/dosya/dosya_internal_test.go

@@ -0,0 +1,156 @@
+package dosya
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
+
+func TestErrorHandlerDecodesAPIError(t *testing.T) {
+	resp := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Status:     "400 Bad Request",
+		Body:       io.NopCloser(strings.NewReader(concurrentLimitBody)),
+	}
+	err := errorHandler(resp)
+	var apiErr *api.Error
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
+	assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
+	assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
+	assert.Contains(t, err.Error(), "uploads in progress")
+	assert.Contains(t, err.Error(), "concurrent_upload_limit")
+}
+
+func TestErrorHandlerNonJSONBody(t *testing.T) {
+	// e.g. an edge (Cloudflare) response that never reached the API
+	resp := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Status:     "429 Too Many Requests",
+		Body:       io.NopCloser(strings.NewReader("error code: 1015\n")),
+	}
+	err := errorHandler(resp)
+	var apiErr *api.Error
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
+	assert.Equal(t, "", apiErr.ErrorCode)
+	assert.Contains(t, err.Error(), "429")
+	assert.Contains(t, err.Error(), "error code: 1015")
+}
+
+func TestShouldRetryAPIErrors(t *testing.T) {
+	ctx := context.Background()
+
+	// The concurrent upload limit is handled by initUpload's slot wait,
+	// not by the pacer's low level retries.
+	limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
+	retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
+	assert.False(t, retry)
+	assert.Equal(t, limitErr, err)
+
+	retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
+	assert.True(t, retry)
+
+	retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
+	assert.False(t, retry)
+}
+
+// swapWaitTimings shortens the slot wait timings for a test and returns a
+// function which restores them
+func swapWaitTimings(delay, max time.Duration) func() {
+	oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
+	concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
+	return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
+}
+
+// newTestFs makes an Fs which talks to srv
+func newTestFs(srv *httptest.Server) *Fs {
+	ctx := context.Background()
+	f := &Fs{
+		name: "test",
+		opt:  Options{WorkspaceID: "ws_test"},
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(
+			pacer.MinSleep(time.Millisecond),
+			pacer.MaxSleep(time.Millisecond),
+			pacer.DecayConstant(decayConstant),
+			pacer.AttackConstant(attackConstant),
+		)),
+	}
+	f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
+	return f
+}
+
+// limitServer answers the first n init calls with the concurrent upload
+// limit error and the rest with success
+func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/upload/init", r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		if calls.Add(1) <= n {
+			w.WriteHeader(http.StatusBadRequest)
+			_, _ = w.Write([]byte(concurrentLimitBody))
+			return
+		}
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+}
+
+func TestInitUploadWaitsForFreeSlot(t *testing.T) {
+	defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
+	var calls atomic.Int32
+	srv := limitServer(t, 2, &calls)
+	defer srv.Close()
+	f := newTestFs(srv)
+
+	resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
+	require.NoError(t, err)
+	assert.True(t, resp.OK)
+	assert.Equal(t, int32(3), calls.Load())
+}
+
+func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
+	defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
+	var calls atomic.Int32
+	srv := limitServer(t, 1<<30, &calls)
+	defer srv.Close()
+	f := newTestFs(srv)
+
+	_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
+	require.Error(t, err)
+	var apiErr *api.Error
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
+	assert.Contains(t, err.Error(), "couldn't init upload")
+	assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
+}
+
+func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
+	defer swapWaitTimings(time.Minute, time.Hour)()
+	var calls atomic.Int32
+	srv := limitServer(t, 1<<30, &calls)
+	defer srv.Close()
+	f := newTestFs(srv)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, context.DeadlineExceeded)
+	assert.Less(t, time.Since(start), 5*time.Second)
+}

backend/dosya/dosya_list_test.go

@@ -0,0 +1,68 @@
+package dosya
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeListServer serves GET /api/files the way the API does: files paged at
+// per_page, every folder repeated on each page
+func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		require.Equal(t, "/api/files", r.URL.Path)
+		q := r.URL.Query()
+		assert.Equal(t, "name", q.Get("sort"))
+		assert.Equal(t, "asc", q.Get("dir"))
+		page, _ := strconv.Atoi(q.Get("page"))
+		perPage, _ := strconv.Atoi(q.Get("per_page"))
+		require.Positive(t, page)
+		require.Positive(t, perPage)
+		*pages = append(*pages, page)
+
+		var files []string
+		for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
+			files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
+		}
+		totalPages := max(1, (totalFiles+perPage-1)/perPage)
+		body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
+			`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
+			strings.Join(files, ","), page, perPage, totalFiles, totalPages)
+		return jsonResp(http.StatusOK, body), nil
+	}
+}
+
+func TestListFilesAndFoldersFollowsPages(t *testing.T) {
+	var pages []int
+	f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
+
+	listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
+	require.NoError(t, err)
+
+	assert.Equal(t, []int{1, 2, 3}, pages)
+	require.Len(t, listing.Files, 2*listPageSize+7)
+	assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
+	// folders repeat on every page and must not be duplicated
+	assert.Len(t, listing.Folders, 1)
+}
+
+func TestListFilesAndFoldersSinglePage(t *testing.T) {
+	for _, total := range []int{0, 1, listPageSize} {
+		t.Run(strconv.Itoa(total), func(t *testing.T) {
+			var pages []int
+			f := newFakeFs(fakeListServer(t, total, &pages))
+
+			listing, err := f.listFilesAndFolders(context.Background(), rootID)
+			require.NoError(t, err)
+
+			assert.Equal(t, []int{1}, pages)
+			assert.Len(t, listing.Files, total)
+		})
+	}
+}

backend/dosya/dosya_parallel_test.go

@@ -0,0 +1,153 @@
+package dosya
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// parallelUploadServer answers the multipart endpoints without holding a
+// lock across a request, so parts really do overlap, and records what the
+// ordering rules below are about: how many parts were in flight at once, and
+// whether any part started before part 1 had finished.
+//
+// Part 1 matters because the API creates the R2 multipart upload (and, for a
+// new file, its id and key) on the first part it sees. Parts that race ahead
+// of part 1 would each create their own upload, and only one of those would
+// be completed.
+type parallelUploadServer struct {
+	t          *testing.T
+	resumable  string
+	partDelay  time.Duration
+	failPart   int // answer this part with a 400, 0 for none
+	mu         sync.Mutex
+	inFlight   int
+	maxFlight  int
+	part1Done  bool
+	earlyParts []int // parts that started before part 1 finished
+	parts      map[int]string
+	completes  int
+	aborts     int
+}
+
+func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
+	var body []byte
+	if r.Body != nil {
+		var err error
+		body, err = io.ReadAll(r.Body)
+		require.NoError(s.t, err)
+	}
+	path := r.URL.Path
+	switch {
+	case r.Method == "POST" && path == "/api/upload/init":
+		return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
+	case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
+		var n int
+		_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
+		require.NoError(s.t, err)
+
+		s.mu.Lock()
+		if n != 1 && !s.part1Done {
+			s.earlyParts = append(s.earlyParts, n)
+		}
+		s.inFlight++
+		s.maxFlight = max(s.maxFlight, s.inFlight)
+		s.mu.Unlock()
+
+		time.Sleep(s.partDelay)
+
+		s.mu.Lock()
+		s.inFlight--
+		if n == 1 {
+			s.part1Done = true
+		}
+		s.parts[n] = string(body)
+		fail := n == s.failPart
+		s.mu.Unlock()
+
+		if fail {
+			return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
+		}
+		return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
+	case r.Method == "POST" && path == "/api/upload/upl_1/complete":
+		s.mu.Lock()
+		s.completes++
+		s.mu.Unlock()
+		return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
+	case r.Method == "DELETE" && path == "/api/upload/upl_1":
+		s.mu.Lock()
+		s.aborts++
+		s.mu.Unlock()
+		return jsonResp(http.StatusOK, `{"ok":true}`), nil
+	}
+	s.t.Fatalf("unexpected request %s %s", r.Method, path)
+	return nil, nil
+}
+
+func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
+	return &parallelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
+}
+
+const tenParts = `{"part_size":2,"total_parts":10}`
+
+func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
+	srv := newParallelServer(t, tenParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 4
+	content := "aabbccddeeffgghhiijj"
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+
+	assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
+	assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
+	assert.Equal(t, 1, srv.completes)
+	for i := 1; i <= 10; i++ {
+		assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
+	}
+}
+
+func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
+	srv := newParallelServer(t, tenParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 1
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, 1, srv.maxFlight)
+	assert.Equal(t, 1, srv.completes)
+}
+
+func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
+	srv := newParallelServer(t, tenParts)
+	srv.failPart = 6
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 4
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "part 6")
+	assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
+}
+
+func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
+	// 20 bytes declared, 15 supplied: the short read is found by the
+	// reader, whichever parts are already in flight, and nothing completes.
+	srv := newParallelServer(t, tenParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 4
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	assert.Equal(t, 0, srv.completes)
+	assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
+}

backend/dosya/dosya_test.go

@@ -0,0 +1,15 @@
+// Test Dosya filesystem interface
+package dosya
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestDosya:",
+	})
+}

backend/dosya/dosya_upload_test.go

@@ -0,0 +1,247 @@
+package dosya
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// rtFunc adapts a function into an http.RoundTripper so the upload path
+// can be driven without a network. Unlike a real server it always reads
+// the whole request body and answers, which is exactly the misbehaviour
+// (accepting a short body as a complete upload) these tests guard against.
+type rtFunc func(*http.Request) (*http.Response, error)
+
+func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func jsonResp(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode:    status,
+		Status:        fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Header:        http.Header{"Content-Type": {"application/json"}},
+		Body:          io.NopCloser(strings.NewReader(body)),
+		ContentLength: int64(len(body)),
+	}
+}
+
+func newFakeFs(rt http.RoundTripper) *Fs {
+	ctx := context.Background()
+	f := &Fs{
+		name: "test",
+		opt:  Options{WorkspaceID: "ws_test"},
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(
+			pacer.MinSleep(time.Millisecond),
+			pacer.MaxSleep(time.Millisecond),
+			pacer.DecayConstant(decayConstant),
+			pacer.AttackConstant(attackConstant),
+		)),
+	}
+	f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
+	return f
+}
+
+// fakeUploadServer records what an upload sends and lets a test inject
+// failures per request
+type fakeUploadServer struct {
+	t          *testing.T
+	mu         sync.Mutex
+	resumable  string           // JSON for the init "resumable" field, "" for a single PUT
+	puts       [][]byte         // bodies of PUT /api/upload/<id>
+	parts      map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
+	completes  int
+	aborts     int             // DELETE /api/upload/<id>
+	failFirst  map[string]bool // path -> answer the first request with a 502
+	seenBefore map[string]bool
+}
+
+func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
+	return &fakeUploadServer{
+		t:          t,
+		resumable:  resumable,
+		parts:      map[int][][]byte{},
+		failFirst:  map[string]bool{},
+		seenBefore: map[string]bool{},
+	}
+}
+
+func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var body []byte
+	if r.Body != nil {
+		var err error
+		body, err = io.ReadAll(r.Body)
+		require.NoError(s.t, err)
+	}
+	path := r.URL.Path
+	// Bodies are recorded for every attempt, including the ones answered
+	// with a 502, so tests can see exactly what each attempt carried.
+	fail := s.failFirst[path] && !s.seenBefore[path]
+	s.seenBefore[path] = true
+	bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
+	switch {
+	case r.Method == "POST" && path == "/api/upload/init":
+		if s.resumable != "" {
+			return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
+		}
+		return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
+	case r.Method == "PUT" && path == "/api/upload/upl_1":
+		s.puts = append(s.puts, body)
+		if fail {
+			return bad, nil
+		}
+		return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
+	case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
+		var n int
+		_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
+		require.NoError(s.t, err)
+		s.parts[n] = append(s.parts[n], body)
+		if fail {
+			return bad, nil
+		}
+		return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
+	case r.Method == "POST" && path == "/api/upload/upl_1/complete":
+		s.completes++
+		return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
+	case r.Method == "DELETE" && path == "/api/upload/upl_1":
+		s.aborts++
+		return jsonResp(http.StatusOK, `{"ok":true}`), nil
+	}
+	s.t.Fatalf("unexpected request %s %s", r.Method, path)
+	return nil, nil
+}
+
+func TestUploadSmallSendsWholeBody(t *testing.T) {
+	srv := newFakeUploadServer(t, "")
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	content := strings.Repeat("a", 100)
+
+	resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "fil_1", resp.File.ID)
+	require.Len(t, srv.puts, 1)
+	assert.Equal(t, content, string(srv.puts[0]))
+}
+
+func TestUploadSmallRejectsShortSource(t *testing.T) {
+	// The source declares 100 bytes but only supplies 60. A server which
+	// accepts that as a complete upload must not turn into a success.
+	srv := newFakeUploadServer(t, "")
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
+}
+
+func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
+	// A retried PUT would resend an already-consumed reader, i.e. an
+	// empty body, which the server would store as an empty file. The
+	// single PUT therefore must not be retried at the low level.
+	srv := newFakeUploadServer(t, "")
+	srv.failFirst["/api/upload/upl_1"] = true
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	content := strings.Repeat("a", 100)
+	_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
+	assert.Equal(t, content, string(srv.puts[0]))
+}
+
+const threeParts = `{"part_size":4,"total_parts":3}`
+
+func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
+	srv := newFakeUploadServer(t, threeParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	content := "0123456789"
+
+	resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "fil_1", resp.File.ID)
+	assert.Equal(t, 1, srv.completes)
+	require.Len(t, srv.parts, 3)
+	assert.Equal(t, "0123", string(srv.parts[1][0]))
+	assert.Equal(t, "4567", string(srv.parts[2][0]))
+	assert.Equal(t, "89", string(srv.parts[3][0]))
+}
+
+func TestUploadMultipartRejectsShortSource(t *testing.T) {
+	// 10 bytes declared, 8 supplied: the last part must never be sent
+	// and the upload must not be completed.
+	srv := newFakeUploadServer(t, threeParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
+	assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
+	assert.Equal(t, 0, srv.completes, "upload must not be completed")
+}
+
+func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
+	// A part which fails transiently must be resent with all its bytes,
+	// not with whatever is left of a consumed reader.
+	srv := newFakeUploadServer(t, threeParts)
+	srv.failFirst["/api/upload/upl_1/part/2"] = true
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
+	assert.Equal(t, "4567", string(srv.parts[2][0]))
+	assert.Equal(t, "4567", string(srv.parts[2][1]))
+	assert.Equal(t, "89", string(srv.parts[3][0]))
+	assert.Equal(t, 1, srv.completes)
+}
+
+// An upload that fails after init has to hand its session back: the session
+// holds one of the workspace's upload slots for ten minutes otherwise, and
+// rclone's high-level retry opens a new one rather than resuming it.
+func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		resumable string
+		content   string
+		size      int64
+	}{
+		{"small file short source", "", "0123", 10},
+		{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := newFakeUploadServer(t, tc.resumable)
+			f := newFakeFs(rtFunc(srv.roundTrip))
+
+			_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
+				"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
+
+			require.Error(t, err)
+			assert.Equal(t, 1, srv.aborts)
+		})
+	}
+}
+
+// A successful upload must not abort anything
+func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
+	srv := newFakeUploadServer(t, "")
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
+		"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+
+	require.NoError(t, err)
+	assert.Equal(t, 0, srv.aborts)
+}

backend/dosya/dosya_workspace_hint_test.go

@@ -0,0 +1,73 @@
+package dosya
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Every by-id call names the workspace, so the API can find an id it minted
+// moments ago before its file_home index row lands.
+func TestByIDCallsSendWorkspaceHint(t *testing.T) {
+	seen := map[string]string{}
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
+		switch r.URL.Path {
+		case "/api/folders/fld_1":
+			return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+		case "/api/files/fil_1":
+			return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
+		case "/api/folders/fld_1/move":
+			return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
+		default:
+			return jsonResp(http.StatusOK, `{"ok":true}`), nil
+		}
+	}))
+	ctx := context.Background()
+
+	require.NoError(t, f.removeFolder(ctx, "fld_1", false))
+	require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
+	_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
+	require.NoError(t, err)
+	require.NoError(t, f.deleteFile(ctx, "fil_1"))
+	require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
+	require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
+	_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
+	require.NoError(t, err)
+	_, err = f.createShareLink(ctx, "fil_1", 0)
+	require.NoError(t, err)
+
+	for call, header := range seen {
+		assert.Equal(t, "ws_test", header, call)
+	}
+	assert.Len(t, seen, 8)
+}
+
+// The download's API request carries the hint; the presigned R2 fetch must not.
+func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
+	var apiHint, blobHint string
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/files/fil_1/download" {
+			apiHint = r.Header.Get("X-Dosya-Workspace")
+			http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
+			return
+		}
+		blobHint = r.Header.Get("X-Dosya-Workspace")
+		_, _ = w.Write([]byte("bytes"))
+	}))
+	defer srv.Close()
+	f := newTestFs(srv)
+	f.opt.APIURL = srv.URL
+
+	rc, err := f.downloadFile(context.Background(), "fil_1", nil)
+	require.NoError(t, err)
+	_ = rc.Close()
+
+	assert.Equal(t, "ws_test", apiHint)
+	assert.Equal(t, "", blobHint)
+}

backend/dosya/dosya_workspace_test.go

@@ -0,0 +1,52 @@
+package dosya
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeWorkspaceServer answers GET /api/workspaces with the given body
+func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		require.Equal(t, "/api/workspaces", r.URL.Path)
+		*calls++
+		return jsonResp(http.StatusOK, body), nil
+	}
+}
+
+func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
+	calls := 0
+	f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
+	f.opt.WorkspaceID = ""
+
+	require.NoError(t, f.findWorkspace(context.Background()))
+	assert.Equal(t, "ws_only", f.opt.WorkspaceID)
+	assert.Equal(t, 1, calls)
+}
+
+func TestFindWorkspaceListsTheChoices(t *testing.T) {
+	calls := 0
+	f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
+		`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
+	f.opt.WorkspaceID = ""
+
+	err := f.findWorkspace(context.Background())
+	require.Error(t, err)
+	// the ids have to be in the message: they are the only place the user
+	// can read them, which is the whole point of listing them
+	assert.Contains(t, err.Error(), "ws_one (One)")
+	assert.Contains(t, err.Error(), "ws_two (Two)")
+	assert.Empty(t, f.opt.WorkspaceID)
+}
+
+func TestFindWorkspaceWithNone(t *testing.T) {
+	calls := 0
+	f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
+	f.opt.WorkspaceID = ""
+
+	require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
+}

backend/drime/drime.go

@@ -692,13 +692,27 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // deleteObject removes an object by ID
 func (f *Fs) deleteObject(ctx context.Context, id string) error {
+	if f.opt.HardDelete {
+		// Deleting forever doesn't invalidate the server's cache of
+		// the parent folder listing, so the entry keeps being listed
+		// for about a second. Moving to the trash first does.
+		err := f.deleteEntry(ctx, id, false)
+		if err != nil {
+			return err
+		}
+	}
+	return f.deleteEntry(ctx, id, f.opt.HardDelete)
+}
+
+// deleteEntry moves the entry with ID to the trash or deletes it forever
+func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/file-entries/delete",
 	}
 	request := api.DeleteRequest{
 		EntryIDs:      []string{id},
-		DeleteForever: f.opt.HardDelete,
+		DeleteForever: deleteForever,
 	}
 	var result api.DeleteResponse
 	err := f.pacer.Call(func() (bool, error) {
@@ -1040,17 +1054,28 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
 }
 
 // copy and rename a file or folder to directoryID with leaf
-func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
-	// Can have duplicates so don't have to be careful here
-
+//
+// If existing is set it is removed once the copy has succeeded.
+func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
 	// Copy to dstDirectoryID first
 	info, err = f.copy(ctx, srcID, dstDirectoryID)
 	if err != nil {
 		return nil, err
 	}
 
+	// The server names the copy "name (1)" if the destination
+	// directory already has an entry called "name" and refuses to
+	// rename it to "name" until that entry is removed.
+	if existing != nil {
+		fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
+		err = existing.Remove(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Rename if required
-	if srcLeaf != dstLeaf {
+	if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
 		info, err = f.rename(ctx, info.ID.String(), dstLeaf)
 		if err != nil {
 			return nil, err
@@ -1074,8 +1099,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	srcLeaf := path.Base(srcObj.remote)
-
 	srcPath := srcObj.fs.rootSlash() + srcObj.remote
 	dstPath := f.rootSlash() + remote
 	if srcPath == dstPath {
@@ -1084,15 +1107,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 
 	// Find existing object
 	existingObj, err := f.NewObject(ctx, remote)
-	if err == nil {
-		defer func() {
-			// Don't remove existing object if returning an error
-			if err != nil {
-				return
-			}
-			fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
-			err = existingObj.Remove(ctx)
-		}()
+	if err != nil {
+		existingObj = nil
 	}
 
 	// Create temporary object
@@ -1102,7 +1118,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	// Copy the object
-	info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
+	info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
 	if err != nil {
 		return nil, err
 	}

backend/drive/drive.go

@@ -2766,7 +2766,12 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	}
 	trashedFiles := false
 	if check {
-		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, true, func(item *drive.File) bool {
+		// trashedFiles is only consulted to decide between trashing and hard deleting
+		// below, so when the directory is going to be trashed anyway there is no reason
+		// to enumerate trashed children: let the server filter them out instead of
+		// paging through them. Hard deletes still need to see them (#1040).
+		includeAll := !f.opt.UseTrash || f.opt.TrashedOnly
+		found, err := f.list(ctx, []string{directoryID}, "", false, false, f.opt.TrashedOnly, includeAll, func(item *drive.File) bool {
 			if !item.Trashed {
 				fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
 				return true

backend/drive/drive_internal_test.go

@@ -8,10 +8,14 @@ import (
 	"fmt"
 	"io"
 	"mime"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	stdsync "sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -25,13 +29,113 @@ import (
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
+	"google.golang.org/api/option"
 )
 
+func newPermissionTestFs(t *testing.T, handler http.Handler) *Fs {
+	t.Helper()
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+	service, err := drive.NewService(
+		context.Background(),
+		option.WithHTTPClient(server.Client()),
+		option.WithEndpoint(server.URL+"/"),
+	)
+	require.NoError(t, err)
+	return &Fs{
+		svc:           service,
+		pacer:         fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
+		permissionsMu: new(stdsync.Mutex),
+		permissions:   make(map[string]*drive.Permission),
+	}
+}
+
+func TestInternalGetPermissionConcurrent(t *testing.T) {
+	const requestCount = 2
+	arrived := make(chan string, requestCount)
+	release := make(chan struct{})
+	var releaseOnce stdsync.Once
+	releaseRequests := func() {
+		releaseOnce.Do(func() { close(release) })
+	}
+	defer releaseRequests()
+
+	f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		permissionID := path.Base(r.URL.Path)
+		arrived <- permissionID
+		<-release
+		w.Header().Set("Content-Type", "application/json")
+		_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
+		assert.NoError(t, err)
+	}))
+
+	type permissionResult struct {
+		permission *drive.Permission
+		inherited  bool
+		err        error
+	}
+	results := make(chan permissionResult, requestCount)
+	permissionIDs := []string{"permission-1", "permission-2"}
+	for _, permissionID := range permissionIDs {
+		go func() {
+			permission, inherited, err := f.getPermission(context.Background(), "file-id", permissionID, false)
+			results <- permissionResult{permission: permission, inherited: inherited, err: err}
+		}()
+	}
+
+	arrivedIDs := make(map[string]bool, requestCount)
+	timer := time.NewTimer(time.Second)
+	defer timer.Stop()
+	for len(arrivedIDs) < requestCount {
+		select {
+		case permissionID := <-arrived:
+			arrivedIDs[permissionID] = true
+		case <-timer.C:
+			releaseRequests()
+			t.Fatalf("timed out waiting for concurrent permission requests: got %d of %d", len(arrivedIDs), requestCount)
+		}
+	}
+	releaseRequests()
+
+	assert.Equal(t, map[string]bool{"permission-1": true, "permission-2": true}, arrivedIDs)
+	returnedIDs := make(map[string]bool, requestCount)
+	for range requestCount {
+		result := <-results
+		require.NoError(t, result.err)
+		require.NotNil(t, result.permission)
+		assert.False(t, result.inherited)
+		returnedIDs[result.permission.Id] = true
+	}
+	assert.Equal(t, arrivedIDs, returnedIDs)
+}
+
+func TestInternalGetPermissionCacheHit(t *testing.T) {
+	var requests atomic.Int32
+	f := newPermissionTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		permissionID := path.Base(r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		_, err := fmt.Fprintf(w, `{"id":%q,"role":"reader","type":"user"}`, permissionID)
+		assert.NoError(t, err)
+	}))
+
+	first, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
+	require.NoError(t, err)
+	assert.False(t, inherited)
+	second, inherited, err := f.getPermission(context.Background(), "file-id", "permission-1", true)
+	require.NoError(t, err)
+	assert.False(t, inherited)
+
+	assert.Same(t, first, second)
+	assert.Equal(t, int32(1), requests.Load())
+}
+
 func TestDriveScopes(t *testing.T) {
 	for _, test := range []struct {
 		in       string

backend/drive/metadata.go

@@ -106,10 +106,10 @@ var permissionsFields = googleapi.Field(strings.Join([]string{
 
 // getPermission returns permissions for the fileID and permissionID passed in
 func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
-	f.permissionsMu.Lock()
-	defer f.permissionsMu.Unlock()
 	if useCache {
+		f.permissionsMu.Lock()
 		perm = f.permissions[permissionID]
+		f.permissionsMu.Unlock()
 		if perm != nil {
 			return perm, false, nil
 		}
@@ -131,7 +131,9 @@ func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, use
 	cleanPermission(perm)
 
 	// cache the permission
+	f.permissionsMu.Lock()
 	f.permissions[permissionID] = perm
+	f.permissionsMu.Unlock()
 
 	return perm, inherited, err
 }

backend/drive/upload.go

@@ -11,7 +11,6 @@
 package drive
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -22,6 +21,7 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/readers"
 	"google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
@@ -112,8 +112,13 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 }
 
 // Make an http.Request for the range passed in
+//
+// The body is wrapped in readers.NoCloser so the transport can't upgrade it
+// to an io.Closer and close it after each attempt — the chunk buffer is
+// pool-backed, so an early close would free its pages and a retry would then
+// read a dead buffer. The upload loop owns the buffer's lifetime.
 func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
-	req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, body)
+	req, _ := http.NewRequestWithContext(ctx, "POST", rx.URI, readers.NoCloser(body))
 	req.ContentLength = reqSize
 	totalSize := "*"
 	if rx.ContentLength >= 0 {
@@ -168,37 +173,40 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 	start := int64(0)
 	var StatusCode int
 	var err error
-	buf := make([]byte, int(rx.f.opt.ChunkSize))
+	chunkSize := int64(rx.f.opt.ChunkSize)
 	for finished := false; !finished; {
-		var reqSize int64
-		var chunk io.ReadSeeker
+		reqSize := chunkSize
 		if rx.ContentLength >= 0 {
-			// If size known use repeatable reader for smoother bwlimit
 			if start >= rx.ContentLength {
 				break
 			}
-			reqSize = min(rx.ContentLength-start, int64(rx.f.opt.ChunkSize))
-			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
-		} else {
-			// If size unknown read into buffer
-			var n int
-			n, err = readers.ReadFill(rx.Media, buf)
+			reqSize = min(rx.ContentLength-start, chunkSize)
+		}
+
+		// Buffer the chunk in memory from the global pool so reads are
+		// repeatable for retries
+		rw := multipart.NewRW()
+		var n int64
+		n, err = io.CopyN(rw, rx.Media, reqSize)
+		if rx.ContentLength < 0 {
 			if err == io.EOF {
 				// Send the last chunk with the correct ContentLength
 				// otherwise Google doesn't know we've finished
-				rx.ContentLength = start + int64(n)
+				rx.ContentLength = start + n
 				finished = true
-			} else if err != nil {
-				return nil, err
+				err = nil
 			}
-			reqSize = int64(n)
-			chunk = bytes.NewReader(buf[:reqSize])
+			reqSize = n
+		}
+		if err != nil {
+			_ = rw.Close()
+			return nil, err
 		}
 
 		// Transfer the chunk
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
-			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
+			StatusCode, err = rx.transferChunk(ctx, start, rw, reqSize)
 			again, err := rx.f.shouldRetry(ctx, err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
@@ -206,9 +214,13 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 			}
 			return again, err
 		})
+		closeErr := rw.Close()
 		if err != nil {
 			return nil, err
 		}
+		if closeErr != nil {
+			return nil, closeErr
+		}
 
 		start += reqSize
 	}

backend/drive/upload_internal_test.go

@@ -0,0 +1,57 @@
+package drive
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestResumableUploadRetry checks that a chunk which fails with a 5xx is
+// retried successfully. The chunk buffer is pool-backed and implements
+// io.Closer, so if it reaches the http transport unwrapped the transport
+// closes it after the failed attempt, returning its pages to the pool, and
+// the retry then reads a freed buffer.
+func TestResumableUploadRetry(t *testing.T) {
+	content := bytes.Repeat([]byte("resumable"), 512)
+	var attempts atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, err := io.ReadAll(r.Body)
+		require.NoError(t, err)
+		if attempts.Add(1) == 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+		assert.Equal(t, content, body, "retried chunk should re-send the full chunk")
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"id":"fake-id","name":"remote"}`))
+	}))
+	defer server.Close()
+
+	f := &Fs{
+		pacer:  fs.NewPacer(context.Background(), pacer.NewGoogleDrive()),
+		client: server.Client(),
+	}
+	f.opt.ChunkSize = fs.SizeSuffix(len(content))
+	rx := &resumableUpload{
+		f:             f,
+		remote:        "remote",
+		URI:           server.URL,
+		Media:         bytes.NewReader(content),
+		MediaType:     "application/octet-stream",
+		ContentLength: int64(len(content)),
+	}
+	info, err := rx.Upload(context.Background())
+	require.NoError(t, err)
+	require.NotNil(t, info)
+	assert.Equal(t, "fake-id", info.Id)
+	assert.Equal(t, int32(2), attempts.Load(), "expected exactly one failed attempt and one retry")
+}

backend/dropbox/batcher.go

@@ -19,7 +19,7 @@ func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinish
 		Entries: items,
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
+		complete, err = f.srv.UploadSessionFinishBatchV2Context(ctx, arg)
 		if retry, err := shouldRetryExclude(ctx, err); !retry {
 			return retry, err
 		}

backend/dropbox/dropbox.go

@@ -51,6 +51,7 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -382,20 +383,20 @@ type Options struct {
 
 // Fs represents a remote dropbox server
 type Fs struct {
-	name           string         // name of this remote
-	root           string         // the path we are working on
-	opt            Options        // parsed options
-	ci             *fs.ConfigInfo // global config
-	features       *fs.Features   // optional features
-	srv            files.Client   // the connection to the dropbox server
-	svc            files.Client   // the connection to the dropbox server (unauthorized)
-	sharing        sharing.Client // as above, but for generating sharing links
-	users          users.Client   // as above, but for accessing user information
-	team           team.Client    // for the Teams API
-	slashRoot      string         // root with "/" prefix, lowercase
-	slashRootSlash string         // root with "/" prefix and postfix, lowercase
-	pacer          *fs.Pacer      // To pace the API calls
-	ns             string         // The namespace we are using or "" for none
+	name           string                // name of this remote
+	root           string                // the path we are working on
+	opt            Options               // parsed options
+	ci             *fs.ConfigInfo        // global config
+	features       *fs.Features          // optional features
+	srv            files.ContextClient   // the connection to the dropbox server
+	svc            files.ContextClient   // the connection to the dropbox server (unauthorized)
+	sharing        sharing.ContextClient // as above, but for generating sharing links
+	users          users.ContextClient   // as above, but for accessing user information
+	team           team.ContextClient    // for the Teams API
+	slashRoot      string                // root with "/" prefix
+	slashRootSlash string                // root with "/" prefix and postfix
+	pacer          *fs.Pacer             // To pace the API calls
+	ns             string                // The namespace we are using or "" for none
 	batcher        *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
 	exportExts     []exportExtension
 }
@@ -577,7 +578,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
-	f.team = team.New(cfg)
+	f.team = team.NewContext(cfg)
 
 	if opt.Impersonate != "" {
 		user := team.UserSelectorArg{
@@ -588,7 +589,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		members := []*team.UserSelectorArg{&user}
 		args := team.NewMembersGetInfoArgs(members)
 
-		memberIDs, err := f.team.MembersGetInfo(args)
+		memberIDs, err := f.team.MembersGetInfoContext(ctx, args)
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
@@ -603,10 +604,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		cfg.AsAdminID = opt.ImpersonateAdmin
 	}
 
-	f.srv = files.New(cfg)
-	f.svc = files.New(ucfg)
-	f.sharing = sharing.New(cfg)
-	f.users = users.New(cfg)
+	f.srv = files.NewContext(cfg)
+	f.svc = files.NewContext(ucfg)
+	f.sharing = sharing.NewContext(cfg)
+	f.users = users.NewContext(cfg)
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
 		ReadMimeType:            false,
@@ -633,13 +634,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			return f, nil // our root it empty so we probably want to list shared folders
 		}
 
-		dir := path.Dir(f.root)
-		if dir == "." {
-			dir = f.root
-		}
-
 		// root is not empty so we have find the right shared folder if it exists
-		id, err := f.findSharedFolder(ctx, dir)
+		id, err := f.findSharedFolder(ctx, sharedFolderName(f.root))
 		if err != nil {
 			// if we didn't find the specified shared folder we have to bail out here
 			return nil, err
@@ -673,7 +669,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		// If root starts with / then use the actual root
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
-			acc, err = f.users.GetCurrentAccount()
+			acc, err = f.users.GetCurrentAccountContext(ctx)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -693,7 +689,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	// See if the root is actually an object
 	if f.root != "" {
-		_, err = f.getFileMetadata(ctx, f.slashRoot)
+		_, _, err = f.getFileMetadata(ctx, f.slashRoot)
 		if err == nil {
 			newRoot := path.Dir(f.root)
 			if newRoot == "." {
@@ -728,15 +724,16 @@ func (f *Fs) setRoot(root string) {
 }
 
 type getMetadataResult struct {
-	entry    files.IsMetadata
-	notFound bool
-	err      error
+	entry              files.IsMetadata
+	remoteIsExportPath bool
+	notFound           bool
+	err                error
 }
 
 // getMetadata gets the metadata for a file or directory
 func (f *Fs) getMetadata(ctx context.Context, objPath string) (res getMetadataResult) {
 	res.err = f.pacer.Call(func() (bool, error) {
-		res.entry, res.err = f.srv.GetMetadata(&files.GetMetadataArg{
+		res.entry, res.err = f.srv.GetMetadataContext(ctx, &files.GetMetadataArg{
 			Path: f.opt.Enc.FromStandardPath(objPath),
 		})
 		return shouldRetry(ctx, res.err)
@@ -779,6 +776,7 @@ func (f *Fs) getMetadataForExt(ctx context.Context, filePath string, wantExportE
 				ch <- getMetadataResult{notFound: true}
 				return
 			}
+			res.remoteIsExportPath = true
 		}
 
 		// Return our real result or error
@@ -824,7 +822,7 @@ func (f *Fs) possibleMetadatas(ctx context.Context, filePath string) (ret []<-ch
 }
 
 // getFileMetadata gets the metadata for a file
-func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, error) {
+func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileMetadata, bool, error) {
 	var res getMetadataResult
 
 	// Try all possible metadatas
@@ -833,7 +831,7 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
 		res = <-ch
 
 		if res.err != nil {
-			return nil, res.err
+			return nil, false, res.err
 		}
 		if !res.notFound {
 			break
@@ -841,17 +839,17 @@ func (f *Fs) getFileMetadata(ctx context.Context, filePath string) (*files.FileM
 	}
 
 	if res.notFound {
-		return nil, fs.ErrorObjectNotFound
+		return nil, false, fs.ErrorObjectNotFound
 	}
 
 	fileInfo, ok := res.entry.(*files.FileMetadata)
 	if !ok {
 		if _, ok = res.entry.(*files.FolderMetadata); ok {
-			return nil, fs.ErrorIsDir
+			return nil, false, fs.ErrorIsDir
 		}
-		return nil, fs.ErrorNotAFile
+		return nil, false, fs.ErrorNotAFile
 	}
-	return fileInfo, nil
+	return fileInfo, res.remoteIsExportPath, nil
 }
 
 // getDirMetadata gets the metadata for a directory
@@ -910,7 +908,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
 				Limit: 100,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListFolders(&arg)
+				res, err = f.sharing.ListFoldersContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -922,7 +920,7 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
 				Cursor: res.Cursor,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListFoldersContinue(&arg)
+				res, err = f.sharing.ListFoldersContinueContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -945,13 +943,21 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e
 	return nil
 }
 
+// sharedFolderName returns the shared folder name in root, which is its first
+// path component. root must be the trimmed root as produced by setRoot (no
+// leading slash).
+func sharedFolderName(root string) string {
+	name, _, _ := strings.Cut(root, "/")
+	return name
+}
+
 // findSharedFolder find the id for a given shared folder name
 // somewhat annoyingly there is no endpoint to query a shared folder by it's name
 // so our only option is to iterate over all shared folders
 func (f *Fs) findSharedFolder(ctx context.Context, name string) (id string, err error) {
 	errFoundFile := errors.New("found file")
 	err = f.listSharedFolders(ctx, func(entry fs.DirEntry) error {
-		if entry.(*fs.Dir).Remote() == name {
+		if strings.EqualFold(entry.(*fs.Dir).Remote(), name) {
 			id = entry.(*fs.Dir).ID()
 			return errFoundFile
 		}
@@ -971,7 +977,7 @@ func (f *Fs) mountSharedFolder(ctx context.Context, id string) error {
 		SharedFolderId: id,
 	}
 	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.sharing.MountFolder(&arg)
+		_, err := f.sharing.MountFolderContext(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -988,7 +994,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 				Limit: 100,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListReceivedFiles(&arg)
+				res, err = f.sharing.ListReceivedFilesContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1000,7 +1006,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 				Cursor: res.Cursor,
 			}
 			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListReceivedFilesContinue(&arg)
+				res, err = f.sharing.ListReceivedFilesContinueContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1008,7 +1014,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 			}
 		}
 		for _, entry := range res.Entries {
-			entryPath := entry.Name
+			entryPath := f.opt.Enc.ToStandardName(entry.Name)
 			o := &Object{
 				fs:      f,
 				url:     entry.PreviewUrl,
@@ -1033,7 +1039,7 @@ func (f *Fs) listReceivedFiles(ctx context.Context, callback func(fs.DirEntry) e
 func (f *Fs) findSharedFile(ctx context.Context, name string) (o *Object, err error) {
 	errFoundFile := errors.New("found file")
 	err = f.listReceivedFiles(ctx, func(entry fs.DirEntry) error {
-		if entry.(*Object).remote == name {
+		if strings.EqualFold(entry.(*Object).remote, name) {
 			o = entry.(*Object)
 			return errFoundFile
 		}
@@ -1107,7 +1113,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 				arg.Path = "" // Specify root folder as empty string
 			}
 			err = f.pacer.Call(func() (bool, error) {
-				res, err = f.srv.ListFolder(arg)
+				res, err = f.srv.ListFolderContext(ctx, arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1125,7 +1131,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 				Cursor: res.Cursor,
 			}
 			err = f.pacer.Call(func() (bool, error) {
-				res, err = f.srv.ListFolderContinue(&arg)
+				res, err = f.srv.ListFolderContinueContext(ctx, &arg)
 				return shouldRetry(ctx, err)
 			})
 			if err != nil {
@@ -1162,7 +1168,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (
 						var sfMeta *sharing.SharedFolderMetadata
 						err = f.pacer.Call(func() (bool, error) {
 							var apiErr error
-							sfMeta, apiErr = f.sharing.GetFolderMetadata(sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
+							sfMeta, apiErr = f.sharing.GetFolderMetadataContext(ctx, sharing.NewGetMetadataArgs(folderInfo.SharingInfo.SharedFolderId))
 							return shouldRetry(ctx, apiErr)
 						})
 						if err != nil {
@@ -1249,7 +1255,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 		return cErr
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.CreateFolderV2(&arg2)
+		_, err = f.srv.CreateFolderV2Context(ctx, &arg2)
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -1277,7 +1283,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		}
 		var res *files.ListFolderResult
 		err = f.pacer.Call(func() (bool, error) {
-			res, err = f.srv.ListFolder(arg)
+			res, err = f.srv.ListFolderContext(ctx, arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1301,7 +1307,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	// remove it
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
+		_, err = f.srv.DeleteV2Context(ctx, &files.DeleteArg{Path: encRoot})
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -1360,7 +1366,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
-		result, err = f.srv.CopyV2(&arg)
+		result, err = f.srv.CopyV2Context(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1421,7 +1427,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
-		result, err = f.srv.MoveV2(&arg)
+		result, err = f.srv.MoveV2Context(ctx, &arg)
 		switch e := err.(type) {
 		case files.MoveV2APIError:
 			// There seems to be a bit of eventual consistency here which causes this to
@@ -1475,7 +1481,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
-		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+		linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
 		return shouldRetry(ctx, err)
 	})
 
@@ -1484,7 +1490,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		fs.Debugf(absPath, "can't create link with expiry, trying without")
 		createArg.Settings.Expires = nil
 		err = f.pacer.Call(func() (bool, error) {
-			linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+			linkRes, err = f.sharing.CreateSharedLinkWithSettingsContext(ctx, &createArg)
 			return shouldRetry(ctx, err)
 		})
 	}
@@ -1498,7 +1504,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		}
 		var listRes *sharing.ListSharedLinksResult
 		err = f.pacer.Call(func() (bool, error) {
-			listRes, err = f.sharing.ListSharedLinks(&listArg)
+			listRes, err = f.sharing.ListSharedLinksContext(ctx, &listArg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1559,7 +1565,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		},
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.MoveV2(&arg)
+		_, err = f.srv.MoveV2Context(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1573,7 +1579,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	var q *users.SpaceUsage
 	err = f.pacer.Call(func() (bool, error) {
-		q, err = f.users.GetSpaceUsage()
+		q, err = f.users.GetSpaceUsageContext(ctx)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1660,7 +1666,7 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
 			arg.Path = ""
 		}
 
-		startCursor, err = f.srv.ListFolderGetLatestCursor(arg)
+		startCursor, err = f.srv.ListFolderGetLatestCursorContext(ctx, arg)
 
 		return shouldRetry(ctx, err)
 	})
@@ -1670,6 +1676,34 @@ func (f *Fs) changeNotifyCursor(ctx context.Context) (cursor string, err error)
 	return startCursor.Cursor, nil
 }
 
+// trimPrefixFold returns s with the leading prefix removed, matching
+// case insensitively rune by rune so folded runes of differing UTF-8
+// length still match.
+//
+// If s is exactly the root that prefix names (prefix without its
+// trailing "/") it returns "". If prefix doesn't match it returns s
+// unchanged.
+func trimPrefixFold(s, prefix string) string {
+	if strings.HasSuffix(prefix, "/") && strings.EqualFold(s, strings.TrimSuffix(prefix, "/")) {
+		return ""
+	}
+
+	offset := 0
+	for prefix != "" {
+		if offset >= len(s) {
+			return s
+		}
+		sRune, sSize := utf8.DecodeRuneInString(s[offset:])
+		prefixRune, prefixSize := utf8.DecodeRuneInString(prefix)
+		if !strings.EqualFold(string(sRune), string(prefixRune)) {
+			return s
+		}
+		offset += sSize
+		prefix = prefix[prefixSize:]
+	}
+	return s[offset:]
+}
+
 func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), startCursor string) (newCursor string, err error) {
 	cursor := startCursor
 	var res *files.ListFolderLongpollResult
@@ -1693,7 +1727,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			Timeout: timeout,
 		}
 
-		res, err = f.svc.ListFolderLongpoll(&args)
+		res, err = f.svc.ListFolderLongpollContext(ctx, &args)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1716,7 +1750,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			Cursor: cursor,
 		}
 		err = f.pacer.Call(func() (bool, error) {
-			changeList, err = f.srv.ListFolderContinue(&arg)
+			changeList, err = f.srv.ListFolderContinueContext(ctx, &arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -1729,17 +1763,18 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			switch info := entry.(type) {
 			case *files.FolderMetadata:
 				entryType = fs.EntryDirectory
-				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
+				entryPath = info.PathDisplay
 			case *files.FileMetadata:
 				entryType = fs.EntryObject
-				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
+				entryPath = info.PathDisplay
 			case *files.DeletedMetadata:
 				entryType = fs.EntryObject
-				entryPath = strings.TrimPrefix(info.PathDisplay, f.slashRootSlash)
+				entryPath = info.PathDisplay
 			default:
 				fs.Errorf(entry, "dropbox ChangeNotify: ignoring unknown EntryType %T", entry)
 				continue
 			}
+			entryPath = trimPrefixFold(entryPath, f.slashRootSlash)
 
 			if entryPath != "" {
 				notifyFunc(f.opt.Enc.ToStandardPath(entryPath), entryType)
@@ -1851,7 +1886,7 @@ func (o *Object) Size() int64 {
 	return o.bytes
 }
 
-func (o *Object) setMetadataForExport(info *files.FileMetadata) {
+func (o *Object) setMetadataForExport(info *files.FileMetadata, remoteIsExportPath bool) {
 	o.bytes = -1
 	o.hash = ""
 
@@ -1872,8 +1907,10 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata) {
 		o.exportType = exportExportable
 		// get rid of any paper extension, if present
 		o.remote = strings.TrimSuffix(o.remote, paperExtension)
-		// add the export extension
-		o.remote += "." + string(exportExt)
+		if !remoteIsExportPath {
+			// add the export extension
+			o.remote += "." + string(exportExt)
+		}
 	}
 }
 
@@ -1881,19 +1918,23 @@ func (o *Object) setMetadataForExport(info *files.FileMetadata) {
 //
 // This isn't a complete set of metadata and has an inaccurate date
 func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
+	return o.setMetadataFromEntryWithExportPath(info, false)
+}
+
+func (o *Object) setMetadataFromEntryWithExportPath(info *files.FileMetadata, remoteIsExportPath bool) error {
 	o.id = info.Id
 	o.bytes = int64(info.Size)
 	o.modTime = time.Time(info.ClientModified)
 	o.hash = info.ContentHash
 
 	if !info.IsDownloadable {
-		o.setMetadataForExport(info)
+		o.setMetadataForExport(info, remoteIsExportPath)
 	}
 	return nil
 }
 
 // Reads the entry for a file from dropbox
-func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, error) {
+func (o *Object) readEntry(ctx context.Context) (*files.FileMetadata, bool, error) {
 	return o.fs.getFileMetadata(ctx, o.remotePath())
 }
 
@@ -1903,11 +1944,11 @@ func (o *Object) readEntryAndSetMetadata(ctx context.Context) error {
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	entry, err := o.readEntry(ctx)
+	entry, remoteIsExportPath, err := o.readEntry(ctx)
 	if err != nil {
 		return err
 	}
-	return o.setMetadataFromEntry(entry)
+	return o.setMetadataFromEntryWithExportPath(entry, remoteIsExportPath)
 }
 
 // Returns the remote path for the object
@@ -1961,7 +2002,7 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) {
 	arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)}
 	var exportResult *files.ExportResult
 	err = o.fs.pacer.Call(func() (bool, error) {
-		exportResult, in, err = o.fs.srv.Export(&arg)
+		exportResult, in, err = o.fs.srv.ExportContext(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -1983,7 +2024,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			Url: o.url,
 		}
 		err = o.fs.pacer.Call(func() (bool, error) {
-			_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
+			_, in, err = o.fs.sharing.GetSharedLinkFileContext(ctx, &arg)
 			return shouldRetry(ctx, err)
 		})
 		if err != nil {
@@ -2003,7 +2044,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		ExtraHeaders: headers,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, in, err = o.fs.srv.Download(&arg)
+		_, in, err = o.fs.srv.DownloadContext(ctx, &arg)
 		return shouldRetry(ctx, err)
 	})
 
@@ -2027,7 +2068,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 	// start upload
 	var res *files.UploadSessionStartResult
 	err = o.fs.pacer.Call(func() (bool, error) {
-		res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, nil)
+		res, err = o.fs.srv.UploadSessionStartContext(ctx, &files.UploadSessionStartArg{}, nil)
 		return shouldRetry(ctx, err)
 	})
 	if err != nil {
@@ -2042,11 +2083,6 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 
 	// write chunks
 	in := readers.NewCountingReader(in0)
-	bufSize := chunkSize
-	if size >= 0 && size < bufSize {
-		bufSize = size
-	}
-	buf := make([]byte, int(bufSize))
 	cursor := files.UploadSessionCursor{
 		SessionId: res.SessionId,
 		Offset:    0,
@@ -2066,14 +2102,21 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 			fs.Debugf(o, "Uploading chunk %d/%d", currentChunk, chunks)
 		}
 
-		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
+		// Buffer the chunk in memory from the global pool for retries
+		rw := multipart.NewRW()
+		var n int64
+		n, err = io.CopyN(rw, in, chunkSize)
+		if err != nil && err != io.EOF {
+			_ = rw.Close()
+			return nil, err
+		}
 		skip := int64(0)
 		err = o.fs.pacer.Call(func() (bool, error) {
 			// seek to the start in case this is a retry
-			if _, err = chunk.Seek(skip, io.SeekStart); err != nil {
+			if _, err = rw.Seek(skip, io.SeekStart); err != nil {
 				return false, err
 			}
-			err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
+			err = o.fs.srv.UploadSessionAppendV2Context(ctx, &appendArg, rw)
 			// after session is started, we retry everything
 			if err != nil {
 				// Check for incorrect offset error and retry with new offset
@@ -2085,10 +2128,10 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 						what := fmt.Sprintf("incorrect offset error received: sent %d, need %d, skip %d", cursor.Offset, correctOffset, skip)
 						if skip < 0 {
 							return false, fmt.Errorf("can't seek backwards to correct offset: %s", what)
-						} else if skip == chunkSize {
+						} else if skip == n {
 							fs.Debugf(o, "%s: chunk received OK - continuing", what)
 							return false, nil
-						} else if skip > chunkSize {
+						} else if skip > n {
 							// This error should never happen
 							return false, fmt.Errorf("can't seek forwards by more than a chunk to correct offset: %s", what)
 						}
@@ -2098,11 +2141,27 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 					}
 				}
 			}
+			// Don't waste the low level retries if the context has
+			// been cancelled
+			if fserrors.ContextError(ctx, &err) {
+				return false, err
+			}
 			return err != nil, err
 		})
+		closeErr := rw.Close()
 		if err != nil {
 			return nil, err
 		}
+		if closeErr != nil {
+			return nil, closeErr
+		}
+		if size >= 0 {
+			// Check for sources which truncate early
+			expected := min(uint64(currentChunk)*uint64(chunkSize), uint64(size))
+			if in.BytesRead() < expected {
+				return nil, fmt.Errorf("expected %d bytes in input, but only read %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
+			}
+		}
 		if appendArg.Close {
 			break
 		}
@@ -2132,7 +2191,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		entry, err = o.fs.srv.UploadSessionFinish(args, nil)
+		entry, err = o.fs.srv.UploadSessionFinishContext(ctx, args, nil)
 		if retry, err := shouldRetryExclude(ctx, err); !retry {
 			return retry, err
 		}
@@ -2200,7 +2259,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		entry, err = o.uploadChunked(ctx, in, commitInfo, size)
 	} else {
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			entry, err = o.fs.srv.Upload(&files.UploadArg{CommitInfo: *commitInfo}, in)
+			entry, err = o.fs.srv.UploadContext(ctx, &files.UploadArg{CommitInfo: *commitInfo}, in)
 			return shouldRetry(ctx, err)
 		})
 	}
@@ -2225,7 +2284,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		return errNotSupportedInSharedMode
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
+		_, err = o.fs.srv.DeleteV2Context(ctx, &files.DeleteArg{
 			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
 		})
 		return shouldRetry(ctx, err)

backend/dropbox/dropbox_internal_test.go

@@ -2,17 +2,155 @@ package dropbox
 
 import (
 	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
 	"io"
+	"net/http"
+	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/batcher"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+type paperMetadataClient struct {
+	files.ContextClient
+	info *files.FileMetadata
+}
+
+func (c paperMetadataClient) GetMetadataContext(ctx context.Context, arg *files.GetMetadataArg) (files.IsMetadata, error) {
+	if arg.Path == "document" {
+		return c.info, nil
+	}
+	return nil, files.GetMetadataAPIError{
+		APIError: dropbox.APIError{ErrorSummary: "path/not_found/"},
+		EndpointError: &files.GetMetadataError{
+			Tagged: dropbox.Tagged{Tag: files.GetMetadataErrorPath},
+			Path: &files.LookupError{
+				Tagged: dropbox.Tagged{Tag: files.LookupErrorNotFound},
+			},
+		},
+	}
+}
+
+func TestInternalGetMetadataCancellation(t *testing.T) {
+	requestStarted := make(chan struct{})
+	releaseRequest := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		close(requestStarted)
+		<-releaseRequest
+	}))
+	defer server.Close()
+	defer close(releaseRequest)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	f := &Fs{
+		srv: files.NewContext(dropbox.Config{
+			Client: server.Client(),
+			URLGenerator: func(hostType string, namespace string, route string) string {
+				return server.URL
+			},
+		}),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(0), pacer.MaxSleep(time.Millisecond))),
+	}
+
+	result := make(chan getMetadataResult, 1)
+	go func() {
+		result <- f.getMetadata(ctx, "/file")
+	}()
+
+	select {
+	case <-requestStarted:
+	case <-time.After(time.Second):
+		t.Fatal("Dropbox request did not start")
+	}
+	cancel()
+
+	select {
+	case res := <-result:
+		require.ErrorIs(t, res.err, context.Canceled)
+	case <-time.After(time.Second):
+		t.Fatal("Dropbox request did not observe cancellation")
+	}
+}
+
+type changeNotifyClient struct {
+	files.ContextClient
+	entries []files.IsMetadata
+}
+
+func (c changeNotifyClient) ListFolderLongpollContext(context.Context, *files.ListFolderLongpollArg) (*files.ListFolderLongpollResult, error) {
+	return files.NewListFolderLongpollResult(true), nil
+}
+
+func (c changeNotifyClient) ListFolderContinueContext(context.Context, *files.ListFolderContinueArg) (*files.ListFolderResult, error) {
+	return files.NewListFolderResult(c.entries, "next", false), nil
+}
+
+func TestTrimPrefixFold(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		path   string
+		prefix string
+		want   string
+	}{
+		{name: "exact casing", path: "/Docs/Sub", prefix: "/Docs/", want: "Sub"},
+		{name: "different casing", path: "/docs/Sub", prefix: "/Docs/", want: "Sub"},
+		{name: "different UTF-8 lengths", path: "/K/Sub", prefix: "/K/", want: "Sub"},
+		{name: "exact root", path: "/docs", prefix: "/Docs/", want: ""},
+		{name: "sibling boundary", path: "/Docs2/File", prefix: "/Docs/", want: "/Docs2/File"},
+		{name: "root remote", path: "/File", prefix: "/", want: "File"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.want, trimPrefixFold(test.path, test.prefix))
+		})
+	}
+}
+
+func TestChangeNotifyTrimsRootCaseInsensitively(t *testing.T) {
+	ctx := context.Background()
+	client := changeNotifyClient{entries: []files.IsMetadata{
+		&files.FolderMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub"}},
+		&files.FileMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Sub/File.TXT"}},
+		&files.DeletedMetadata{Metadata: files.Metadata{PathDisplay: "/docs/Gone.md"}},
+	}}
+	f := &Fs{
+		ci:             fs.GetConfig(ctx),
+		srv:            client,
+		svc:            client,
+		slashRootSlash: "/Docs/",
+		pacer:          fs.NewPacer(ctx, pacer.NewDefault()),
+	}
+
+	type notification struct {
+		path      string
+		entryType fs.EntryType
+	}
+	var notifications []notification
+	cursor, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
+		notifications = append(notifications, notification{path: path, entryType: entryType})
+	}, "start")
+	require.NoError(t, err)
+	assert.Equal(t, "next", cursor)
+	assert.Equal(t, []notification{
+		{path: "Sub", entryType: fs.EntryDirectory},
+		{path: "Sub/File.TXT", entryType: fs.EntryObject},
+		{path: "Gone.md", entryType: fs.EntryObject},
+	}, notifications)
+}
+
 func TestInternalCheckPathLength(t *testing.T) {
 	rep := func(n int, r rune) (out string) {
 		rs := make([]rune, n)
@@ -50,6 +188,170 @@ func TestInternalCheckPathLength(t *testing.T) {
 	}
 }
 
+func TestInternalSharedFolderName(t *testing.T) {
+	for _, test := range []struct {
+		root string
+		want string
+	}{
+		{root: "", want: ""},
+		{root: "SharedFolder", want: "SharedFolder"},
+		{root: "SharedFolder/subdir", want: "SharedFolder"},
+		{root: "SharedFolder/subdir/deeper", want: "SharedFolder"},
+		{root: "SharedFolder/subdir/deeper/deepest", want: "SharedFolder"},
+	} {
+		assert.Equal(t, test.want, sharedFolderName(test.root), test.root)
+	}
+}
+
+func TestPaperExportRemote(t *testing.T) {
+	ctx := context.Background()
+	info := &files.FileMetadata{
+		ExportInfo: &files.ExportInfo{ExportAs: "markdown"},
+	}
+	f := &Fs{
+		exportExts: []exportExtension{"md"},
+		pacer:      fs.NewPacer(ctx, pacer.NewDefault()),
+		srv:        paperMetadataClient{info: info},
+	}
+
+	direct, err := f.NewObject(ctx, "document.md")
+	require.NoError(t, err)
+	assert.Equal(t, "document.md", direct.Remote())
+
+	listed, err := f.newObjectWithInfo(ctx, "document.md", info)
+	require.NoError(t, err)
+	assert.Equal(t, "document.md.md", listed.Remote())
+
+	legacy, err := f.newObjectWithInfo(ctx, "document.paper", info)
+	require.NoError(t, err)
+	assert.Equal(t, "document.md", legacy.Remote())
+}
+
+// uploadSessionClient is a mock files.ContextClient which records the
+// chunked upload calls made to it
+type uploadSessionClient struct {
+	files.ContextClient
+	appends      int    // number of UploadSessionAppendV2Context calls
+	maxAppends   int    // fail the append after this many calls to stop runaway loops
+	bytesWritten int64  // bytes received by UploadSessionAppendV2Context
+	finishCalled bool   // set if UploadSessionFinishContext was called
+	appended     func() // if set, called after each successful append
+}
+
+var errTooManyAppends = errors.New("too many appends - upload looping?")
+
+func (c *uploadSessionClient) UploadSessionStartContext(ctx context.Context, arg *files.UploadSessionStartArg, content io.Reader) (*files.UploadSessionStartResult, error) {
+	return &files.UploadSessionStartResult{SessionId: "session"}, nil
+}
+
+func (c *uploadSessionClient) UploadSessionAppendV2Context(ctx context.Context, arg *files.UploadSessionAppendArg, content io.Reader) error {
+	// the real client fails the request if the context is cancelled
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	c.appends++
+	if c.appends > c.maxAppends {
+		return errTooManyAppends
+	}
+	n, err := io.Copy(io.Discard, content)
+	if err != nil {
+		return err
+	}
+	c.bytesWritten += n
+	if c.appended != nil {
+		c.appended()
+	}
+	return nil
+}
+
+func (c *uploadSessionClient) UploadSessionFinishContext(ctx context.Context, arg *files.UploadSessionFinishArg, content io.Reader) (*files.FileMetadata, error) {
+	c.finishCalled = true
+	return &files.FileMetadata{}, nil
+}
+
+// newUploadTestFs makes an Fs with a mock srv for testing uploadChunked
+func newUploadTestFs(t *testing.T, srv files.ContextClient, chunkSize fs.SizeSuffix) *Fs {
+	ctx := context.Background()
+	f := &Fs{
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond))),
+		srv:   srv,
+	}
+	f.opt.ChunkSize = chunkSize
+	batcherOptions := defaultBatcherOptions
+	batcherOptions.Mode = "off"
+	var err error
+	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
+	require.NoError(t, err)
+	return f
+}
+
+// endlessReader supplies bytes forever
+type endlessReader struct{}
+
+func (endlessReader) Read(p []byte) (int, error) {
+	for i := range p {
+		p[i] = 'x'
+	}
+	return len(p), nil
+}
+
+func TestUploadChunkedEarlyEOF(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("MultiChunk", func(t *testing.T) {
+		// The declared size spans 4 chunks but the source ends after 1.5
+		client := &uploadSessionClient{maxAppends: 8}
+		f := newUploadTestFs(t, client, 100)
+		o := &Object{fs: f, remote: "test.bin"}
+		_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
+		require.Error(t, err)
+		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+		assert.False(t, client.finishCalled, "must not commit a truncated upload")
+	})
+
+	t.Run("SingleChunk", func(t *testing.T) {
+		// The declared size fits in one chunk but the source ends early
+		client := &uploadSessionClient{maxAppends: 8}
+		f := newUploadTestFs(t, client, 500)
+		o := &Object{fs: f, remote: "test.bin"}
+		_, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 150)), files.NewCommitInfo("/test.bin"), 400)
+		require.Error(t, err)
+		assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+		assert.False(t, client.finishCalled, "must not commit a truncated upload")
+	})
+
+	t.Run("Complete", func(t *testing.T) {
+		// A source which supplies exactly the declared size uploads OK
+		client := &uploadSessionClient{maxAppends: 8}
+		f := newUploadTestFs(t, client, 100)
+		o := &Object{fs: f, remote: "test.bin"}
+		entry, err := o.uploadChunked(ctx, strings.NewReader(strings.Repeat("a", 250)), files.NewCommitInfo("/test.bin"), 250)
+		require.NoError(t, err)
+		require.NotNil(t, entry)
+		assert.True(t, client.finishCalled)
+		assert.Equal(t, int64(250), client.bytesWritten)
+	})
+}
+
+func TestUploadChunkedCancel(t *testing.T) {
+	// Cancelling the context must stop the upload even though every
+	// append is succeeding
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	client := &uploadSessionClient{maxAppends: 8}
+	client.appended = func() {
+		if client.appends == 2 {
+			cancel()
+		}
+	}
+	f := newUploadTestFs(t, client, 100)
+	o := &Object{fs: f, remote: "test.bin"}
+	_, err := o.uploadChunked(ctx, endlessReader{}, files.NewCommitInfo("/test.bin"), -1)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, context.Canceled)
+	assert.False(t, client.finishCalled)
+}
+
 func (f *Fs) importPaperForTest(t *testing.T) {
 	content := `# test doc
 
@@ -64,7 +366,7 @@ Lorem ipsum __dolor__ sit amet
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		reader := strings.NewReader(content)
-		_, err = f.srv.PaperCreate(&arg, reader)
+		_, err = f.srv.PaperCreateContext(context.Background(), &arg, reader)
 		return shouldRetry(context.Background(), err)
 	})
 	require.NoError(t, err)
@@ -100,3 +402,136 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// newSharingTestFs builds a minimal *Fs whose sharing client talks to a
+// local mock server instead of the real Dropbox API.
+func newSharingTestFs(t *testing.T, handler http.HandlerFunc) *Fs {
+	t.Helper()
+	ts := httptest.NewServer(handler)
+	t.Cleanup(ts.Close)
+
+	cfg := dropbox.Config{
+		Client: &http.Client{Transport: http.DefaultTransport},
+		URLGenerator: func(hostType, namespace, route string) string {
+			return fmt.Sprintf("%s/2/%s/%s", ts.URL, namespace, route)
+		},
+	}
+
+	return &Fs{
+		opt: Options{
+			Enc: encoder.Base |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDel |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8,
+		},
+		sharing: sharing.NewContext(cfg),
+		pacer:   fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond))),
+	}
+}
+
+func mustParseDBXTime(t *testing.T, s string) *dropbox.DBXTime {
+	t.Helper()
+	tm, err := time.Parse(time.RFC3339, s)
+	require.NoError(t, err)
+	return (*dropbox.DBXTime)(&tm)
+}
+
+func receivedFilesHandler(t *testing.T, rawName string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.URL.Path, "list_received_files") {
+			http.NotFound(w, r)
+			return
+		}
+		resp := sharing.ListFilesResult{
+			Entries: []*sharing.SharedFileMetadata{{
+				Id:          "id:123",
+				Name:        rawName,
+				PreviewUrl:  "https://dropbox.com/preview/123",
+				Policy:      &sharing.FolderPolicy{},
+				TimeInvited: mustParseDBXTime(t, "2024-01-01T00:00:00Z"),
+			}},
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(resp)
+	}
+}
+
+// TestListReceivedFilesDecodesName confirms received-file names are decoded
+// via Enc.ToStandardName before being returned, matching listSharedFolders'
+// handling of shared-folder names.
+func TestListReceivedFilesDecodesName(t *testing.T) {
+	const rawName = "report.txt␠"
+	f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
+
+	wantName := f.opt.Enc.ToStandardName(rawName)
+	require.NotEqual(t, rawName, wantName)
+
+	var gotNames []string
+	err := f.listReceivedFiles(context.Background(), func(entry fs.DirEntry) error {
+		gotNames = append(gotNames, entry.Remote())
+		return nil
+	})
+	require.NoError(t, err)
+	require.Len(t, gotNames, 1)
+	assert.Equal(t, wantName, gotNames[0])
+}
+
+// TestFindSharedFileResolvesDecodedName confirms findSharedFile can look up
+// a received file by its standard, decoded rclone-visible name.
+func TestFindSharedFileResolvesDecodedName(t *testing.T) {
+	const rawName = "report.txt␠"
+	f := newSharingTestFs(t, receivedFilesHandler(t, rawName))
+
+	encodedName := f.opt.Enc.ToStandardName(rawName)
+	require.NotEqual(t, rawName, encodedName)
+
+	o, err := f.findSharedFile(context.Background(), encodedName)
+	require.NoError(t, err)
+	assert.Equal(t, encodedName, o.remote)
+}
+
+// sharedFoldersHandler serves a single shared folder from list_folders.
+func sharedFoldersHandler(name, id string) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if !strings.Contains(r.URL.Path, "list_folders") {
+			http.NotFound(w, r)
+			return
+		}
+		resp := sharing.ListFoldersResult{
+			Entries: []*sharing.SharedFolderMetadata{{Name: name, SharedFolderId: id}},
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(resp)
+	}
+}
+
+// The Dropbox backend advertises CaseInsensitive: true, so a shared folder
+// lookup must match regardless of the case of the requested name.
+func TestInternalFindSharedFolderCaseInsensitive(t *testing.T) {
+	f := newSharingTestFs(t, sharedFoldersHandler("TestFolder", "folder-id"))
+
+	for _, name := range []string{"TestFolder", "testfolder", "TESTFOLDER"} {
+		id, err := f.findSharedFolder(context.Background(), name)
+		require.NoError(t, err, name)
+		assert.Equal(t, "folder-id", id, name)
+	}
+
+	_, err := f.findSharedFolder(context.Background(), "no-such-folder")
+	assert.ErrorIs(t, err, fs.ErrorDirNotFound)
+}
+
+// The Dropbox backend advertises CaseInsensitive: true, so a received file
+// lookup must match regardless of the case of the requested name.
+func TestInternalFindSharedFileCaseInsensitive(t *testing.T) {
+	f := newSharingTestFs(t, receivedFilesHandler(t, "TestFile.txt"))
+
+	for _, name := range []string{"TestFile.txt", "testfile.txt", "TESTFILE.TXT"} {
+		o, err := f.findSharedFile(context.Background(), name)
+		require.NoError(t, err, name)
+		assert.Equal(t, "TestFile.txt", o.remote, name)
+	}
+
+	_, err := f.findSharedFile(context.Background(), "no-such-file.txt")
+	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}

backend/filelu/filelu_client.go

@@ -16,13 +16,16 @@ import (
 
 // multipartInit starts a new multipart upload and returns server details.
 func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
+	encodedFolderPath := f.fromStandardPath(folderPath)
+	encodedFilename := f.fromStandardPath(filename)
+
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   "/multipart/init",
 		Parameters: url.Values{
 			"key":         {f.opt.Key},
-			"filename":    {filename},
-			"folder_path": {folderPath},
+			"filename":    {encodedFilename},
+			"folder_path": {encodedFolderPath},
 		},
 	}
 

backend/filelu/filelu_file_uploader.go

@@ -1,7 +1,6 @@
 package filelu
 
 import (
-	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -11,9 +10,11 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 
 	"github.com/rclone/rclone/fs"
+	rclonemultipart "github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -25,7 +26,9 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
 	}
 
 	if dir != "" {
-		_ = f.Mkdir(ctx, dir)
+		if _, err := f.createFolder(ctx, dir); err != nil {
+			return fmt.Errorf("failed to create multipart folder: %w", err)
+		}
 	}
 
 	folder := strings.Trim(dir, "/")
@@ -45,40 +48,32 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
 	server := initResp.Result.Server
 	objectPath := initResp.Result.ObjectPath
 
-	chunkSize := int(f.opt.ChunkSize)
-	buf := make([]byte, 0, chunkSize)
-	tmp := make([]byte, 1024*1024)
-	partNo := 1
-
-	for {
-		n, errRead := in.Read(tmp)
+	chunkSize := int64(f.opt.ChunkSize)
+	if chunkSize <= 0 {
+		return fmt.Errorf("multipart upload: chunk_size must be positive: %v", f.opt.ChunkSize)
+	}
+	for partNo := 1; ; partNo++ {
+		// Buffer the part in memory from the global pool so it can be
+		// re-sent on retry
+		rw := rclonemultipart.NewRW()
+		n, err := io.CopyN(rw, in, chunkSize)
+		if err != nil && err != io.EOF {
+			_ = rw.Close()
+			return fmt.Errorf("read failed: %w", err)
+		}
 		if n > 0 {
-			buf = append(buf, tmp[:n]...)
-
-			// If buffer reached chunkSize, upload a full part
-			if len(buf) >= chunkSize {
-				err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
-				if err != nil {
-					return fmt.Errorf("upload part %d failed: %w", partNo, err)
-				}
-				partNo++
-				buf = buf[:0]
+			uploadErr := f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, rw, n)
+			if uploadErr != nil {
+				_ = rw.Close()
+				return fmt.Errorf("upload part %d failed: %w", partNo, uploadErr)
 			}
 		}
-
-		if errRead == io.EOF {
+		if closeErr := rw.Close(); closeErr != nil {
+			return closeErr
+		}
+		if err == io.EOF {
 			break
 		}
-		if errRead != nil {
-			return fmt.Errorf("read failed: %w", errRead)
-		}
-	}
-
-	if len(buf) > 0 {
-		err = f.uploadPart(ctx, server, uploadID, sessID, objectPath, partNo, bytes.NewReader(buf))
-		if err != nil {
-			return fmt.Errorf("upload part %d failed: %w", partNo, err)
-		}
 	}
 
 	err = f.completeMultipart(ctx, server, uploadID, sessID, objectPath)
@@ -89,31 +84,40 @@ func (f *Fs) multipartUpload(ctx context.Context, in io.Reader, remote string) e
 	return nil
 }
 
-// uploadPart sends a single multipart chunk to the upload server.
-func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.Reader) error {
-	url := fmt.Sprintf("%s?partNumber=%d&uploadId=%s", server, partNo, uploadID)
-
-	req, err := http.NewRequestWithContext(ctx, "PUT", url, r)
-	if err != nil {
-		return err
+// uploadPart sends a single multipart chunk of size bytes to the upload server.
+func (f *Fs) uploadPart(ctx context.Context, server, uploadID, sessID, objectPath string, partNo int, r io.ReadSeeker, size int64) error {
+	opts := rest.Opts{
+		Method:  "PUT",
+		RootURL: server,
+		Parameters: url.Values{
+			"partNumber": {strconv.Itoa(partNo)},
+			"uploadId":   {uploadID},
+		},
+		Body:          r,
+		ContentLength: &size,
+		ExtraHeaders: map[string]string{
+			"X-RC-Upload-Id": uploadID,
+			"X-RC-Part-No":   strconv.Itoa(partNo),
+			"X-Sess-ID":      sessID,
+			"X-Object-Path":  objectPath,
+		},
+		NoResponse: true,
 	}
-
-	req.Header.Set("X-RC-Upload-Id", uploadID)
-	req.Header.Set("X-RC-Part-No", fmt.Sprintf("%d", partNo))
-	req.Header.Set("X-Sess-ID", sessID)
-	req.Header.Set("X-Object-Path", objectPath)
-
-	resp, err := f.client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer func() { _ = resp.Body.Close() }()
-
-	if resp.StatusCode != 200 {
-		return fmt.Errorf("uploadPart failed: %s", resp.Status)
-	}
-
-	return nil
+	return f.pacer.Call(func() (bool, error) {
+		// rewind the pooled buffer so each attempt sends the whole chunk
+		_, err := r.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		resp, err := f.srv.Call(ctx, &opts)
+		if err != nil {
+			if resp != nil {
+				return shouldRetryHTTP(resp.StatusCode), fmt.Errorf("uploadPart failed: %w", err)
+			}
+			return shouldRetry(err), err
+		}
+		return false, nil
+	})
 }
 
 // uploadFile uploads a file to FileLu

backend/filelu/filelu_internal_test.go

@@ -0,0 +1,29 @@
+package filelu
+
+import (
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// SetUploadChunkSize changes the configured chunk size, returning the old value.
+//
+// It is only called by the integration tests while no transfer is in progress.
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	return old, nil
+}
+
+// SetUploadCutoff changes the configured upload cutoff, returning the old value.
+//
+// It is only called by the integration tests while no transfer is in progress.
+func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
+	return old, nil
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)

backend/filelu/filelu_object.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -141,6 +142,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 // Update updates the object with new data
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	size := src.Size()
+	counter := readers.NewCountingReader(in)
+	in = counter
 
 	if size <= int64(o.fs.opt.UploadCutoff) {
 		err := o.fs.uploadFile(ctx, in, o.remote)
@@ -155,6 +158,15 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload
+	if size >= 0 && int64(counter.BytesRead()) != size {
+		if removeErr := o.Remove(ctx); removeErr != nil {
+			fs.Errorf(o, "Failed to remove partially transferred object: %v", removeErr)
+		}
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	o.size = size
 	o.modTime = src.ModTime(ctx)
 	return nil

backend/filelu/filelu_test.go

@@ -12,5 +12,8 @@ func TestIntegration(t *testing.T) {
 		RemoteName:      "TestFileLu:",
 		NilObject:       nil,
 		SkipInvalidUTF8: true,
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: 1,
+		},
 	})
 }

backend/filescom/filescom.go

@@ -19,6 +19,7 @@ import (
 	"github.com/Files-com/files-sdk-go/v3/file"
 	file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
 	"github.com/Files-com/files-sdk-go/v3/folder"
+	files_sdk_lib "github.com/Files-com/files-sdk-go/v3/lib"
 	"github.com/Files-com/files-sdk-go/v3/session"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -176,6 +177,13 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
 		}
 	}
 
+	// Errors from the upload storage servers are of this type
+	var httpErr files_sdk_lib.ResponseError
+	if errors.As(err, &httpErr) && slices.Contains(retryErrorCodes, httpErr.StatusCode) {
+		fs.Debugf(nil, "Retrying HTTP error %v", err)
+		return true, err
+	}
+
 	return fserrors.ShouldRetry(err), err
 }
 
@@ -345,7 +353,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	for it.Next() {
-		item := ptr(it.File())
+		item := new(it.File())
 		remote := f.opt.Enc.ToStandardPath(item.DisplayName)
 		remote = path.Join(dir, remote)
 		if remote == dir {
@@ -416,7 +424,7 @@ func (f *Fs) mkdir(ctx context.Context, path string) error {
 
 	params := files_sdk.FolderCreateParams{
 		Path:         path,
-		MkdirParents: ptr(true),
+		MkdirParents: new(true),
 	}
 
 	err := f.pacer.Call(func() (bool, error) {
@@ -458,7 +466,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 
 	params := files_sdk.FileDeleteParams{
 		Path:      path,
-		Recursive: ptr(!check),
+		Recursive: new(!check),
 	}
 
 	err := f.pacer.Call(func() (bool, error) {
@@ -529,7 +537,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.
 	params := files_sdk.FileCopyParams{
 		Path:        srcPath,
 		Destination: dstPath,
-		Overwrite:   ptr(true),
+		Overwrite:   new(true),
 	}
 
 	var action files_sdk.FileAction
@@ -672,7 +680,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 		Paths: []string{f.absPath(remote)},
 	}
 	if expire < fs.DurationOff {
-		params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
+		params.ExpiresAt = new(time.Now().Add(time.Duration(expire)))
 	}
 
 	var bundle files_sdk.Bundle
@@ -834,11 +842,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return
 }
 
-// Returns a pointer to t - useful for returning pointers to constants
-func ptr[T any](t T) *T {
-	return &t
-}
-
 func isFolderNotEmpty(err error) bool {
 	var re files_sdk.ResponseError
 	ok := errors.As(err, &re)
@@ -858,7 +861,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		file.UploadWithProvidedMtime(src.ModTime(ctx)),
 	}
 
-	err := o.fs.pacer.Call(func() (bool, error) {
+	err := o.fs.pacer.CallNoRetry(func() (bool, error) {
 		err := o.fs.fileClient.Upload(uploadOpts...)
 		return shouldRetry(ctx, err)
 	})

backend/ftp/ftp_internal_test.go

@@ -43,7 +43,7 @@ func TestCommandEncodingNeutralisesCRLF(t *testing.T) {
 type settings map[string]any
 
 func deriveFs(ctx context.Context, t *testing.T, f fs.Fs, opts settings) fs.Fs {
-	fsName := strings.Split(f.Name(), "{")[0] // strip off hash
+	fsName, _, _ := strings.Cut(f.Name(), "{") // strip off hash
 	configMap := configmap.Simple{}
 	for key, val := range opts {
 		configMap[key] = fmt.Sprintf("%v", val)

backend/gofile/gofile.go

@@ -194,8 +194,7 @@ var retryErrorCodes = []int{
 
 // Return true if the api error has the status given
 func isAPIErr(err error, status string) bool {
-	var apiErr api.Error
-	if errors.As(err, &apiErr) {
+	if apiErr, ok := errors.AsType[api.Error](err); ok {
 		return apiErr.Status == status
 	}
 	return false

backend/hidrive/helpers.go

@@ -10,7 +10,6 @@ package hidrive
 // to be resolved prior to execution with resolvePath(...).
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"io"
@@ -24,8 +23,9 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/multipart"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/ranges"
-	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/semaphore"
@@ -454,17 +454,17 @@ func (f *Fs) deleteObject(ctx context.Context, path string) error {
 }
 
 // createFile creates a file at the given path
-// with the content of the io.ReadSeeker.
+// with the content of the buffer.
 // This guarantees that existing files will not be overwritten.
 // The maximum size of the content is limited by MaximumUploadBytes.
-// The io.ReadSeeker should be resettable by seeking to its start.
+// The caller remains responsible for closing the buffer.
 // If modTime is not the zero time instant,
 // it will be set as the file's modification time after the operation.
 //
 // This returns fs.ErrorDirNotFound
 // if the parent directory of the file is not found.
 // This returns ErrorFileExists if a file already exists at the specified path.
-func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
+func (f *Fs) createFile(ctx context.Context, path string, content *pool.RW, modTime time.Time, onExist OnExistAction) (*api.HiDriveObject, error) {
 	parameters := api.NewQueryParameters()
 	parameters.SetFileInDirectory(path)
 	if onExist == AutoNameOnExist {
@@ -479,12 +479,14 @@ func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker,
 		}
 	}
 
+	contentLength := content.Size()
 	opts := rest.Opts{
-		Method:      "POST",
-		Path:        "/file",
-		Body:        content,
-		ContentType: "application/octet-stream",
-		Parameters:  parameters.Values,
+		Method:        "POST",
+		Path:          "/file",
+		Body:          content,
+		ContentType:   "application/octet-stream",
+		ContentLength: &contentLength,
+		Parameters:    parameters.Values,
 	}
 
 	var result api.HiDriveObject
@@ -510,16 +512,16 @@ func (f *Fs) createFile(ctx context.Context, path string, content io.ReadSeeker,
 }
 
 // overwriteFile updates the content of the file at the given path
-// with the content of the io.ReadSeeker.
+// with the content of the buffer.
 // If the file does not exist it will be created.
 // The maximum size of the content is limited by MaximumUploadBytes.
-// The io.ReadSeeker should be resettable by seeking to its start.
+// The caller remains responsible for closing the buffer.
 // If modTime is not the zero time instant,
 // it will be set as the file's modification time after the operation.
 //
 // This returns fs.ErrorDirNotFound
 // if the parent directory of the file is not found.
-func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeeker, modTime time.Time) (*api.HiDriveObject, error) {
+func (f *Fs) overwriteFile(ctx context.Context, path string, content *pool.RW, modTime time.Time) (*api.HiDriveObject, error) {
 	parameters := api.NewQueryParameters()
 	parameters.SetFileInDirectory(path)
 
@@ -531,12 +533,14 @@ func (f *Fs) overwriteFile(ctx context.Context, path string, content io.ReadSeek
 		}
 	}
 
+	contentLength := content.Size()
 	opts := rest.Opts{
-		Method:      "PUT",
-		Path:        "/file",
-		Body:        content,
-		ContentType: "application/octet-stream",
-		Parameters:  parameters.Values,
+		Method:        "PUT",
+		Path:          "/file",
+		Body:          content,
+		ContentType:   "application/octet-stream",
+		ContentLength: &contentLength,
+		Parameters:    parameters.Values,
 	}
 
 	var result api.HiDriveObject
@@ -634,11 +638,18 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
 		}
 
 		// Read a chunk of data.
-		chunkReader, bytesRead, readErr := readerForChunk(content, chunkSize)
-		if bytesRead < chunkSize {
+		var (
+			chunkReader *pool.RW
+			bytesRead   int64
+		)
+		chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
+		if bytesRead < int64(chunkSize) {
 			startMoreTransfers = false
 		}
 		if readErr != nil || bytesRead <= 0 {
+			if chunkReader != nil {
+				_ = chunkReader.Close()
+			}
 			break
 		}
 
@@ -648,11 +659,12 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
 			// After this upload is done,
 			// signal that another transfer can be started.
 			defer transferSemaphore.Release(1)
-			uploadErr := f.patchFile(gCtx, path, cachedReader(chunkReader), chunkOffset, zeroTime)
+			defer func() { _ = chunkReader.Close() }()
+			uploadErr := f.patchFile(gCtx, path, chunkReader, chunkOffset, zeroTime)
 			if uploadErr == nil {
 				// Remember successfully written chunks.
 				okChunksMu.Lock()
-				okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: int64(bytesRead)})
+				okChunks = append(okChunks, ranges.Range{Pos: int64(chunkOffset), Size: bytesRead})
 				okChunksMu.Unlock()
 				fs.Debugf(f, "Done uploading chunk of size %v at offset %v.", bytesRead, chunkOffset)
 			} else {
@@ -696,10 +708,10 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
 // starting at the given offset.
 //
 // Replaces the file contents starting from the given byte offset
-// with the content of the io.ReadSeeker.
+// with the content of the buffer.
 // If the offset is beyond the file end, the file is extended up to the offset.
 // The maximum size of the update is limited by MaximumUploadBytes.
-// The io.ReadSeeker should be resettable by seeking to its start.
+// The caller remains responsible for closing the buffer.
 // If modTime is not the zero time instant,
 // it will be set as the file's modification time after the operation.
 //
@@ -711,7 +723,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
 // usually expand the holes into regions of 0-bytes.
 //
 // This returns fs.ErrorObjectNotFound if the object is not found.
-func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker, offset uint64, modTime time.Time) error {
+func (f *Fs) patchFile(ctx context.Context, path string, content *pool.RW, offset uint64, modTime time.Time) error {
 	parameters := api.NewQueryParameters()
 	parameters.SetPath(path)
 	parameters.Set("offset", strconv.FormatUint(offset, 10))
@@ -723,13 +735,15 @@ func (f *Fs) patchFile(ctx context.Context, path string, content io.ReadSeeker,
 		}
 	}
 
+	contentLength := content.Size()
 	opts := rest.Opts{
-		Method:      "PATCH",
-		Path:        "/file",
-		Body:        content,
-		ContentType: "application/octet-stream",
-		Parameters:  parameters.Values,
-		NoResponse:  true,
+		Method:        "PATCH",
+		Path:          "/file",
+		Body:          content,
+		ContentType:   "application/octet-stream",
+		ContentLength: &contentLength,
+		Parameters:    parameters.Values,
+		NoResponse:    true,
 	}
 
 	var resp *http.Response
@@ -839,41 +853,48 @@ func createHiDriveScopes(role string, access string) []string {
 	return []string{}
 }
 
-// cachedReader returns a version of the reader that caches its contents and
-// can therefore be reset using Seek.
-func cachedReader(reader io.Reader) io.ReadSeeker {
-	bytesReader, ok := reader.(*bytes.Reader)
-	if ok {
-		return bytesReader
-	}
-
-	repeatableReader, ok := reader.(*readers.RepeatableReader)
-	if ok {
-		return repeatableReader
-	}
-
-	return readers.NewRepeatableReader(reader)
+// unwrapAccounting splits any transfer accounting off reader,
+// returning the raw stream and the accounting (nil if there is none),
+// so the stream can be buffered and accounted when the buffer is uploaded.
+func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Account) {
+	// Any kind of accounter re-wraps into the one type UnWrapAccounting
+	// knows how to take the *Account out of.
+	unwrapped, wrap := accounting.UnWrap(reader)
+	_, acc = accounting.UnWrapAccounting(wrap(unwrapped))
+	return unwrapped, acc
 }
 
-// readerForChunk reads a chunk of bytes from reader (after handling any accounting).
-// Returns a new io.Reader (chunkReader) for that chunk
-// and the number of bytes that have been read from reader.
-func readerForChunk(reader io.Reader, length int) (chunkReader io.Reader, bytesRead int, err error) {
-	// Unwrap any accounting from the input if present.
-	reader, wrap := accounting.UnWrap(reader)
+// readerForChunk reads up to length bytes from reader into a buffer
+// from the global memory pool and returns it
+// along with the number of bytes that have been read from reader.
+// Reaching the end of reader early is not an error.
+//
+// If accountUpload is set, any accounting on reader is applied
+// when the buffer is read instead, so the upload is what gets accounted.
+//
+// This must not be set for a buffer which creates or replaces a file
+// in a single request. Accounting can fail part way through the upload,
+// for example when --max-transfer is reached, and HiDrive keeps
+// what it has received of an aborted request as the file.
+//
+// The caller must Close the buffer when done with it.
+func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
+	chunkReader = multipart.NewRW()
+	if accountUpload {
+		var acc *accounting.Account
+		reader, acc = unwrapAccounting(reader)
+		if acc != nil {
+			chunkReader.SetAccounting(acc.AccountRead)
+		}
+	}
 
-	// Read a chunk of data.
-	buffer := make([]byte, length)
-	bytesRead, err = io.ReadFull(reader, buffer)
-	if err == io.EOF || err == io.ErrUnexpectedEOF {
+	bytesRead, err = io.CopyN(chunkReader, reader, length)
+	if err == io.EOF {
 		err = nil
 	}
 	if err != nil {
+		_ = chunkReader.Close()
 		return nil, bytesRead, err
 	}
-	// Truncate unused capacity.
-	buffer = buffer[:bytesRead]
-
-	// Use wrap to put any accounting back for chunkReader.
-	return wrap(bytes.NewReader(buffer)), bytesRead, nil
+	return chunkReader, bytesRead, nil
 }

backend/hidrive/hidrive.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -514,8 +515,14 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	// (i.e. everything up to the cutoff) in the first request,
 	// avoids files being created on upload failure for small files.
 	// (As opposed to creating an empty file and then uploading the content.)
-	tmpReader, bytesRead, err := readerForChunk(in, int(f.opt.UploadCutoff))
-	cutoffReader := cachedReader(tmpReader)
+	//
+	// Only buffer as much as the source declares it has,
+	// so small files do not cost the whole cutoff in memory.
+	prefixSize := int64(f.opt.UploadCutoff)
+	if size := src.Size(); size >= 0 && size < prefixSize {
+		prefixSize = size
+	}
+	cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
 	if err != nil {
 		return nil, err
 	}
@@ -540,6 +547,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 		}
 		return false, createErr
 	})
+	_ = cutoffReader.Close()
 
 	if err != nil {
 		return nil, err
@@ -556,7 +564,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	}
 	// If there is more left to write, o.Update needs to skip ahead.
 	// Use a fs.SeekOption with the current offset to do this.
-	options = append(options, &fs.SeekOption{Offset: int64(bytesRead)})
+	options = append(options, &fs.SeekOption{Offset: bytesRead})
 	err = o.Update(ctx, in, src, options...)
 
 	if err == nil {
@@ -971,7 +979,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Metadata should be updated even if the upload fails.
 		info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
 	} else {
-		info, err = o.fs.overwriteFile(ctx, resolvedPath, cachedReader(in), modTime)
+		var content *pool.RW
+		content, _, err = readerForChunk(in, src.Size(), false)
+		if err == nil {
+			info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
+			_ = content.Close()
+		}
 		metaErr = err
 	}
 

backend/http/http.go

@@ -57,7 +57,13 @@ The input format is comma separated list of key,value pairs.  Standard
 
 For example, to set a Cookie use 'Cookie,name=value', or '"Cookie","name=value"'.
 
-You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.`,
+You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'.
+
+The headers are only sent to the host in the configured URL. If the
+server redirects to another host (including a subdomain or a different
+port) the headers are not sent to it, or to any further hop in that
+redirect chain. When headers are set, a redirect from https to http is
+refused as it would send them in cleartext.`,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
 		}, {
@@ -277,12 +283,17 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	if err != nil {
 		return false, err
 	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
+	u, err := rest.URLJoinRoot(base, f.root)
 	if err != nil {
 		return false, err
 	}
 
 	client := fshttp.NewClient(ctx)
+	// Without configured headers keep the default policy which
+	// public mirrors that redirect from https to http rely on.
+	if len(opt.Headers) > 0 {
+		client.CheckRedirect = checkRedirect(opt)
+	}
 
 	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
 	fs.Debugf(nil, "Root: %s", endpoint)
@@ -510,6 +521,25 @@ func addHeaders(req *http.Request, opt *Options) {
 	}
 }
 
+// checkRedirect returns an http.Client.CheckRedirect function which
+// follows redirects but refuses an HTTPS to HTTP downgrade with
+// rest.ErrHTTPSDowngrade and strips the configured headers when the
+// redirect chain has left the originally requested host at any point.
+func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
+	headers := make([]string, 0, len(opt.Headers)/2)
+	for i := 0; i < len(opt.Headers); i += 2 {
+		headers = append(headers, opt.Headers[i])
+	}
+	stripHeaders := rest.StripHeadersOnCrossHostRedirectFn(headers...)
+	return func(req *http.Request, via []*http.Request) error {
+		err := stripHeaders(req, via)
+		if errors.Is(err, rest.ErrHTTPSDowngrade) {
+			err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
+		}
+		return err
+	}
+}
+
 // Adds the configured headers to the request if any
 func (f *Fs) addHeaders(req *http.Request) {
 	addHeaders(req, &f.opt)
@@ -543,7 +573,7 @@ func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error
 		return nil, fmt.Errorf("failed to readDir: %w", err)
 	}
 
-	contentType := strings.SplitN(res.Header.Get("Content-Type"), ";", 2)[0]
+	contentType, _, _ := strings.Cut(res.Header.Get("Content-Type"), ";")
 	switch contentType {
 	case "text/html":
 		names, err = parse(u, res.Body)

backend/http/http_internal_test.go

@@ -581,3 +581,213 @@ func TestFsNoSlashRoots(t *testing.T) {
 		}
 	}
 }
+
+// readRedirected makes an http remote with the configured headers
+// pointing at url and reads file.txt from it, checking the content
+func readRedirected(t *testing.T, url string) {
+	configfile.Install()
+	m := configmap.Simple{
+		"type":    "http",
+		"url":     url + "/",
+		"headers": strings.Join(headers, ","),
+	}
+	f, err := NewFs(context.Background(), remoteName, "", m)
+	require.NoError(t, err)
+
+	o, err := f.NewObject(context.Background(), "file.txt")
+	require.NoError(t, err)
+	fd, err := o.Open(context.Background())
+	require.NoError(t, err)
+	data, err := io.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	assert.Equal(t, "hello", string(data))
+}
+
+func TestRedirectKeepsHeadersOnSameHost(t *testing.T) {
+	var got http.Header
+	var ts *httptest.Server
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
+			return
+		}
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer ts.Close()
+
+	readRedirected(t, ts.URL)
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Equal(t, headers[i+1], got.Get(headers[i]))
+	}
+}
+
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+	// A plaintext server to be redirected to
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer target.Close()
+
+	// An HTTPS server which redirects to plaintext HTTP
+	ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
+	}))
+	defer ts.Close()
+
+	// open makes a remote pointing at ts with or without headers
+	// and opens file.txt on it
+	open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
+		configfile.Install()
+		m := configmap.Simple{
+			"type":    "http",
+			"url":     ts.URL + "/",
+			"no_head": "true",
+		}
+		if withHeaders {
+			m.Set("headers", strings.Join(headers, ","))
+		}
+		ctx, ci := fs.AddConfig(context.Background())
+		ci.InsecureSkipVerify = true
+		f, err := NewFs(ctx, remoteName, "", m)
+		require.NoError(t, err)
+
+		// no_head means NewObject doesn't make a request so use Open to make one
+		o, err := f.NewObject(ctx, "file.txt")
+		require.NoError(t, err)
+		return o.Open(ctx)
+	}
+
+	t.Run("WithHeaders", func(t *testing.T) {
+		// Headers could leak so the downgrade must be refused
+		_, err := open(t, true)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithoutHeaders", func(t *testing.T) {
+		// Nothing to leak so the downgrade is followed
+		fd, err := open(t, false)
+		require.NoError(t, err)
+		data, err := io.ReadAll(fd)
+		require.NoError(t, err)
+		require.NoError(t, fd.Close())
+		assert.Equal(t, "hello", string(data))
+	})
+}
+
+func TestRedirectKeepsHeadersOnSameHostDifferentCase(t *testing.T) {
+	// The redirect spells the host differently but it is the same
+	// host so the headers must be kept
+	var got http.Header
+	var ts *httptest.Server
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, "http://LOCALHOST:"+port(t, ts)+"/moved"+r.URL.Path, http.StatusFound)
+			return
+		}
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer ts.Close()
+
+	readRedirected(t, "http://localhost:"+port(t, ts))
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Equal(t, headers[i+1], got.Get(headers[i]))
+	}
+}
+
+// port returns the port ts is listening on
+func port(t *testing.T, ts *httptest.Server) string {
+	u, err := url.Parse(ts.URL)
+	require.NoError(t, err)
+	return u.Port()
+}
+
+func TestRedirectStripsHeadersAfterHostChange(t *testing.T) {
+	// The configured server redirects to a second server which
+	// redirects back to the configured server. The headers must not
+	// be sent on the way back as the other host chose the path.
+	var got http.Header
+	var ts *httptest.Server
+	bouncer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, ts.URL+"/moved"+r.URL.Path, http.StatusFound)
+	}))
+	defer bouncer.Close()
+
+	ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !strings.HasPrefix(r.URL.Path, "/moved/") {
+			http.Redirect(w, r, bouncer.URL+r.URL.Path, http.StatusFound)
+			return
+		}
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer ts.Close()
+
+	readRedirected(t, ts.URL)
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Empty(t, got.Values(headers[i]), "header %q sent after cross-host redirect", headers[i])
+	}
+}
+
+func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
+	// Serve the final response from a second server. httptest servers
+	// listen on 127.0.0.1:port so the two servers have different hosts
+	// for the purposes of the redirect check.
+	var got http.Header
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		got = r.Header.Clone()
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer target.Close()
+
+	redirector := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusFound)
+	}))
+	defer redirector.Close()
+
+	readRedirected(t, redirector.URL)
+
+	require.NotNil(t, got)
+	for i := 0; i < len(headers); i += 2 {
+		assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
+	}
+}
+
+func TestRootCannotChangeHost(t *testing.T) {
+	// A root starting with "//" is a network-path reference under the
+	// URL standard. It must not be able to move the remote (and the
+	// configured headers) to a host other than the configured one.
+	var requests int
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer other.Close()
+
+	m := prepareServer(t)
+	otherHost := strings.TrimPrefix(other.URL, "http://")
+	for _, root := range []string{
+		"//" + otherHost + "/",
+		"//" + otherHost + "/one%.txt",
+		"//" + otherHost,
+		"//user:pass@" + otherHost + "/",
+	} {
+		f, err := NewFs(context.Background(), remoteName, root, m)
+		assert.ErrorContains(t, err, "must not change the scheme, host or user", root)
+		assert.Nil(t, f, root)
+	}
+	assert.Equal(t, 0, requests, "request sent to the other host")
+
+	// A same-origin absolute path must still work
+	f, err := NewFs(context.Background(), remoteName, "/", m)
+	require.NoError(t, err)
+	assert.Equal(t, m["url"]+"/", f.String())
+}

backend/huaweidrive/api/types.go

@@ -37,44 +37,44 @@ type FileList struct {
 
 // File represents a file or folder in Huawei Drive
 type File struct {
-	ID                        string                 `json:"id"`
-	FileName                  string                 `json:"fileName"`
-	OriginalFilename          string                 `json:"originalFilename,omitempty"`
-	Description               string                 `json:"description,omitempty"`
-	MimeType                  string                 `json:"mimeType"`
-	Category                  string                 `json:"category"`
-	Size                      int64                  `json:"size,omitempty"`
-	SHA256                    string                 `json:"sha256,omitempty"`
-	CreatedTime               time.Time              `json:"createdTime"`
-	EditedTime                time.Time              `json:"editedTime"`
-	EditedByMeTime            time.Time              `json:"editedByMeTime,omitempty"`
-	ParentFolder              []string               `json:"parentFolder,omitempty"`
-	Owners                    []User                 `json:"owners"`
-	LastEditor                User                   `json:"lastEditor"`
-	Permissions               []Permission           `json:"permissions"`
-	PermissionIDs             []string               `json:"permissionIds"`
-	Capabilities              FileCapabilities       `json:"capabilities"`
-	OwnedByMe                 bool                   `json:"ownedByMe"`
-	EditedByMe                bool                   `json:"editedByMe"`
-	ViewedByMe                bool                   `json:"viewedByMe"`
-	HasShared                 bool                   `json:"hasShared"`
-	Recycled                  bool                   `json:"recycled"`
-	DirectlyRecycled          bool                   `json:"directlyRecycled"`
-	Favorite                  bool                   `json:"favorite"`
-	ExistThumbnail            bool                   `json:"existThumbnail"`
-	ThumbnailVersion          int64                  `json:"thumbnailVersion"`
-	IconDownloadLink          string                 `json:"iconDownloadLink,omitempty"`
-	ContentDownloadLink       string                 `json:"contentDownloadLink,omitempty"`
-	ContentVersion            string                 `json:"contentVersion,omitempty"`
-	LastHistoryVersionID      string                 `json:"lastHistoryVersionId,omitempty"`
-	OccupiedSpace             int64                  `json:"occupiedSpace,omitempty"`
-	Version                   int64                  `json:"version"`
-	WritersHasSharePermission bool                   `json:"writersHasSharePermission"`
-	WriterHasCopyPermission   bool                   `json:"writerHasCopyPermission"`
-	Containers                []string               `json:"containers"`
-	Properties                map[string]interface{} `json:"properties,omitempty"`
-	AppSettings               map[string]interface{} `json:"appSettings,omitempty"`
-	ContentExtras             *ContentExtras         `json:"contentExtras,omitempty"`
+	ID                        string           `json:"id"`
+	FileName                  string           `json:"fileName"`
+	OriginalFilename          string           `json:"originalFilename,omitempty"`
+	Description               string           `json:"description,omitempty"`
+	MimeType                  string           `json:"mimeType"`
+	Category                  string           `json:"category"`
+	Size                      int64            `json:"size,omitempty"`
+	SHA256                    string           `json:"sha256,omitempty"`
+	CreatedTime               time.Time        `json:"createdTime"`
+	EditedTime                time.Time        `json:"editedTime"`
+	EditedByMeTime            time.Time        `json:"editedByMeTime"`
+	ParentFolder              []string         `json:"parentFolder,omitempty"`
+	Owners                    []User           `json:"owners"`
+	LastEditor                User             `json:"lastEditor"`
+	Permissions               []Permission     `json:"permissions"`
+	PermissionIDs             []string         `json:"permissionIds"`
+	Capabilities              FileCapabilities `json:"capabilities"`
+	OwnedByMe                 bool             `json:"ownedByMe"`
+	EditedByMe                bool             `json:"editedByMe"`
+	ViewedByMe                bool             `json:"viewedByMe"`
+	HasShared                 bool             `json:"hasShared"`
+	Recycled                  bool             `json:"recycled"`
+	DirectlyRecycled          bool             `json:"directlyRecycled"`
+	Favorite                  bool             `json:"favorite"`
+	ExistThumbnail            bool             `json:"existThumbnail"`
+	ThumbnailVersion          int64            `json:"thumbnailVersion"`
+	IconDownloadLink          string           `json:"iconDownloadLink,omitempty"`
+	ContentDownloadLink       string           `json:"contentDownloadLink,omitempty"`
+	ContentVersion            string           `json:"contentVersion,omitempty"`
+	LastHistoryVersionID      string           `json:"lastHistoryVersionId,omitempty"`
+	OccupiedSpace             int64            `json:"occupiedSpace,omitempty"`
+	Version                   int64            `json:"version"`
+	WritersHasSharePermission bool             `json:"writersHasSharePermission"`
+	WriterHasCopyPermission   bool             `json:"writerHasCopyPermission"`
+	Containers                []string         `json:"containers"`
+	Properties                map[string]any   `json:"properties,omitempty"`
+	AppSettings               map[string]any   `json:"appSettings,omitempty"`
+	ContentExtras             *ContentExtras   `json:"contentExtras,omitempty"`
 }
 
 // IsDir returns true if the file is a directory
@@ -125,41 +125,41 @@ type Thumbnail struct {
 
 // CreateFolderRequest represents request to create a folder
 type CreateFolderRequest struct {
-	FileName     string                 `json:"fileName"`
-	Description  string                 `json:"description,omitempty"`
-	MimeType     string                 `json:"mimeType"`
-	ParentFolder []string               `json:"parentFolder,omitempty"`
-	Favorite     bool                   `json:"favorite,omitempty"`
-	Properties   map[string]interface{} `json:"properties,omitempty"`
-	AppSettings  map[string]interface{} `json:"appSettings,omitempty"`
-	AutoRename   int                    `json:"autoRename,omitempty"`
+	FileName     string         `json:"fileName"`
+	Description  string         `json:"description,omitempty"`
+	MimeType     string         `json:"mimeType"`
+	ParentFolder []string       `json:"parentFolder,omitempty"`
+	Favorite     bool           `json:"favorite,omitempty"`
+	Properties   map[string]any `json:"properties,omitempty"`
+	AppSettings  map[string]any `json:"appSettings,omitempty"`
+	AutoRename   int            `json:"autoRename,omitempty"`
 }
 
 // UpdateFileRequest represents request to update a file
 type UpdateFileRequest struct {
-	FileName                  string                 `json:"fileName,omitempty"`
-	Description               string                 `json:"description,omitempty"`
-	MimeType                  string                 `json:"mimeType,omitempty"`
-	Favorite                  *bool                  `json:"favorite,omitempty"`
-	Recycled                  *bool                  `json:"recycled,omitempty"`
-	OriginalFilename          string                 `json:"originalFilename,omitempty"`
-	WriterHasCopyPermission   *bool                  `json:"writerHasCopyPermission,omitempty"`
-	WritersHasSharePermission *bool                  `json:"writersHasSharePermission,omitempty"`
-	Properties                map[string]interface{} `json:"properties,omitempty"`
-	AppSettings               map[string]interface{} `json:"appSettings,omitempty"`
-	AddParentFolder           []string               `json:"addParentFolder,omitempty"`
-	RemoveParentFolder        []string               `json:"removeParentFolder,omitempty"`
-	CreatedTime               *time.Time             `json:"createdTime,omitempty"`
-	EditedTime                *time.Time             `json:"editedTime,omitempty"`
+	FileName                  string         `json:"fileName,omitempty"`
+	Description               string         `json:"description,omitempty"`
+	MimeType                  string         `json:"mimeType,omitempty"`
+	Favorite                  *bool          `json:"favorite,omitempty"`
+	Recycled                  *bool          `json:"recycled,omitempty"`
+	OriginalFilename          string         `json:"originalFilename,omitempty"`
+	WriterHasCopyPermission   *bool          `json:"writerHasCopyPermission,omitempty"`
+	WritersHasSharePermission *bool          `json:"writersHasSharePermission,omitempty"`
+	Properties                map[string]any `json:"properties,omitempty"`
+	AppSettings               map[string]any `json:"appSettings,omitempty"`
+	AddParentFolder           []string       `json:"addParentFolder,omitempty"`
+	RemoveParentFolder        []string       `json:"removeParentFolder,omitempty"`
+	CreatedTime               *time.Time     `json:"createdTime,omitempty"`
+	EditedTime                *time.Time     `json:"editedTime,omitempty"`
 }
 
 // CopyFileRequest represents request to copy a file
 type CopyFileRequest struct {
-	FileName     string                 `json:"fileName,omitempty"`
-	MimeType     string                 `json:"mimeType,omitempty"`
-	ParentFolder []string               `json:"parentFolder,omitempty"`
-	Properties   map[string]interface{} `json:"properties,omitempty"`
-	AppSettings  map[string]interface{} `json:"appSettings,omitempty"`
+	FileName     string         `json:"fileName,omitempty"`
+	MimeType     string         `json:"mimeType,omitempty"`
+	ParentFolder []string       `json:"parentFolder,omitempty"`
+	Properties   map[string]any `json:"properties,omitempty"`
+	AppSettings  map[string]any `json:"appSettings,omitempty"`
 }
 
 // ResumeUploadInitResponse represents response to resume upload init
@@ -224,11 +224,6 @@ var GlobalDomains = map[string]bool{
 	"drive.cloud.hicloud.com":       true,
 }
 
-// BoolPtr returns a pointer to a bool value (helper for UpdateFileRequest)
-func BoolPtr(b bool) *bool {
-	return &b
-}
-
 // StartCursor represents response to Changes.getStartCursor endpoint
 type StartCursor struct {
 	Category    string `json:"category"` // "drive#startCursor"

backend/huaweidrive/huaweidrive.go

@@ -33,6 +33,8 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/oauth2"
 )
@@ -1030,7 +1032,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Debugf(f, "Copy: failed to get metadata options: %v", err)
 	}
 	if len(mergedMeta) > 0 {
-		copyReq.Properties = make(map[string]interface{})
+		copyReq.Properties = make(map[string]any)
 		for key, value := range mergedMeta {
 			switch key {
 			case "content-type", "sha256", "btime", "mtime", "utime",
@@ -1720,7 +1722,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	}
 
 	// Create metadata
-	metadata := map[string]interface{}{
+	metadata := map[string]any{
 		"fileName": leaf,
 		"mimeType": mimeType,
 	}
@@ -1740,9 +1742,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 		metadata["autoRename"] = 3
 	}
 
-	// Create multipart form
-	var buf bytes.Buffer
-	writer := multipart.NewWriter(&buf)
+	// Build the multipart body in pooled memory so it can be re-sent on retry
+	rw := pool.NewRW(pool.Global())
+	defer fs.CheckClose(rw, &err)
+	writer := multipart.NewWriter(rw)
 
 	// Add metadata part - use exact format from Huawei docs
 	metadataWriter, err := writer.CreatePart(textproto.MIMEHeader{
@@ -1771,25 +1774,34 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	}
 
 	// Read and write file content
-	_, err = io.Copy(fileWriter, in)
+	counter := readers.NewCountingReader(in)
+	_, err = io.Copy(fileWriter, counter)
 	if err != nil {
 		return fmt.Errorf("failed to copy file content: %w", err)
 	}
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload.
+	if int64(counter.BytesRead()) != size {
+		return fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	err = writer.Close()
 	if err != nil {
 		return fmt.Errorf("failed to close multipart writer: %w", err)
 	}
 
-	bodyBytes := buf.Bytes()
+	bodySize := rw.Size()
 	opts := rest.Opts{
 		Method: "POST",
 		Parameters: url.Values{
 			"uploadType": []string{"multipart"},
 			"fields":     []string{"*"},
 		},
-		ContentType: fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
-		RootURL:     o.fs.uploadURL,
+		ContentType:   fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary()),
+		ContentLength: &bodySize,
+		RootURL:       o.fs.uploadURL,
+		Body:          rw,
 	}
 
 	if o.id != "" {
@@ -1803,7 +1815,10 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 
 	var info *api.File
 	err = o.fs.pacer.Call(func() (bool, error) {
-		opts.Body = bytes.NewReader(bodyBytes)
+		_, err := rw.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
 		resp, err := srv.CallJSON(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
@@ -1864,7 +1879,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 	}
 
 	// Prepare metadata for resumable upload initialization
-	metadata := map[string]interface{}{
+	metadata := map[string]any{
 		"fileName": leaf,
 	}
 
@@ -1897,25 +1912,26 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 	}
 
 	// Now upload the content in chunks
-	chunkSize := int64(o.fs.opt.ChunkSize)
-	if chunkSize < 256*1024 {
-		chunkSize = 256 * 1024 // Minimum chunk size according to Huawei Drive API
-	}
-	if chunkSize > 64*1024*1024 {
-		chunkSize = 64 * 1024 * 1024 // Maximum single upload size
-	}
+	// Note 256k is the minimum chunk size and 64M the maximum
+	// single upload size according to the Huawei Drive API
+	chunkSize := min(max(int64(o.fs.opt.ChunkSize), 256*1024), 64*1024*1024)
 
-	buf := make([]byte, chunkSize)
 	var offset int64
 
 	for offset < size {
-		n, err := io.ReadFull(in, buf)
-		if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+		// Buffer the chunk in pooled memory so it can be re-sent on retry
+		rw := pool.NewRW(pool.Global())
+		n, err := io.CopyN(rw, in, min(size-offset, chunkSize))
+		if err != nil {
+			// io.EOF here means the source ran out before the declared size
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			_ = rw.Close()
 			return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)
 		}
 
-		chunk := buf[:n]
-		end := offset + int64(n) - 1
+		end := offset + n - 1
 
 		// Upload this chunk using the original client with OAuth authentication
 		chunkOpts := rest.Opts{
@@ -1923,17 +1939,20 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 			RootURL: location,
 			Path:    "", // Empty path since RootURL contains the full URL
 			ExtraHeaders: map[string]string{
-				"Content-Range":  fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
-				"Content-Length": strconv.Itoa(n),
-				"Content-Type":   mimeType,
+				"Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),
+				"Content-Type":  mimeType,
 			},
+			ContentLength: &n,
+			Body:          rw,
 		}
 
 		var info *api.File
 		var chunkResp *http.Response
 		err = o.fs.pacer.Call(func() (bool, error) {
-			chunkOpts.Body = bytes.NewReader(chunk)
-			var err error
+			_, err := rw.Seek(0, io.SeekStart)
+			if err != nil {
+				return false, err
+			}
 			chunkResp, err = srv.CallJSON(ctx, &chunkOpts, nil, &info)
 			// 308 means continue uploading (incomplete)
 			if chunkResp != nil && chunkResp.StatusCode == 308 {
@@ -1947,11 +1966,15 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 			}
 			return shouldRetry(ctx, chunkResp, err)
 		})
+		closeErr := rw.Close()
 		if err != nil {
 			return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)
 		}
+		if closeErr != nil {
+			return closeErr
+		}
 
-		offset += int64(n)
+		offset += n
 
 		// If we got file info back, we're done
 		if info != nil && info.ID != "" {
@@ -2082,8 +2105,8 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
 
 	// Prepare the update request payload
 	updateReq := api.UpdateFileRequest{
-		Properties:  make(map[string]interface{}),
-		AppSettings: make(map[string]interface{}),
+		Properties:  make(map[string]any),
+		AppSettings: make(map[string]any),
 	}
 
 	// Process metadata and separate into properties and app settings
@@ -2094,7 +2117,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
 			updateReq.Description = value
 		case "favorite":
 			if favorite, err := strconv.ParseBool(value); err == nil {
-				updateReq.Favorite = api.BoolPtr(favorite)
+				updateReq.Favorite = new(favorite)
 			}
 		case "content-type":
 			// Allow setting/overriding MIME type

backend/huaweidrive/huaweidrive_test.go

@@ -19,15 +19,49 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 )
 
+// Limits on the chunk size accepted by the resumable upload API
+const (
+	minChunkSize = 256 * fs.Kibi
+	maxChunkSize = 64 * fs.Mebi
+)
+
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:               "TestHuaweiDrive:",
 		NilObject:                (*Object)(nil),
 		SkipBadWindowsCharacters: true,
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize:  minChunkSize,
+			MaxChunkSize:  maxChunkSize,
+			CeilChunkSize: fstests.NextPowerOfTwo,
+		},
 	})
 }
 
+// SetUploadChunkSize changes the configured chunk size, returning the old value.
+//
+// It is only called by the integration tests while no transfer is in progress.
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	return old, nil
+}
+
+// SetUploadCutoff changes the configured upload cutoff, returning the old value.
+//
+// It is only called by the integration tests while no transfer is in progress.
+func (f *Fs) SetUploadCutoff(cutoff fs.SizeSuffix) (fs.SizeSuffix, error) {
+	var old fs.SizeSuffix
+	old, f.opt.UploadCutoff = f.opt.UploadCutoff, cutoff
+	return old, nil
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)
+
 // TestNewFs tests the NewFs constructor
 func TestNewFs(t *testing.T) {
 	ctx := context.Background()

backend/iclouddrive/api/drive.go

@@ -498,7 +498,7 @@ func (d *DriveService) CopyDocByItemID(ctx context.Context, itemID string) (*Dri
 }
 
 // CreateUpload creates an url for an upload.
-func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string) (*UploadResponse, *http.Response, error) {
+func (d *DriveService) CreateUpload(ctx context.Context, size int64, name, zone string) (*UploadResponse, *http.Response, error) {
 	// first we need to request an upload url
 	values := map[string]any{
 		"filename":     name,
@@ -513,7 +513,7 @@ func (d *DriveService) CreateUpload(ctx context.Context, size int64, name string
 
 	opts := rest.Opts{
 		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/upload/web",
+		Path:         "/ws/" + zone + "/upload/web",
 		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
 		RootURL:      d.docsEndpoint,
 		Body:         body,
@@ -552,14 +552,14 @@ func (d *DriveService) Upload(ctx context.Context, in io.Reader, size int64, nam
 // ctx: the context.Context object for the request.
 // r: a pointer to the UpdateFileInfo struct containing the information for the file update.
 // Returns a pointer to the DriveItem struct representing the updated file, the http.Response object, and an error if any.
-func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*DriveItem, *http.Response, error) {
+func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo, zone string) (*DriveItem, *http.Response, error) {
 	body, err := IntoReader(r)
 	if err != nil {
 		return nil, nil, err
 	}
 	opts := rest.Opts{
 		Method:       "POST",
-		Path:         "/ws/" + defaultZone + "/update/documents",
+		Path:         "/ws/" + zone + "/update/documents",
 		ExtraHeaders: d.icloud.Session.GetHeaders(map[string]string{}),
 		RootURL:      d.docsEndpoint,
 		Body:         body,
@@ -572,7 +572,7 @@ func (d *DriveService) UpdateFile(ctx context.Context, r *UpdateFileInfo) (*Driv
 
 	doc := responseInfo.Results[0].Document
 	item := DriveItem{
-		Drivewsid:    "FILE::com.apple.CloudDocs::" + doc.DocumentID,
+		Drivewsid:    ConstructDriveID(doc.DocumentID, zone, "FILE"),
 		Docwsid:      doc.DocumentID,
 		Itemid:       doc.ItemID,
 		Etag:         doc.Etag,
@@ -892,6 +892,18 @@ func DeconstructDriveID(id string) (docType, zone, docid string) {
 	return split[0], split[1], split[2]
 }
 
+// ZoneFromDriveID returns the zone a drive ID belongs to, or the default zone
+// for an ID that carries none. Items in an app container -- Obsidian, Pages,
+// Shortcuts -- live in that app's zone rather than com.apple.CloudDocs, and a
+// write addressed to the wrong zone cannot resolve its parent.
+func ZoneFromDriveID(id string) string {
+	_, zone, _ := DeconstructDriveID(id)
+	if zone == "" {
+		return defaultZone
+	}
+	return zone
+}
+
 // ConstructDriveID constructs a drive ID from the given components.
 func ConstructDriveID(id string, zone string, t string) string {
 	return strings.Join([]string{t, zone, id}, "::")

backend/iclouddrive/api/drive_test.go

@@ -0,0 +1,32 @@
+package api
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestZoneFromDriveID(t *testing.T) {
+	for _, test := range []struct {
+		name string
+		id   string
+		want string
+	}{
+		{"app container", "FOLDER::iCloud.md.obsidian::documents#o2v", "iCloud.md.obsidian"},
+		{"another app container", "FOLDER::com.apple.Pages::documents#7qt", "com.apple.Pages"},
+		{"ordinary folder", "FOLDER::com.apple.CloudDocs::B847FE2D", "com.apple.CloudDocs"},
+		{"file in a container", "FILE::iCloud.md.obsidian::abc123", "iCloud.md.obsidian"},
+		{"no zone", "root", defaultZone},
+		{"empty", "", defaultZone},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			assert.Equal(t, test.want, ZoneFromDriveID(test.id))
+		})
+	}
+}
+
+func TestZoneFromDriveIDRoundTrip(t *testing.T) {
+	id := ConstructDriveID("abc123", "iCloud.md.obsidian", "FILE")
+	assert.Equal(t, "FILE::iCloud.md.obsidian::abc123", id)
+	assert.Equal(t, "iCloud.md.obsidian", ZoneFromDriveID(id))
+}

backend/iclouddrive/api/photos.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"net/http"
 	"os"
 	"path"
@@ -1617,9 +1618,7 @@ func (ps *PhotosService) GetLibraryAlbumCounts(ctx context.Context) (map[string]
 		if err := ps.requestForArea(ctx, area, "internal/records/query/batch", map[string]any{"batch": batch}, &response); err != nil {
 			return nil, fmt.Errorf("failed to get library album counts: %w", err)
 		}
-		for k, v := range response.toCounts(order) {
-			counts[k] = v
-		}
+		maps.Copy(counts, response.toCounts(order))
 	}
 	return counts, nil
 }
@@ -2032,7 +2031,7 @@ func (album *Album) fetchPhotosParallel(ctx context.Context, totalPhotos int64)
 	sem := make(chan struct{}, workers)
 	var wg sync.WaitGroup
 
-	for i := 0; i < numPartitions; i++ {
+	for i := range numPartitions {
 		wg.Add(1)
 		go func(idx int) {
 			defer wg.Done()

backend/iclouddrive/api/photos_test.go

@@ -1191,7 +1191,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
 	done := make(chan struct{})
 	go func() {
 		defer close(done)
-		for i := 0; i < 100; i++ {
+		for range 100 {
 			lib.deltaMu.Lock()
 			lib.pendingDelta = &deltaPayload{
 				records:   []json.RawMessage{json.RawMessage(`{"recordName":"m1","recordType":"CPLMaster"}`)},
@@ -1201,7 +1201,7 @@ func TestFlushCaches_NoPendingDeltaRace(t *testing.T) {
 			lib.applyPendingDelta(context.Background())
 		}
 	}()
-	for i := 0; i < 100; i++ {
+	for range 100 {
 		ps.FlushCaches()
 		// Re-add the library since FlushCaches clears it
 		ps.mu.Lock()

backend/iclouddrive/api/session.go

@@ -718,7 +718,7 @@ func (s *Session) GetAuthState(ctx context.Context) (*AuthStateResponse, error)
 		Path:          "",
 		ExtraHeaders:  s.GetAuthHeaders(map[string]string{}),
 		RootURL:       authEndpoint,
-		ContentLength: int64Ptr(0),
+		ContentLength: new(int64(0)),
 	}
 	// Use srv.Call directly to capture the raw response body for debugging
 	resp, err := s.srv.Call(ctx, &opts)
@@ -828,7 +828,7 @@ func (s *Session) TrustSession(ctx context.Context) error {
 		ExtraHeaders:  s.GetAuthHeaders(map[string]string{}),
 		RootURL:       authEndpoint,
 		NoResponse:    true,
-		ContentLength: int64Ptr(0),
+		ContentLength: new(int64(0)),
 	}
 
 	_, err := s.Request(ctx, opts, nil, nil)
@@ -846,7 +846,7 @@ func (s *Session) ValidateSession(ctx context.Context) error {
 		Path:          "/validate",
 		ExtraHeaders:  s.GetHeaders(map[string]string{}),
 		RootURL:       setupEndpoint,
-		ContentLength: int64Ptr(0),
+		ContentLength: new(int64(0)),
 	}
 	_, err := s.Request(ctx, opts, nil, &s.AccountInfo)
 	if err != nil {
@@ -904,8 +904,6 @@ func GetCommonHeaders(overwrite map[string]string) map[string]string {
 	return headers
 }
 
-func int64Ptr(v int64) *int64 { return &v }
-
 // NewSession creates a new Session instance with default values
 func NewSession() *Session {
 	session := &Session{

backend/iclouddrive/iclouddrive.go

@@ -92,7 +92,7 @@ func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, fou
 		item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir))
 		return shouldRetry(ctx, resp, err)
 	}); err != nil {
-		if item == nil && resp.StatusCode == 404 {
+		if item == nil && resp != nil && resp.StatusCode == 404 {
 			return nil, false, nil
 		}
 		return nil, false, err
@@ -343,7 +343,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	var item *api.DriveItem
 	if err = f.pacer.Call(func() (bool, error) {
-		item, resp, err = f.service.UpdateFile(ctx, &r)
+		item, resp, err = f.service.UpdateFile(ctx, &r, api.ZoneFromDriveID(pathID))
 		return retryResultUnknown(ctx, resp, err)
 	}); err != nil {
 		return nil, err
@@ -949,7 +949,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Create document
 	var uploadInfo *api.UploadResponse
 	if err = o.fs.pacer.Call(func() (bool, error) {
-		uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name)
+		uploadInfo, resp, err = o.fs.service.CreateUpload(ctx, size, name, api.ZoneFromDriveID(dirID))
 		return ignoreResultUnknown(ctx, resp, err)
 	}); err != nil {
 		return err
@@ -988,7 +988,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Update metadata
 	var item *api.DriveItem
 	if err = o.fs.pacer.Call(func() (bool, error) {
-		item, resp, err = o.fs.service.UpdateFile(ctx, &r)
+		item, resp, err = o.fs.service.UpdateFile(ctx, &r, api.ZoneFromDriveID(dirID))
 		return ignoreResultUnknown(ctx, resp, err)
 	}); err != nil {
 		return err

backend/iclouddrive/icloudphotos.go

@@ -376,8 +376,7 @@ func (f *PhotosFs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut
 		return "", false, nil
 	}
 
-	if strings.HasPrefix(pathID, "lib:") {
-		libraryName := strings.TrimPrefix(pathID, "lib:")
+	if libraryName, ok := strings.CutPrefix(pathID, "lib:"); ok {
 		libraries, err := photosService.GetLibraries(ctx)
 		if err != nil {
 			return "", false, err
@@ -736,10 +735,7 @@ func (f *PhotosFs) ListR(ctx context.Context, dir string, callback fs.ListRCallb
 	}
 	close(jobCh)
 
-	workers := fs.GetConfig(ctx).Checkers
-	if len(jobs) < workers {
-		workers = len(jobs)
-	}
+	workers := min(len(jobs), fs.GetConfig(ctx).Checkers)
 	errs := make(chan error, workers)
 
 	for range workers {

backend/imagekit/imagekit.go

@@ -477,7 +477,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 			FolderPath: f.EncodePath(path.Join(f.root, dir)),
 		})
 
-		if res.StatusCode == http.StatusNotFound {
+		if res != nil && res.StatusCode == http.StatusNotFound {
 			return false, fs.ErrorDirNotFound
 		}
 
@@ -502,7 +502,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 			FolderPath: f.EncodePath(remote),
 		})
 
-		if res.StatusCode == http.StatusNotFound {
+		if res != nil && res.StatusCode == http.StatusNotFound {
 			return false, fs.ErrorDirNotFound
 		}
 

backend/internetarchive/internetarchive.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -854,11 +855,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	// make a PUT request at (IAS3)/encoded(:item/:path)
+	counter := readers.NewCountingReader(in)
 	var resp *http.Response
 	opts := rest.Opts{
 		Method:        "PUT",
 		Path:          "/" + url.PathEscape(path.Join(bucket, bucketPath)),
-		Body:          in,
+		Body:          counter,
 		ContentLength: &size,
 		ExtraHeaders:  headers,
 	}
@@ -868,6 +870,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return o.fs.shouldRetry(resp, err)
 	})
 
+	// Check the source supplied the number of bytes it declared
+	// otherwise a truncated file would be stored as a good upload
+	if err == nil && size >= 0 && int64(counter.BytesRead()) != size {
+		err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)
+	}
+
 	// we can't update/find metadata here as IA will "ingest" uploaded file(s)
 	// upon uploads. (you can find its progress at https://archive.org/history/ItemNameHere )
 	// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)

backend/internxt/auth.go

@@ -28,6 +28,7 @@ type userInfo struct {
 	Bucket       string
 	BridgeUser   string
 	UserID       string
+	NewToken     string
 }
 
 type userInfoConfig struct {
@@ -61,6 +62,7 @@ func getUserInfo(ctx context.Context, cfg *userInfoConfig) (*userInfo, error) {
 		Bucket:       resp.User.Bucket,
 		BridgeUser:   resp.User.BridgeUser,
 		UserID:       resp.User.UserID,
+		NewToken:     resp.NewToken,
 	}
 
 	fs.Debugf(nil, "User info: rootFolderId=%s, bucket=%s",

backend/internxt/internxt.go

@@ -54,8 +54,7 @@ func (f *Fs) shouldRetry(ctx context.Context, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
 		return false, err
 	}
-	var httpErr *sdkerrors.HTTPError
-	if errors.As(err, &httpErr) {
+	if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
 		switch httpErr.StatusCode() {
 		case 401:
 			if !f.authFailed {
@@ -367,6 +366,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.bridgeUser = userInfo.BridgeUser
 	f.userID = userInfo.UserID
 
+	// The refresh endpoint rotates the token on every successful call.
+	// Persist the rotated token so routine use keeps the stored token
+	// current; otherwise it keeps its original expiry and accounts that
+	// cannot re-login non-interactively (2FA) eventually strand.
+	if userInfo.NewToken != "" {
+		if rotated, rotErr := jwtToOAuth2Token(userInfo.NewToken); rotErr != nil {
+			fs.Debugf(f, "Not adopting rotated token from user info: %v", rotErr)
+		} else {
+			// Use the rotated token for this session even if saving it
+			// fails; persistence is best-effort.
+			f.cfg.Token = userInfo.NewToken
+			if putErr := oauthutil.PutToken(name, m, rotated, false); putErr != nil {
+				fs.Debugf(f, "Failed to save rotated token from user info: %v", putErr)
+			} else {
+				fs.Debugf(f, "Persisted rotated token from user info, expiry: %v", rotated.Expiry)
+			}
+		}
+	}
+
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
@@ -456,8 +474,10 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	if len(childFolders) > 0 {
-		return fs.ErrorDirectoryNotEmpty
+	for _, e := range childFolders {
+		if !recent.stale(e.UUID, id, e.PlainName) {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 
 	var childFiles []folders.File
@@ -469,8 +489,10 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	if len(childFiles) > 0 {
-		return fs.ErrorDirectoryNotEmpty
+	for _, e := range childFiles {
+		if !recent.stale(e.UUID, id, joinNameExt(e.PlainName, e.Type)) {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 
 	// Delete the directory
@@ -484,6 +506,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
+	recent.deleted(id)
 
 	f.dirCache.FlushDir(dir)
 	return nil
@@ -502,6 +525,9 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, e
 		return "", false, err
 	}
 	for _, e := range entries {
+		if recent.stale(e.UUID, pathID, e.PlainName) {
+			continue
+		}
 		if f.opt.Encoding.ToStandardName(e.PlainName) == leaf {
 			return e.UUID, true, nil
 		}
@@ -538,46 +564,166 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (string, error)
 	return resp.UUID, nil
 }
 
-// preUploadCheck checks if a file exists in the given directory
-// Returns the file metadata if it exists, nil if not
-func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
-	// Parse name and extension from the leaf
-	baseName := f.opt.Encoding.FromStandardName(leaf)
-	name := strings.TrimSuffix(baseName, path.Ext(baseName))
-	ext := strings.TrimPrefix(path.Ext(baseName), ".")
+// splitNameExt splits a basename into the (plainName, type) pair that Internxt
+// stores a file under.
+func splitNameExt(baseName string) (name, ext string) {
+	ext = path.Ext(baseName)
+	if ext == baseName {
+		return baseName, ""
+	}
+	return strings.TrimSuffix(baseName, ext), strings.TrimPrefix(ext, ".")
+}
 
-	checkResult, err := files.CheckFilesExistence(ctx, f.cfg, directoryID, []files.FileExistenceCheck{
-		{
-			PlainName:    name,
-			Type:         ext,
-			OriginalFile: struct{}{},
-		},
+// joinNameExt reassembles the basename a stored (plainName, type) pair
+// displays as, the way List builds its entries.
+func joinNameExt(name, ext string) string {
+	if ext == "" {
+		return name
+	}
+	return name + "." + ext
+}
+
+// legacyNameExt splits a basename at the final dot, the way this backend used
+// to before it adopted the convention the other clients share. It differs from
+// splitNameExt only for a name whose sole dot leads it: ".bashrc" splits into
+// an empty name of type "bashrc".
+func legacyNameExt(baseName string) (name, ext string) {
+	return strings.TrimSuffix(baseName, path.Ext(baseName)), strings.TrimPrefix(path.Ext(baseName), ".")
+}
+
+// existenceCheck builds a lookup criterion for one (plainName, type) spelling.
+func existenceCheck(name, ext string) files.FileExistenceCheck {
+	return files.FileExistenceCheck{
+		PlainName:    name,
+		Type:         ext,
+		OriginalFile: struct{}{},
+	}
+}
+
+// staleWindow is how long the API may carry on returning an item from
+// where it was before this process moved or deleted it.
+const staleWindow = time.Minute
+
+// location is where this process last put an item. An empty parentUUID
+// means the item was deleted.
+type location struct {
+	parentUUID string
+	name       string // encoded name, including any extension
+	expires    time.Time
+}
+
+// recentChanges holds the files and folders this process has moved or
+// deleted recently, keyed by UUID.
+//
+// The API serves lookups and listings from read replicas which lag behind
+// writes, so an item can be returned from its old location for a while after
+// it was moved or deleted. Acting on such a stale entry can be destructive,
+// eg overwriting it would delete the file from where it was moved to.
+type recentChanges struct {
+	mu        sync.Mutex
+	items     map[string]location
+	lastPrune time.Time
+}
+
+// recent is shared by all the Fs in the process as UUIDs are global.
+var recent = recentChanges{items: map[string]location{}}
+
+// set records the new location of uuid, pruning expired entries.
+func (r *recentChanges) set(uuid string, loc location) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	now := time.Now()
+	if now.Sub(r.lastPrune) > staleWindow {
+		for k, v := range r.items {
+			if now.After(v.expires) {
+				delete(r.items, k)
+			}
+		}
+		r.lastPrune = now
+	}
+	loc.expires = now.Add(staleWindow)
+	r.items[uuid] = loc
+}
+
+// moved records that uuid was moved to name in parentUUID.
+func (r *recentChanges) moved(uuid, parentUUID, name string) {
+	r.set(uuid, location{parentUUID: parentUUID, name: name})
+}
+
+// deleted records that uuid was deleted.
+func (r *recentChanges) deleted(uuid string) {
+	r.set(uuid, location{})
+}
+
+// stale reports whether an API result saying uuid is called name in
+// parentUUID is out of date because of a change made by this process.
+func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	loc, ok := r.items[uuid]
+	if !ok || time.Now().After(loc.expires) {
+		return false
+	}
+	return loc.parentUUID != parentUUID || loc.name != name
+}
+
+// findFile looks up a single file by name within directoryID. Returns
+// (nil, nil) when the file does not exist; surfaces transport/API errors.
+func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
+	encodedName := f.opt.Encoding.FromStandardName(leaf)
+	name, ext := splitNameExt(encodedName)
+	checks := []files.FileExistenceCheck{existenceCheck(name, ext)}
+	if encodedName != name {
+		checks = append(checks, existenceCheck(encodedName, ""))
+	}
+	if legacyName, legacyExt := legacyNameExt(encodedName); legacyExt != "" && legacyName != name {
+		checks = append(checks, existenceCheck(legacyName, legacyExt))
+	}
+
+	var checkResult *files.CheckFilesExistenceResponse
+	err := f.pacer.Call(func() (bool, error) {
+		var err error
+		checkResult, err = files.CheckFilesExistence(ctx, f.cfg, directoryID, checks)
+		if isNotFoundError(err) {
+			return true, err
+		}
+		return f.shouldRetry(ctx, err)
 	})
-
 	if err != nil {
-		// If existence check fails, assume file doesn't exist to allow upload to proceed
+		return nil, err
+	}
+
+	var uuid string
+	for _, result := range checkResult.Files {
+		if !result.FileExists() || result.UUID == "" {
+			continue
+		}
+		if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
+			fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
+			continue
+		}
+		if joinNameExt(result.PlainName, result.Type) == encodedName {
+			uuid = result.UUID
+			break
+		}
+	}
+	if uuid == "" {
 		return nil, nil
 	}
 
-	if len(checkResult.Files) > 0 && checkResult.Files[0].FileExists() {
-		result := checkResult.Files[0]
-		if result.Type != ext {
-			return nil, nil
-		}
-
-		existingUUID := result.UUID
-		if existingUUID != "" {
-			fileMeta, err := files.GetFileMeta(ctx, f.cfg, existingUUID)
-			if err == nil && fileMeta != nil {
-				return convertFileMetaToFile(fileMeta), nil
-			}
-
-			if err != nil {
-				return nil, err
-			}
-		}
+	var fileMeta *files.FileMeta
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		fileMeta, err = files.GetFileMeta(ctx, f.cfg, uuid)
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
 	}
-	return nil, nil
+	if fileMeta == nil {
+		return nil, nil
+	}
+	return convertFileMetaToFile(fileMeta), nil
 }
 
 // convertFileMetaToFile converts files.FileMeta to folders.File
@@ -615,6 +761,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		return nil, err
 	}
 	for _, e := range foldersList {
+		if recent.stale(e.UUID, dirID, e.PlainName) {
+			continue
+		}
 		remote := path.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
 		out = append(out, fs.NewDir(remote, e.ModificationTime))
 	}
@@ -632,6 +781,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		if len(e.Type) > 0 {
 			remote += "." + e.Type
 		}
+		if recent.stale(e.UUID, dirID, remote) {
+			continue
+		}
 		remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
 		out = append(out, newObjectWithFile(f, remote, &e))
 	}
@@ -656,8 +808,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return nil, err
 	}
 
-	// Check if file already exists
-	existingFile, err := f.preUploadCheck(ctx, leaf, directoryID)
+	existingFile, err := f.findFile(ctx, leaf, directoryID)
 	if err != nil {
 		return nil, err
 	}
@@ -705,44 +856,28 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
 	if err != nil {
 		return err
 	}
+	recent.deleted(dirID)
 	f.dirCache.FlushDir(remote)
 	return nil
 }
 
-// NewObject creates a new object
+// NewObject creates a new object by looking up a single file's metadata.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	parentDir := path.Dir(remote)
-
-	if parentDir == "." {
-		parentDir = ""
-	}
-
-	dirID, err := f.dirCache.FindDir(ctx, parentDir, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
-		return nil, fs.ErrorObjectNotFound
+		if err == fs.ErrorDirNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
 	}
-
-	var files []folders.File
-	err = f.pacer.Call(func() (bool, error) {
-		var err error
-		files, err = folders.ListAllFiles(ctx, f.cfg, dirID)
-		return f.shouldRetry(ctx, err)
-	})
+	file, err := f.findFile(ctx, leaf, directoryID)
 	if err != nil {
 		return nil, err
 	}
-	targetName := path.Base(remote)
-	for _, e := range files {
-		name := e.PlainName
-		if len(e.Type) > 0 {
-			name += "." + e.Type
-		}
-		decodedName := f.opt.Encoding.ToStandardName(name)
-		if decodedName == targetName {
-			return newObjectWithFile(f, remote, &e), nil
-		}
+	if file == nil {
+		return nil, fs.ErrorObjectNotFound
 	}
-	return nil, fs.ErrorObjectNotFound
+	return newObjectWithFile(f, remote, file), nil
 }
 
 // newObjectWithFile returns a new object by file info
@@ -846,19 +981,23 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	// Parse name and extension from the leaf
-	baseName := f.opt.Encoding.FromStandardName(leaf)
-	newName := strings.TrimSuffix(baseName, path.Ext(baseName))
-	newType := strings.TrimPrefix(path.Ext(baseName), ".")
+	newName, newType := splitNameExt(f.opt.Encoding.FromStandardName(leaf))
 
 	// Move the file server-side
 	err = f.pacer.Call(func() (bool, error) {
 		err := files.MoveFile(ctx, f.cfg, srcObj.uuid, directoryID, newName, newType)
+		// The move is checked against lagging read replicas so it can
+		// fail with 404 if the destination directory was just created
+		// or 409 if a file at the destination was just deleted.
+		if err != nil && isStaleReadError(err, true) {
+			return true, err
+		}
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
 	}
+	recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
 
 	dstObj := &Object{
 		f:       f,
@@ -894,11 +1033,17 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	encodedLeaf := f.opt.Encoding.FromStandardName(dstLeaf)
 	err = f.pacer.Call(func() (bool, error) {
 		err := folders.MoveFolder(ctx, f.cfg, srcID, dstDirectoryID, encodedLeaf)
+		// Moving a large folder can outlast the gateway timeout (520/502)
+		// yet still complete, in which case the retry reports this.
+		if err != nil && isConflictError(err) && strings.Contains(err.Error(), "already moved to that location") {
+			return false, nil
+		}
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
 	}
+	recent.moved(srcID, dstDirectoryID, encodedLeaf)
 
 	srcFs.dirCache.FlushDir(srcRemote)
 	return nil
@@ -946,9 +1091,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	remote := o.remote
 
-	origBaseName := path.Base(remote)
-	origName := strings.TrimSuffix(origBaseName, path.Ext(origBaseName))
-	origType := strings.TrimPrefix(path.Ext(origBaseName), ".")
+	origName, origType := splitNameExt(path.Base(remote))
 
 	// Create directory if it doesn't exist
 	_, dirID, err := o.f.dirCache.FindPath(ctx, remote, true)
@@ -965,11 +1108,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Step 1: If file exists, rename to backup (preserves old file during upload)
 	if oldUUID != "" {
-		// Generate unique backup name
-		baseName := path.Base(remote)
-		name := strings.TrimSuffix(baseName, path.Ext(baseName))
-		ext := strings.TrimPrefix(path.Ext(baseName), ".")
-
+		name, ext := splitNameExt(path.Base(remote))
 		backupSuffix := fmt.Sprintf(".rclone-backup-%s", random.String(8))
 		backupName = o.f.opt.Encoding.FromStandardName(name + backupSuffix)
 		backupType = ext
@@ -1042,6 +1181,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			return o.f.tooLargeError(remote, tooLarge)
 		}
 
+		if err != nil && isStaleReadError(err, backupUUID != "") {
+			o.restoreBackupFile(ctx, backupUUID, origName, origType)
+			return fserrors.RetryError(err)
+		}
+
 		if err != nil {
 			meta, err = o.recoverFromTimeoutConflict(ctx, err, remote, dirID)
 		}
@@ -1064,8 +1208,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		err := o.f.pacer.Call(func() (bool, error) {
 			err := files.DeleteFile(ctx, o.f.cfg, backupUUID)
 			if err != nil {
-				var httpErr *sdkerrors.HTTPError
-				if errors.As(err, &httpErr) {
+				if httpErr, ok := errors.AsType[*sdkerrors.HTTPError](err); ok {
 					// Treat 404 (Not Found) and 204 (No Content) as success
 					switch httpErr.StatusCode() {
 					case 404, 204:
@@ -1080,6 +1223,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				backupName, backupType, backupUUID, err)
 			// Don't fail the upload just because backup deletion failed
 		} else {
+			recent.deleted(backupUUID)
 			fs.Debugf(o.f, "Successfully deleted backup file")
 		}
 	}
@@ -1099,6 +1243,19 @@ func isTimeoutError(err error) bool {
 	return false
 }
 
+// isNotFoundError reports whether err is a 404 from the API.
+func isNotFoundError(err error) bool {
+	var httpErr *sdkerrors.HTTPError
+	return errors.As(err, &httpErr) && httpErr.StatusCode() == 404
+}
+
+// isStaleReadError reports whether err is the API contradicting a write this
+// backend has already made, which it does because lookups are served by read
+// replicas that lag behind the primary.
+func isStaleReadError(err error, renamed bool) bool {
+	return isNotFoundError(err) || (renamed && isConflictError(err))
+}
+
 // isConflictError checks if an error indicates a file conflict (409)
 func isConflictError(err error) bool {
 	errMsg := err.Error()
@@ -1117,8 +1274,7 @@ func isEmptyFileLimitError(err error) bool {
 // fileTooLargeError extracts the SDK's FileTooLargeError from a wrapped error
 // chain, returning it (or nil) so callers can branch on the size limit.
 func fileTooLargeError(err error) *sdkerrors.FileTooLargeError {
-	var tooLarge *sdkerrors.FileTooLargeError
-	if errors.As(err, &tooLarge) {
+	if tooLarge, ok := errors.AsType[*sdkerrors.FileTooLargeError](err); ok {
 		return tooLarge
 	}
 	return nil
@@ -1140,40 +1296,22 @@ func (o *Object) recoverFromTimeoutConflict(ctx context.Context, uploadErr error
 	}
 
 	baseName := path.Base(remote)
-	encodedName := o.f.opt.Encoding.FromStandardName(baseName)
 
-	var meta *buckets.CreateMetaResponse
-	checkErr := o.f.pacer.Call(func() (bool, error) {
-		existingFile, err := o.f.preUploadCheck(ctx, encodedName, dirID)
-		if err != nil {
-			return o.f.shouldRetry(ctx, err)
-		}
-		if existingFile != nil {
-			name := strings.TrimSuffix(baseName, path.Ext(baseName))
-			ext := strings.TrimPrefix(path.Ext(baseName), ".")
-
-			meta = &buckets.CreateMetaResponse{
-				UUID:      existingFile.UUID,
-				FileID:    existingFile.FileID,
-				Name:      name,
-				PlainName: name,
-				Type:      ext,
-				Size:      existingFile.Size,
-			}
-			o.id = existingFile.FileID
-		}
-		return false, nil
-	})
-
-	if checkErr != nil {
+	existingFile, err := o.f.findFile(ctx, baseName, dirID)
+	if err != nil || existingFile == nil {
 		return nil, uploadErr
 	}
 
-	if meta != nil {
-		return meta, nil
-	}
-
-	return nil, uploadErr
+	name, ext := splitNameExt(baseName)
+	o.id = existingFile.FileID
+	return &buckets.CreateMetaResponse{
+		UUID:      existingFile.UUID,
+		FileID:    existingFile.FileID,
+		Name:      name,
+		PlainName: name,
+		Type:      ext,
+		Size:      existingFile.Size,
+	}, nil
 }
 
 // restoreBackupFile restores a backup file after upload failure
@@ -1191,8 +1329,13 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or
 
 // Remove deletes a file
 func (o *Object) Remove(ctx context.Context) error {
-	return o.f.pacer.Call(func() (bool, error) {
+	err := o.f.pacer.Call(func() (bool, error) {
 		err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
 		return o.f.shouldRetry(ctx, err)
 	})
+	if err != nil {
+		return err
+	}
+	recent.deleted(o.uuid)
+	return nil
 }

backend/internxt/internxt_internal_test.go

@@ -0,0 +1,191 @@
+package internxt
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/internxt/rclone-adapter/config"
+	"github.com/internxt/rclone-adapter/endpoints"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// storedFile is a file as the service holds it: a (plainName, type) pair
+// split however the client that uploaded it chose to split it.
+type storedFile struct {
+	uuid      string
+	plainName string
+	fileType  string
+}
+
+// wireType renders a stored type the way the service does: a file with no
+// extension comes back as JSON null, not an empty string.
+func wireType(fileType string) any {
+	if fileType == "" {
+		return nil
+	}
+	return fileType
+}
+
+// existenceServer answers the existence check and file meta endpoints from
+// files, matching plainName exactly and only constraining type when the
+// criterion supplies one, as the service does.
+func existenceServer(t *testing.T, stored []storedFile) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if strings.HasSuffix(r.URL.Path, "/meta") {
+			uuid := strings.Split(strings.TrimPrefix(r.URL.Path, "/drive/files/"), "/")[0]
+			for _, f := range stored {
+				if f.uuid == uuid {
+					require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+						"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
+					}))
+					return
+				}
+			}
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		var req struct {
+			Files []struct {
+				PlainName string `json:"plainName"`
+				Type      string `json:"type"`
+			} `json:"files"`
+		}
+		require.NoError(t, json.NewDecoder(r.Body).Decode(&req))
+		var existent []map[string]any
+		for _, f := range stored {
+			for _, c := range req.Files {
+				if c.PlainName == f.plainName && (c.Type == "" || c.Type == f.fileType) {
+					existent = append(existent, map[string]any{
+						"exists": true, "status": "EXISTS",
+						"uuid": f.uuid, "plainName": f.plainName, "type": wireType(f.fileType),
+					})
+					break
+				}
+			}
+		}
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"existentFiles": existent}))
+	}))
+}
+
+func testFs(t *testing.T, server *httptest.Server) *Fs {
+	t.Helper()
+	f := &Fs{
+		opt: Options{Encoding: encoder.EncodeInvalidUtf8},
+		cfg: &config.Config{
+			HTTPClient: server.Client(),
+			Endpoints:  endpoints.NewConfig(server.URL),
+		},
+	}
+	f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+	return f
+}
+
+func TestSplitJoinNameExt(t *testing.T) {
+	for _, test := range []struct {
+		baseName string
+		name     string
+		ext      string
+	}{
+		{"foo.txt", "foo", "txt"},
+		{"foo.tar.gz", "foo.tar", "gz"},
+		{"README", "README", ""},
+		{".bashrc", ".bashrc", ""},
+		{"..hidden", ".", "hidden"},
+		{"", "", ""},
+	} {
+		name, ext := splitNameExt(test.baseName)
+		assert.Equal(t, test.name, name, test.baseName)
+		assert.Equal(t, test.ext, ext, test.baseName)
+		assert.Equal(t, test.baseName, joinNameExt(name, ext), test.baseName)
+	}
+}
+
+func TestFindFile(t *testing.T) {
+	for _, test := range []struct {
+		what     string
+		stored   []storedFile
+		leaf     string
+		wantUUID string
+	}{{
+		what:     "split as rclone splits it",
+		stored:   []storedFile{{"uuid-1", "foo.tar", "gz"}},
+		leaf:     "foo.tar.gz",
+		wantUUID: "uuid-1",
+	}, {
+		what:     "whole name stored as plainName by another client",
+		stored:   []storedFile{{"uuid-2", "foo.tar.gz", ""}},
+		leaf:     "foo.tar.gz",
+		wantUUID: "uuid-2",
+	}, {
+		what:     "extensionless name is not confused with an extended one",
+		stored:   []storedFile{{"uuid-3", "README", "md"}, {"uuid-4", "README", ""}},
+		leaf:     "README",
+		wantUUID: "uuid-4",
+	}, {
+		what:     "dotfile stored the way the web client splits it",
+		stored:   []storedFile{{"uuid-6", ".bashrc", ""}},
+		leaf:     ".bashrc",
+		wantUUID: "uuid-6",
+	}, {
+		what:     "dotfile stored the way older rclone versions split it",
+		stored:   []storedFile{{"uuid-7", "", "bashrc"}},
+		leaf:     ".bashrc",
+		wantUUID: "uuid-7",
+	}, {
+		what:     "trailing dot kept in plainName by a rename",
+		stored:   []storedFile{{"uuid-8", "trailing.", ""}},
+		leaf:     "trailing.",
+		wantUUID: "uuid-8",
+	}, {
+		what:   "missing file",
+		stored: []storedFile{{"uuid-5", "other", "txt"}},
+		leaf:   "foo.txt",
+	}} {
+		t.Run(test.what, func(t *testing.T) {
+			server := existenceServer(t, test.stored)
+			defer server.Close()
+			file, err := testFs(t, server).findFile(context.Background(), test.leaf, "dir-uuid")
+			require.NoError(t, err)
+			if test.wantUUID == "" {
+				assert.Nil(t, file)
+				return
+			}
+			require.NotNil(t, file)
+			assert.Equal(t, test.wantUUID, file.UUID)
+		})
+	}
+}
+
+func TestFindFileIgnoresStale(t *testing.T) {
+	ctx := context.Background()
+	server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})
+	defer server.Close()
+	f := testFs(t, server)
+
+	recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")
+	recent.deleted("uuid-deleted")
+	recent.moved("uuid-renamed", "dir-uuid", "baz.txt")
+
+	file, err := f.findFile(ctx, "foo.txt", "dir-uuid")
+	require.NoError(t, err)
+	assert.Nil(t, file, "moved away")
+
+	file, err = f.findFile(ctx, "bar.txt", "dir-uuid")
+	require.NoError(t, err)
+	assert.Nil(t, file, "deleted")
+
+	file, err = f.findFile(ctx, "baz.txt", "dir-uuid")
+	require.NoError(t, err)
+	require.NotNil(t, file, "moved here")
+	assert.Equal(t, "uuid-renamed", file.UUID)
+}

backend/internxt/upload.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"path"
 	"sort"
-	"strings"
 	"sync"
 
 	"github.com/internxt/rclone-adapter/buckets"
@@ -263,9 +262,7 @@ func (w *internxtChunkWriter) Close(ctx context.Context) error {
 	}
 
 	// Create file metadata in Internxt Drive
-	baseName := w.f.opt.Encoding.FromStandardName(path.Base(w.remote))
-	name := strings.TrimSuffix(baseName, path.Ext(baseName))
-	ext := strings.TrimPrefix(path.Ext(baseName), ".")
+	name, ext := splitNameExt(w.f.opt.Encoding.FromStandardName(path.Base(w.remote)))
 
 	var meta *buckets.CreateMetaResponse
 	err = w.f.pacer.Call(func() (bool, error) {

backend/jottacloud/jottacloud.go

@@ -34,6 +34,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/list"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
@@ -1924,8 +1925,9 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 	// nothing to clean up by default
 	cleanup = func() {}
 
-	// don't cache small files on disk to reduce wear of the disk
-	if size > threshold {
+	// don't cache small files on disk to reduce wear of the disk, but
+	// spool unknown sized streams there as they can't be bounded in memory
+	if size > threshold || size < 0 {
 		var tempFile *os.File
 
 		// create the cache file
@@ -1955,15 +1957,15 @@ func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader,
 		// replace the already read source with a reader of our cached file
 		out = tempFile
 	} else {
-		// that's a small file, just read it into memory
-		var inData []byte
-		inData, err = io.ReadAll(teeReader)
-		if err != nil {
+		// that's a small file, just read it into memory from the global pool
+		rw := multipart.NewRW()
+		cleanup = func() {
+			_ = rw.Close()
+		}
+		if _, err = io.Copy(rw, teeReader); err != nil {
 			return
 		}
-
-		// set the reader to our read memory block
-		out = bytes.NewReader(inData)
+		out = rw
 	}
 	return hex.EncodeToString(md5Hasher.Sum(nil)), out, cleanup, nil
 }

backend/jottacloud/jottacloud_internal_test.go

@@ -5,12 +5,14 @@ import (
 	"crypto/md5"
 	"fmt"
 	"io"
+	"os"
 	"testing"
 	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
@@ -28,11 +30,16 @@ func TestReadMD5(t *testing.T) {
 			wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
 			for _, threshold := range []int64{512, 1024, 10 * 1024, 20 * 1024} {
 				t.Run(fmt.Sprintf("%d", threshold), func(t *testing.T) {
+					inUse := pool.Global().InUse()
 					in := readers.NewPatternReader(size)
 					gotMD5, out, cleanup, err := readMD5(in, size, threshold)
-					defer cleanup()
 					require.NoError(t, err)
 					assert.Equal(t, wantMD5, gotMD5)
+					if size > threshold {
+						assert.IsType(t, (*os.File)(nil), out, "big files should be spooled to disk")
+					} else {
+						assert.IsType(t, (*pool.RW)(nil), out, "small files should be buffered in the pool")
+					}
 
 					// check md5hash of out
 					hasher := md5.New()
@@ -41,10 +48,36 @@ func TestReadMD5(t *testing.T) {
 					assert.Equal(t, n, size)
 					outMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
 					assert.Equal(t, wantMD5, outMD5)
+
+					// check cleanup returns the buffer to the pool
+					cleanup()
+					assert.Equal(t, inUse, pool.Global().InUse(), "pool buffers leaked")
 				})
 			}
 		})
 	}
+
+	// Unknown sized streams can't be bounded in memory so must be spooled to disk
+	t.Run("UnknownSize", func(t *testing.T) {
+		const size = 1024
+		inUse := pool.Global().InUse()
+		hasher := md5.New()
+		_, err := io.Copy(hasher, readers.NewPatternReader(size))
+		require.NoError(t, err)
+		wantMD5 := fmt.Sprintf("%x", hasher.Sum(nil))
+
+		gotMD5, out, cleanup, err := readMD5(readers.NewPatternReader(size), -1, 10*size)
+		require.NoError(t, err)
+		assert.Equal(t, wantMD5, gotMD5)
+		assert.IsType(t, (*os.File)(nil), out)
+		hasher.Reset()
+		n, err := io.Copy(hasher, out)
+		require.NoError(t, err)
+		assert.Equal(t, int64(size), n)
+		assert.Equal(t, wantMD5, fmt.Sprintf("%x", hasher.Sum(nil)))
+		cleanup()
+		assert.Equal(t, inUse, pool.Global().InUse())
+	})
 }
 
 func (f *Fs) InternalTestMetadata(t *testing.T) {

backend/linkbox/linkbox.go

@@ -11,10 +11,10 @@ package linkbox
 */
 
 import (
-	"bytes"
 	"context"
 	"crypto/md5"
 	"crypto/tls"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -35,7 +35,9 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -46,7 +48,8 @@ const (
 	pacerBurst         = 1
 	linkboxAPIURL      = "https://www.linkbox.to/api/open/"
 	linkboxWebAPIURL   = "https://www.linkbox.to/api/"
-	rootID             = "0" // ID of root directory
+	rootID             = "0"              // ID of root directory
+	prefixSize         = 10 * 1024 * 1024 // the API wants the MD5 of the first 10 MiB of a file
 )
 
 func init() {
@@ -667,6 +670,21 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 	return res.Body, nil
 }
 
+// readPrefix reads up to prefixSize bytes of in into a buffer from the
+// global pool returning it along with the hex MD5 of what was read.
+//
+// The caller must Close the returned buffer when done with it.
+func readPrefix(in io.Reader) (rw *pool.RW, md5sum string, err error) {
+	rw = multipart.NewRW()
+	hasher := md5.New()
+	_, err = io.CopyN(rw, io.TeeReader(in, hasher), prefixSize)
+	if err != nil && err != io.EOF {
+		_ = rw.Close()
+		return nil, "", err
+	}
+	return rw, hex.EncodeToString(hasher.Sum(nil)), nil
+}
+
 // Update in to the object with the modTime given of the given size
 //
 // When called from outside an Fs by rclone, src.Size() will always be >= 0.
@@ -701,11 +719,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 	}
 
-	first10m := io.LimitReader(in, 10_485_760)
-	first10mBytes, err := io.ReadAll(first10m)
+	prefix, prefixMD5, err := readPrefix(in)
 	if err != nil {
 		return fmt.Errorf("Update err in reading file: %w", err)
 	}
+	defer func() {
+		_ = prefix.Close()
+	}()
 
 	// get upload authorization (step 1)
 	opts := &rest.Opts{
@@ -715,7 +735,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Options: options,
 		Parameters: url.Values{
 			"token":           {o.fs.opt.Token},
-			"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
+			"fileMd5ofPre10m": {prefixMD5},
 			"fileSize":        {itoa64(size)},
 		},
 	}
@@ -757,7 +777,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			return fmt.Errorf("head upload URL: %w", err)
 		}
 
-		file := io.MultiReader(bytes.NewReader(first10mBytes), in)
+		file := io.MultiReader(prefix, in)
 
 		opts.Method = "PUT"
 		opts.Body = file
@@ -797,7 +817,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Options: options,
 		Parameters: url.Values{
 			"token":           {o.fs.opt.Token},
-			"fileMd5ofPre10m": {fmt.Sprintf("%x", md5.Sum(first10mBytes))},
+			"fileMd5ofPre10m": {prefixMD5},
 			"fileSize":        {itoa64(size)},
 			"pid":             {dirID},
 			"diyName":         {leaf},
@@ -1019,8 +1039,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 	// intermittently returns an HTML challenge page with a 200
 	// status instead of JSON. This surfaces as a JSON syntax error,
 	// so retry it to let the pacer back off until the block lifts.
-	var syntaxErr *json.SyntaxError
-	if errors.As(err, &syntaxErr) {
+	if _, ok := errors.AsType[*json.SyntaxError](err); ok {
 		return true, err
 	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err