docs: bring the Proton Drive handoff up to date after the final review

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.claude/settings.json

@@ -0,0 +1,25 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git status:*)", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)",
+      "Bash(git rev-parse:*)", "Bash(git add:*)", "Bash(git commit:*)",
+      "Bash(go mod download:*)", "Bash(go build:*)", "Bash(go test:*)", "Bash(go vet:*)",
+      "Bash(gofmt -l:*)",
+      "Bash(golangci-lint run:*)"
+    ],
+    "ask": [
+      "Bash(git push:*)", "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git switch:*)",
+      "Bash(git restore:*)", "Bash(git commit --amend:*)", "Bash(rm:*)",
+      "Bash(go get:*)", "Bash(go mod tidy:*)", "Bash(go install:*)",
+      "Bash(make:*)", "Bash(docker:*)"
+    ],
+    "deny": [
+      "Bash(git push --force:*)", "Bash(git push -f:*)", "Bash(git push --force-with-lease:*)",
+      "Bash(git reset --hard:*)", "Bash(git clean:*)", "Bash(git add -f:*)", "Bash(git add --force:*)",
+      "Bash(git commit --no-verify:*)", "Bash(git commit -n:*)",
+      "Bash(rm -r:*)", "Bash(rm -R:*)", "Bash(rm -rf:*)", "Bash(rm -fr:*)", "Bash(rm -Rf:*)",
+      "Bash(rm -fR:*)", "Bash(rm --recursive:*)", "Bash(sudo:*)",
+      "Read(./.env)", "Read(./.env.*)", "Read(./secrets/**)"
+    ]
+  }
+}

.codex/config.toml

@@ -0,0 +1,2 @@
+sandbox_mode = "workspace-write"
+approval_policy = "on-request"

.codex/rules/project.rules

@@ -0,0 +1,20 @@
+# Rendered from docs/PERMISSIONS.md - change both in the same commit.
+# Deny
+prefix_rule(pattern = ["git", "push", ["--force", "-f", "--force-with-lease"]], decision = "forbidden",
+            justification = "Force-push rewrites shared history.")
+prefix_rule(pattern = ["git", "reset", "--hard"], decision = "forbidden", justification = "Discards work.")
+prefix_rule(pattern = ["git", "clean"], decision = "forbidden", justification = "Deletes untracked files.")
+prefix_rule(pattern = ["git", "add", ["-f", "--force"]], decision = "forbidden",
+            justification = "Stages ignored files, e.g. secrets.")
+prefix_rule(pattern = ["git", "commit", ["--no-verify", "-n"]], decision = "forbidden", justification = "Skips hooks.")
+prefix_rule(pattern = ["rm", ["-r", "-R", "-rf", "-fr", "-Rf", "-fR", "--recursive"]], decision = "forbidden",
+            justification = "Recursive delete.")
+prefix_rule(pattern = ["sudo"], decision = "forbidden", justification = "No privilege escalation.")
+# Ask
+prefix_rule(pattern = ["git", ["push", "merge", "rebase", "switch", "restore"]], decision = "prompt")
+prefix_rule(pattern = ["git", "commit", "--amend"], decision = "prompt")
+prefix_rule(pattern = ["rm"], decision = "prompt")
+prefix_rule(pattern = ["go", ["get", "install"]], decision = "prompt")
+prefix_rule(pattern = ["go", "mod", "tidy"], decision = "prompt")
+prefix_rule(pattern = ["make"], decision = "prompt")
+prefix_rule(pattern = ["docker"], decision = "prompt")

.gemini/settings.json

@@ -0,0 +1,20 @@
+{
+  "tools": {
+    "allowed": [
+      "run_shell_command(git status)", "run_shell_command(git diff)", "run_shell_command(git log)",
+      "run_shell_command(git show)", "run_shell_command(git branch --list)",
+      "run_shell_command(git add)", "run_shell_command(git commit)",
+      "run_shell_command(go mod download)", "run_shell_command(go build)",
+      "run_shell_command(go test)", "run_shell_command(go vet)", "run_shell_command(gofmt -l)",
+      "run_shell_command(golangci-lint run)"
+    ],
+    "exclude": [
+      "run_shell_command(git push --force)", "run_shell_command(git push -f)",
+      "run_shell_command(git push --force-with-lease)", "run_shell_command(git reset --hard)",
+      "run_shell_command(git clean)", "run_shell_command(git add -f)", "run_shell_command(git add --force)",
+      "run_shell_command(git commit --no-verify)", "run_shell_command(git commit -n)",
+      "run_shell_command(rm -r)", "run_shell_command(rm -R)", "run_shell_command(rm -rf)",
+      "run_shell_command(rm -fr)", "run_shell_command(rm -Rf)", "run_shell_command(sudo)"
+    ]
+  }
+}

.gitattributes

@@ -2,9 +2,13 @@
 go.mod text eol=lf
 go.sum text eol=lf
 
+# Embedded templates must render the same on every OS
+lib/http/templates/** text eol=lf
+
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true
+cmd/bisync/testdata/*/golden/** linguist-generated=true
 
 # Don't fiddle with the line endings of test data
 **/testdata/** -text

.gitea/workflows/ci.yml

@@ -0,0 +1,56 @@
+# Fork CI for the Proton Drive backend. Every step is hermetic: the
+# tests run against the fake server in backend/protondrive/protontest.
+name: ci
+
+on:
+  push:
+  pull_request:
+
+env:
+  # Never pick up a config holding a real Proton remote.
+  RCLONE_CONFIG: /notfound
+
+jobs:
+  protondrive:
+    runs-on: ubuntu-latest
+    # The race suite takes about 6 minutes on a fast machine and 7 on
+    # two cores; the slow runner may take several times that.
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          # The line the code was verified with, not go.mod's minimum,
+          # so govulncheck sees a patched standard library.
+          go-version: '1.27.x'
+          # Caches the module and build caches, keyed on go.sum.
+          cache: true
+
+      - name: Build
+        run: go build ./...
+
+      - name: Vet
+        run: go vet ./backend/protondrive/...
+
+      - name: Lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.15.0
+          args: ./backend/protondrive/...
+
+      - name: Test
+        # Go's default 10 minute limit per test binary is too short
+        # for the race suite on a slow runner.
+        run: go test -race -timeout 40m ./backend/protondrive/...
+
+      - name: Fuzz
+        run: |
+          for target in $(go test -list '^Fuzz' ./backend/protondrive/ | grep '^Fuzz'); do
+            go test -run '^$' -fuzz "^${target}\$" -fuzztime 20s ./backend/protondrive/
+          done
+
+      - name: Vulnerability check
+        run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./backend/protondrive/...

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '~1.27.0'
+            go: '~1.27.3'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -48,14 +48,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '~1.27.0'
+            go: '~1.27.3'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '~1.27.0'
+            go: '~1.27.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -64,14 +64,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '~1.27.0'
+            go: '~1.27.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '~1.27.0'
+            go: '~1.27.3'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '~1.27.0'
+            go: '~1.27.3'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
           - job_name: go1.26
             os: ubuntu-latest
-            go: '~1.26.6'
+            go: '~1.26.10'
             quicktest: true
             racequicktest: true
 
@@ -177,9 +177,12 @@ jobs:
           # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
           # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
           unset HOMEBREW_NO_INSTALL_FROM_API
-          brew untap --force homebrew/core
-          brew untap --force homebrew/cask
-          brew untap --force aws/tap
+          # brew untap fails if the tap isn't installed, which varies between runner images
+          for tap in homebrew/core homebrew/cask aws/tap; do
+            if brew tap | grep -qx "$tap"; then
+              brew untap --force "$tap"
+            fi
+          done
           brew update
           brew install --cask macfuse
           brew install git-annex git-annex-remote-rclone
@@ -284,7 +287,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.27.0'
+          go-version: '~1.27.3'
           check-latest: true
           cache: false
           
@@ -399,7 +402,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.27.0'
+          go-version: '~1.27.3'
           # Caching is handled explicitly below to share the module cache
           # with the other jobs - see the build job for the rationale.
           cache: false

.gitignore

@@ -19,4 +19,12 @@ Thumbs.db
 __pycache__
 .DS_Store
 resource_windows_*.syso
+/librclone.h
 .devcontainer
+/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json
+.env
+.env.*
+CLAUDE.local.md
+.claude/settings.local.json
+.worktrees/
+.superpowers/

AGENTS.md

@@ -20,6 +20,8 @@ The core parts of rclone under `fs` and `vfs` need to work with all backends and
 
 Make sure added tests **actually test the code you have written** and test the intention behind the change. If you are fixing a problem, write the tests first to reproduce the problem before starting on the fix.
 
+Tests in `fs/operations`, `fs/sync`, `cmd/bisync`, `cmd/gitannex` and `vfs` (the definitive list is in `fstest/test_all/config.yaml`) are run on the integration test machine against **all** the backends, so they **must be prepared to run on backends with limited functionality**. Ideally tests, or parts of tests, should be skipped using the `Fs.Features()` flags on backends which can't support them.
+
 ## Build and Test Commands
 
 ```bash
@@ -132,3 +134,16 @@ Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck,
 - Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
 - Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
 - Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
+
+## Fork Workflow
+
+This fork carries agent tooling that upstream does not. Never include `docs/specs/`, `docs/plans/`, `docs/{ARCHITECTURE,DECISIONS,TODO,PERMISSIONS}.md`, `GEMINI.md`, `.claude/`, `.codex/`, `.gemini/` or this section in a PR to rclone/rclone.
+
+- Required skill: `superpowers` - check at session start, install if missing. There is no product UI here, so `impeccable` is not needed.
+- Specs go in `docs/specs/NNN-slug.md` (permanent). Plans go in `docs/plans/NNN-slug.md` and are removed with `git rm` before merge. This overrides the superpowers default paths.
+- Paused or in-flight work is listed under "Now" in `docs/TODO.md` with the handoff to read first.
+- Non-trivial work: brainstorm → spec → plan → TDD → review. For trivial fixes say "trivial, skipping spec" and proceed.
+- Log things noticed but deliberately not fixed in `docs/TODO.md`, record decisions in `docs/DECISIONS.md`, and keep `docs/ARCHITECTURE.md` in step with structural changes - in the same commit as the code.
+- `docs/PERMISSIONS.md` is the source for the `.claude/`, `.codex/` and `.gemini/` permission files - change them together.
+- Never push, force-push or commit directly to `master`. Ask before adding or upgrading a dependency (`go get`, `go mod tidy`).
+- Done means run in this session and seen passing: `go build`, `go test` on the touched packages, and `golangci-lint run` on them.

Dockerfile

@@ -42,6 +42,7 @@ RUN echo "**** Install Dependencies ****" && \
     apk add --no-cache \
         ca-certificates \
         fuse3 \
+        mailcap \
         tzdata && \
     echo "Enable user_allow_other in fuse" && \
     echo "user_allow_other" >> /etc/fuse.conf

GEMINI.md

@@ -0,0 +1,8 @@
+# GEMINI.md
+
+**`AGENTS.md` is the single source of truth for this repository.** Read
+`./AGENTS.md` before doing anything else and follow it exactly.
+
+This file exists only so Gemini CLI finds the rules. It contains no project
+rules of its own - add those to `AGENTS.md`. If anything here ever conflicts
+with `AGENTS.md`, `AGENTS.md` wins.

Makefile

@@ -38,6 +38,9 @@ BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
 BETA_UPLOAD_ROOT := beta.rclone.org:
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
+# Update the beta site's directory listings after an upload - add
+# --changed for the uploaded directories
+BETA_INDEX := --config bin/ci.rclone.conf -v index $(BETA_UPLOAD_ROOT) --use-server-modtime --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
@@ -234,6 +237,7 @@ check_sign:
 upload:
 	rclone -P copy build/ downloads.rclone.org:/$(TAG)
 	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
+	rclone -P index downloads.rclone.org: --changed /$(TAG)/ --checkers 16 --output index.html=html --output index.json=caddy --header-upload "Cache-Control: public, max-age=60" --exclude /favicon.ico
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -266,6 +270,7 @@ ci_upload:
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
 	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
+	./rclone $(BETA_INDEX) --changed $(BETA_PATH)/testbuilds/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,--changed test/testbuilds-latest/)
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 ci_beta:
@@ -275,11 +280,12 @@ ci_beta:
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
 	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
+	rclone $(BETA_INDEX) --changed $(BETA_PATH)/ $(if $(or $(BRANCH_PATH),$(RELEASE_TAG)),,$(if $(BETA_SUBDIR),--changed $(BETA_SUBDIR)/))
 	@echo Beta release ready at $(BETA_URL)
 
 # Fetch the binary builds from GitHub actions
 fetch_binaries:
-	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
+	rclone -P sync --exclude "/testbuilds/**" --exclude "/index.{html,json}" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server --logLevel info -w --disableFastRender --ignoreCache
@@ -296,21 +302,26 @@ retag:
 	@echo "Version is $(VERSION)"
 	git tag -f -s -m "Version $(VERSION)" $(VERSION)
 
+# Start development of the next version after a release: the next
+# minor version on master or the next patch version on a stable branch.
 startdev:
-	@echo "Version is $(VERSION)"
-	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
-	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
+	@branch=$$(git rev-parse --abbrev-ref HEAD); \
+	case "$$branch" in \
+	master) next="$(NEXT_VERSION)" ;; \
+	v*-stable) next="$(NEXT_PATCH_VERSION)" ;; \
+	*) echo "Branch $$branch isn't master or a stable branch" >&2; exit 1 ;; \
+	esac; \
+	if ! git rev-parse -q --verify "refs/tags/$(VERSION)" >/dev/null; then \
+		echo "$(VERSION) hasn't been tagged yet - has development already been started?" >&2; exit 1; \
+	fi; \
+	echo "Version is $(VERSION)"; \
+	echo "Next version is $$next"; \
+	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$$next\"\n" | gofmt > fs/versiontag.go; \
+	echo -n "$$next" > docs/layouts/partials/version.html; \
+	echo "$$next" > VERSION; \
+	git commit -m "Start $$next-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
 
-startstable:
-	@echo "Version is $(VERSION)"
-	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
-	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
+startstable: startdev
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -24,21 +24,25 @@ directories to and from different cloud storage providers.
 ## Storage providers
 
 - 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
+- Abrha Storage [:page_facing_up:](https://rclone.org/s3/#abrhastorage)
 - Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
 - Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
 - Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
 - ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
+- Autonomys Auto Drive [:page_facing_up:](https://rclone.org/s3/#autonomys)
 - Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
 - Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
 - Box [:page_facing_up:](https://rclone.org/box/)
 - Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
 - China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
 - Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
+- Clever Cloud Cellar [:page_facing_up:](https://rclone.org/s3/#clever-cloud-cellar)
 - Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
 - Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
 - Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
 - DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
 - Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
+- dosya.dev [:page_facing_up:](https://rclone.org/dosya/)
 - Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
 - Drime [:page_facing_up:](https://rclone.org/s3/#drime)
 - Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
@@ -70,8 +74,11 @@ directories to and from different cloud storage providers.
 - Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
 - IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
 - Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
+- Infomaniak Swiss Backup (S3) [:page_facing_up:](https://rclone.org/s3/#infomaniak)
+- Infomaniak Swiss Backup (Swift) [:page_facing_up:](https://rclone.org/swift/)
 - Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
 - IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
+- kDrive [:page_facing_up:](https://rclone.org/kdrive/)
 - Koofr [:page_facing_up:](https://rclone.org/koofr/)
 - Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
 - Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
@@ -87,6 +94,7 @@ directories to and from different cloud storage providers.
 - Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
 - Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
 - Minio [:page_facing_up:](https://rclone.org/s3/#minio)
+- Neon Object Storage [:page_facing_up:](https://rclone.org/s3/#neon)
 - Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
 - Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
 - OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
@@ -110,6 +118,7 @@ directories to and from different cloud storage providers.
 - Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
 - Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
 - RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
+- RelAix.Storage [:page_facing_up:](https://rclone.org/s3/#relaix)
 - rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
 - Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
 - Scality (RING / ARTESCA) [:page_facing_up:](https://rclone.org/s3/#scality)

RELEASE.md

@@ -6,27 +6,41 @@ This file describes how to make the various kinds of releases
 
 - [gh the github cli](https://github.com/cli/cli) for uploading packages
 - pandoc for making the html and man pages
+- hugo for building the website
+- golangci-lint and docker for `make check` (markdownlint runs in docker)
+- a gpg key for signing the tag and the checksums
+- rclone remotes `beta.rclone.org:`, `downloads.rclone.org:`,
+  `www.rclone.org:` and `test-rclone-org:`
+- the sponsor logos - see [Sponsor logos](#sponsor-logos)
 
 ## Making a release
 
+- Enable the [release freeze rule](https://github.com/rclone/rclone/settings/rules/24751862)
 - git checkout master # see below for stable branch
 - git pull # IMPORTANT
 - make fetch-gui-and-commit # bump the embedded GUI if rclone-web has a newer release
 - git status - make sure everything is checked in
 - Check GitHub actions build for master is Green
-- make test # see integration test server or run locally
-- make tag
-- edit docs/content/changelog.md # duplicate logs from point rels removed automatically
-  releases
+- Check the integration tests at <https://integration.rclone.org>
+- cat VERSION # check this is the version being released
+- make tag # signed tag on HEAD + new changelog entry + docs
+- edit docs/content/changelog.md # entries already in point releases are
+  removed automatically
 - make tidy
-- make doc
-- git status - to check for new man pages - git add them
+- make doc # again to pick up the changelog edits
+- git status - to check for new command docs in docs/content/commands - git add them
 - git commit -a -v -m "Version v1.XX.0"
-- make check
-- make retag
-- git push origin # without --follow-tags so it doesn't push the tag if it fails
+- make check # if this fails, fix, git commit --amend and run it again
+- make retag # moves the tag onto the Version commit
+- git push origin # without --follow-tags so it doesn't push the tag yet
+- Wait for the GitHub actions build of master to complete and check it is
+  green - this is the first time CI sees any commits which were only local,
+  such as security fixes, so it must pass before the tag is pushed
+  - if it fails: fix, git commit, make retag, git push origin and wait again
 - git push --follow-tags origin
-- \# Wait for the GitHub builds to complete then...
+- \# Wait for the GitHub builds of the tag to complete then...
+- \# Don't commit or push anything until after make upload_github as these steps
+  use the tag on HEAD to find the version
 - make fetch_binaries
 - make tarball
 - make vendorball
@@ -35,19 +49,31 @@ This file describes how to make the various kinds of releases
 - make upload
 - make upload_test_website # check sponsors are correct/unchanged
 - make upload_website
-- make upload_github
-- make startdev # make startstable for stable branch
-- \# announce with forum post, twitter post, and post to annoucements list
+- make upload_github # this triggers the docker plugin and winget workflows
+- Check the docker image, docker plugin and winget GitHub actions succeeded
+- make startdev # works out the next version for master or a stable branch
+- git push
+- Disable the release freeze rule
+- Release any security advisories
+  - request a CVE for each advisory if not done already
+  - set the patched version on each advisory and publish it
+  - when the CVEs are assigned replace `CVE-PENDING` in the changelog on
+    master and the stable branch then `make upload_website`
+- \# announce with forum post, twitter post, and post to announcements list
 
 ## Update dependencies
 
 Early in the next release cycle update the dependencies.
 
 - Active Pins
-  - `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
-    - See: [#9539](https://github.com/rclone/rclone/issues/9539)
+  - `github.com/coreos/go-systemd/v22` v22.6.0 - v22.7.0 fails to compile on netbsd
+  - `github.com/hanwen/go-fuse/v2` v2.11.0 - v2.12.0 fails to compile on 32
+    bit linux, awaiting fix for
+    [go-fuse/issues/628](https://github.com/hanwen/go-fuse/issues/628)
 - Review Active Pins in go.mod and remove if possible
 - `make updatedirect`
+- The update ignores the pins so put them back, eg
+  `go get github.com/coreos/go-systemd/v22@v22.6.0` then `go mod tidy`
 - `make GOTAGS=cmount`
 - `make compiletest`
 - Fix anything which doesn't compile at this point and commit changes here
@@ -190,20 +216,53 @@ First make the release branch.  If this is a second point release then
 this will be done already.
 
 - git co -b ${BASE_TAG}-stable ${BASE_TAG}.0
-- make startstable
+- make startdev # sets the version to ${BASE_TAG}.1
+- git push -u origin ${BASE_TAG}-stable
 
 Now
 
-- git co ${BASE_TAG}-stable
-- `git cherry-pick -x` any fixes
-- make startstable
-- Do the steps as above
+- Enable release freeze rule
 - git co master
-- `#` cherry pick the changes to the changelog - check the diff to make sure it
-  is correct
-- git checkout ${BASE_TAG}-stable docs/content/changelog.md
-- git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
+- git pull --rebase
+- make fetch-gui-and-commit # update the GUI on master first
 - git push
+- git co ${BASE_TAG}-stable
+- git pull
+- cat VERSION # check this is ${NEW_TAG} - if not run make startdev
+- `git cherry-pick -x` any fixes, including the GUI update if there was one
+  - for security releases consider updating the Go patch version in
+    `.github/workflows/build.yml` and dependencies with CVE fixes
+  - fixes may depend on commits which weren't picked so push to CI
+    and check it is green before tagging
+- Do the steps in "Making a release" above, except
+  - stay on ${BASE_TAG}-stable - don't checkout master or run
+    make fetch-gui-and-commit
+  - check the GitHub actions build for ${BASE_TAG}-stable not master
+  - run the integration tests on ${BASE_TAG}-stable as described below
+  - the release freeze rule is already enabled - leave it until the end
+- git co master
+- git pull --rebase
+- git checkout ${BASE_TAG}-stable docs/content/changelog.md
+- git diff --cached # check only the ${NEW_TAG} entry has changed
+- git commit -v -m "Changelog updates from Version ${NEW_TAG}" docs/content/changelog.md
+- git push
+- Disable release freeze rule
+
+Point releases are only made for the latest release. Once v1.(XX+1).0
+is out the ${BASE_TAG}-stable branch is abandoned.
+
+### Running the integration tests on the stable branch
+
+The integration tests at <https://integration.rclone.org> test master.
+To test the stable branch push it to the testing server and run the
+tests there in `screen` as they take a while.
+
+```console
+git push -f rclone@rclone-testing:private-rclone.git ${BASE_TAG}-stable
+ssh rclone@rclone-testing
+screen
+cd ~/integration-test && go run integration-test.go -repo /home/rclone/private-rclone.git -branch v1.XX-stable
+```
 
 ## Sponsor logos
 

VERSION

@@ -1 +1 @@
-v1.76.0
+v1.77.0

backend/all/all.go

@@ -16,6 +16,7 @@ import (
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
 	_ "github.com/rclone/rclone/backend/doi"
+	_ "github.com/rclone/rclone/backend/dosya"
 	_ "github.com/rclone/rclone/backend/drime"
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/dropbox"
@@ -38,6 +39,7 @@ import (
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/internxt"
 	_ "github.com/rclone/rclone/backend/jottacloud"
+	_ "github.com/rclone/rclone/backend/kdrive"
 	_ "github.com/rclone/rclone/backend/koofr"
 	_ "github.com/rclone/rclone/backend/linkbox"
 	_ "github.com/rclone/rclone/backend/local"

backend/archive/archive.go

@@ -194,8 +194,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 		// Path to the archive
 		archiveRemote := remote[:len(foundArchive.remote)]
-		// Remote is archive leaf name
-		foundArchive.remote = path.Base(archiveRemote)
+		// Remote is archive leaf name. This must understand remote
+		// names as the archive may be in the root, e.g. "remote:file.zip"
+		_, foundArchive.remote, err = fspath.Split(archiveRemote)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse archive path %q: %w", archiveRemote, err)
+		}
 		foundArchive.prefix = ""
 		// Point remote to archive file
 		remote = archiveRemote

backend/archive/archive_internal_test.go

@@ -12,16 +12,19 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
 
+	_ "github.com/rclone/rclone/backend/alias"
 	"github.com/rclone/rclone/backend/archive/archiver"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
@@ -175,6 +178,18 @@ func testArchive(t *testing.T, archiveName string, archiveFn func(t *testing.T,
 	checkTree(ctx, "SubDir", t, ":archive:"+zipFile+"/"+subDir, filepath.Join(input, subDir), 0)
 
 	// Now check a single file
+	//
+	// The filter below applies to both sides, so check first that the
+	// archive exposes only that file without it
+	Fsingle, err := cache.Get(ctx, ":archive:"+zipFile+"/"+aFile)
+	require.NoError(t, err)
+	entries, err := Fsingle.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	_, err = Fsingle.List(ctx, subDir)
+	assert.Equal(t, fs.ErrorDirNotFound, err)
+	_, err = Fsingle.NewObject(ctx, subDir)
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
 	fiCtx, fi := filter.AddConfig(ctx)
 	require.NoError(t, fi.AddRule("+ "+aFile))
 	require.NoError(t, fi.AddRule("- *"))
@@ -309,6 +324,50 @@ func TestArchiveUncleanRoot(t *testing.T) {
 	}
 }
 
+// TestArchiveInRemoteRoot checks that an archive in the root of a
+// remote, with no "/" between the remote name and the archive name, can
+// be read, as can a path within it.
+func TestArchiveInRemoteRoot(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create("sub/a.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test.zip"), buf.Bytes(), 0600))
+	t.Chdir(dir)
+
+	for _, test := range []struct {
+		name   string
+		remote string
+	}{
+		{name: "plain", remote: ":local:test.zip"},
+		{name: "slash in remote name", remote: ":alias,remote='" + dir + "':test.zip"},
+	} {
+		for _, sub := range []struct {
+			path string
+			want string
+		}{
+			{path: "", want: "sub"},
+			{path: "/sub", want: "a.txt"},
+		} {
+			t.Run(test.name+sub.path, func(t *testing.T) {
+				f, err := cache.Get(ctx, ":archive:"+test.remote+sub.path)
+				require.NoError(t, err)
+				entries, err := f.List(ctx, "")
+				require.NoError(t, err)
+				require.Len(t, entries, 1)
+				assert.Equal(t, sub.want, entries[0].Remote())
+			})
+		}
+	}
+}
+
 // escapingObject is an object whose remote is not where it was asked for.
 type escapingObject struct {
 	fs.Object
@@ -403,3 +462,103 @@ func TestIsDirectChild(t *testing.T) {
 		assert.Equal(t, test.want, isDirectChild(test.dir, test.remote), "dir=%q remote=%q", test.dir, test.remote)
 	}
 }
+
+// A symlink in a squashfs image is skipped unless -l/--links is in use,
+// as it is on the other backends.  With the flag it is exposed under its
+// name with the link suffix and reads back as its target.
+//
+// Note that this uses mksquashfs as an external binary.
+func TestArchiveSquashfsSymlink(t *testing.T) {
+	fstest.Initialise()
+	skipIfNoExe(t, "mksquashfs")
+	ctx := context.Background()
+
+	dir := t.TempDir()
+	input := filepath.Join(dir, "input")
+	require.NoError(t, os.Mkdir(input, 0777))
+	require.NoError(t, os.WriteFile(filepath.Join(input, "file.txt"), []byte("hello"), 0600))
+	require.NoError(t, os.Symlink("file.txt", filepath.Join(input, "link.txt")))
+	// a regular file which happens to carry the link suffix must stay
+	// readable under the flag, not be taken for a symlink
+	require.NoError(t, os.WriteFile(filepath.Join(input, "note"+fs.LinkSuffix), []byte("plain"), 0600))
+	image := filepath.Join(dir, "test.sqfs")
+	run(t, "mksquashfs", input, image)
+
+	remotes := func(t *testing.T, f fs.Fs) []string {
+		entries, err := f.List(ctx, "")
+		require.NoError(t, err)
+		names := make([]string, 0, len(entries))
+		for _, entry := range entries {
+			names = append(names, entry.Remote())
+		}
+		sort.Strings(names)
+		return names
+	}
+
+	read := func(t *testing.T, f fs.Fs, remote string) string {
+		o, err := f.NewObject(ctx, remote)
+		require.NoError(t, err)
+		rc, err := o.Open(ctx)
+		require.NoError(t, err)
+		contents, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		require.NoError(t, rc.Close())
+		assert.Equal(t, o.Size(), int64(len(contents)))
+		return string(contents)
+	}
+
+	f, err := fs.NewFs(ctx, ":archive:"+image)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt", "note" + fs.LinkSuffix}, remotes(t, f))
+
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = true
+
+	f, err = fs.NewFs(ctx, ":archive:"+image)
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix, "note" + fs.LinkSuffix}, remotes(t, f))
+
+	assert.Equal(t, "file.txt", read(t, f, "link.txt"+fs.LinkSuffix))
+	assert.Equal(t, "plain", read(t, f, "note"+fs.LinkSuffix))
+}
+
+// A hostile archive must not be able to write outside the destination.
+// A symlink pointing above the root with a member inside it is the
+// classic way to try it, and the refusal comes from the local backend,
+// so pin it from end to end here rather than trusting it stays.
+func TestArchiveZipSymlinkEscape(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = true
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	hdr := &zip.FileHeader{Name: "a"}
+	hdr.SetMode(os.ModeSymlink | 0777)
+	w, err := zw.CreateHeader(hdr)
+	require.NoError(t, err)
+	_, err = w.Write([]byte("../outside"))
+	require.NoError(t, err)
+	w, err = zw.Create("a/pwned.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("pwned"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+
+	dir := t.TempDir()
+	zipPath := filepath.Join(dir, "evil.zip")
+	require.NoError(t, os.WriteFile(zipPath, buf.Bytes(), 0600))
+	dst := filepath.Join(dir, "dst")
+	require.NoError(t, os.Mkdir(dst, 0777))
+
+	fsrc, err := fs.NewFs(ctx, ":archive:"+zipPath)
+	require.NoError(t, err)
+	fdst, err := fs.NewFs(ctx, dst)
+	require.NoError(t, err)
+
+	assert.Error(t, sync.CopyDir(ctx, fdst, fsrc, false))
+
+	_, err = os.Stat(filepath.Join(dir, "outside", "pwned.txt"))
+	assert.True(t, os.IsNotExist(err), "the archive wrote outside the destination")
+}

backend/archive/squashfs/squashfs.go

@@ -42,6 +42,8 @@ type Fs struct {
 	prefix      string   // position for objects
 	prefixSlash string   // position for objects with a slash on
 	root        string   // position to read from within the archive
+	file        string   // remote of the single file root points at, if any
+	links       bool     // read symlinks as .rclonelink files
 }
 
 // recoverParsePanic converts a panic raised while parsing a squashfs image
@@ -98,6 +100,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
 		c:           c,
 		remote:      remote,
 		root:        strings.Trim(root, "/"),
+		links:       fs.GetConfig(ctx).Links,
 		prefix:      prefix,
 		prefixSlash: prefix + "/",
 	}
@@ -115,10 +118,13 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (_ f
 			_, err := f.newObjectNative(native)
 			if err == nil {
 				// If it pointed to a file, find the directory above
+				// and remember the file so that only it is listed
 				f.root = path.Dir(f.root)
 				if f.root == "." || f.root == "/" {
 					f.root = ""
 				}
+				f.file = f.prefixSlash + path.Base(native)
+				singleObject = true
 			}
 		}
 	}
@@ -150,7 +156,9 @@ func (f *Fs) Name() string {
 
 // Root of the remote (as passed into NewFs)
 func (f *Fs) Root() string {
-	return f.root
+	// Include the file so the fs cache can tell this Fs apart from
+	// the one for the directory containing it
+	return path.Join(f.root, f.file)
 }
 
 // Features returns the optional features of this Fs
@@ -214,6 +222,30 @@ func (f *Fs) objectFromFileInfo(nativeDir string, item os.FileInfo) *Object {
 	}
 }
 
+// Convert a symlink entry into an Object holding its target
+func (f *Fs) objectFromLink(nativeDir string, item os.FileInfo, target string) *Object {
+	return &Object{
+		fs:      f,
+		remote:  f.fromNative(nativeDir, item.Name()) + fs.LinkSuffix,
+		size:    int64(len(target)),
+		modTime: item.ModTime(),
+		isLink:  true,
+		link:    target,
+	}
+}
+
+// readlink returns the target of the symlink the info describes.
+//
+// A symlink may carry an empty target, so the caller decides what is a
+// symlink from the mode rather than from the target being set.
+func readlink(info os.FileInfo) (string, error) {
+	st, ok := info.Sys().(*squashfs.StatT)
+	if !ok {
+		return "", fmt.Errorf("squashfs entry %q has no stat information", info.Name())
+	}
+	return st.LinkTarget, nil
+}
+
 // List the objects and directories in dir into entries.  The
 // entries can be returned in any order but should be for a
 // complete directory.
@@ -227,6 +259,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	defer log.Trace(f, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
 	defer recoverParsePanic(&err)
 
+	if f.file != "" && dir != "" {
+		return nil, fs.ErrorDirNotFound
+	}
+
 	nativeDir, err := f.toNative(dir)
 	if err != nil {
 		return nil, err
@@ -239,6 +275,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 	entries = make(fs.DirEntries, 0, len(items))
 	skipped := 0
+	skippedLinks := 0
 	for _, item := range items {
 		// fs.Debugf(item.Name(), "entry = %#v", item)
 		// Skip entry names that aren't safe
@@ -261,10 +298,27 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
 			}
 			entry = f.objectFromFileInfo(nativeDir, info)
+		} else if item.Type()&os.ModeSymlink != 0 {
+			if !f.links {
+				skippedLinks++
+				continue
+			}
+			info, err := item.Info()
+			if err != nil {
+				return nil, fmt.Errorf("error reading item %q: %w", item.Name(), err)
+			}
+			target, err := readlink(info)
+			if err != nil {
+				return nil, err
+			}
+			entry = f.objectFromLink(nativeDir, info, target)
 		} else {
 			fs.Debugf(item.Name(), "FIXME Not regular file - skipping")
 			continue
 		}
+		if f.file != "" && entry.Remote() != f.file {
+			continue
+		}
 		entries = append(entries, entry)
 	}
 
@@ -273,6 +327,9 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if skipped > 0 {
 		fs.Logf(f, "Skipped %d squashfs entries in %q whose names escape the archive", skipped, dir)
 	}
+	if skippedLinks > 0 {
+		fs.Logf(f, "Skipped %d symlinks in %q - use -l/--links to read them", skippedLinks, dir)
+	}
 
 	// fs.Debugf(f, "dir=%q, entries=%v", dir, entries)
 	return entries, nil
@@ -294,7 +351,17 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 	}
 
 	for _, fi := range fis {
-		if fi.Name() == leaf {
+		// Name the entry as List does, so that the two agree on
+		// which names exist
+		isLink := fi.Type()&os.ModeSymlink != 0
+		if isLink && !f.links {
+			continue
+		}
+		name := fi.Name()
+		if isLink {
+			name += fs.LinkSuffix
+		}
+		if name == leaf {
 			if fi.IsDir() {
 				return nil, fs.ErrorNotAFile
 			}
@@ -302,6 +369,14 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 			if err != nil {
 				return nil, fmt.Errorf("error reading item %q: %w", fi.Name(), err)
 			}
+			if isLink {
+				target, err := readlink(info)
+				if err != nil {
+					return nil, err
+				}
+				o = f.objectFromLink(dir, info, target)
+				break
+			}
 			o = f.objectFromFileInfo(dir, info)
 			break
 		}
@@ -316,6 +391,9 @@ func (f *Fs) newObjectNative(nativePath string) (o fs.Object, err error) {
 func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err error) {
 	defer log.Trace(f, "remote=%q", remote)("obj=%v, err=%v", &o, &err)
 
+	if f.file != "" && remote != f.file {
+		return nil, fs.ErrorObjectNotFound
+	}
 	nativePath, err := f.toNative(remote)
 	if err != nil {
 		return nil, err
@@ -377,6 +455,8 @@ type Object struct {
 	remote  string
 	size    int64
 	modTime time.Time
+	isLink  bool   // whether this is a symlink
+	link    string // target of the symlink
 }
 
 // Fs returns read only access to the Fs that this object is part of
@@ -448,6 +528,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 		}
 	}
 
+	if o.isLink {
+		// Clamp offset into range to avoid panic
+		if offset < 0 {
+			offset = 0
+		}
+		if offset > int64(len(o.link)) {
+			offset = int64(len(o.link))
+		}
+		return readers.NewLimitedReadCloser(io.NopCloser(strings.NewReader(o.link[offset:])), limit), nil
+	}
+
 	remote, err := o.fs.toNative(o.remote)
 	if err != nil {
 		return nil, err

backend/archive/zip/zip.go

@@ -43,6 +43,7 @@ type Fs struct {
 	prefix      string          // position for objects
 	prefixSlash string          // position for objects with a slash on
 	root        string          // position to read from within the archive
+	links       bool            // read symlinks as .rclonelink files
 	dt          dirtree.DirTree // read from zipfile
 }
 
@@ -67,6 +68,7 @@ func New(ctx context.Context, wrappedFs fs.Fs, remote, prefix, root string) (fs.
 		node:        node,
 		remote:      remote,
 		root:        root,
+		links:       fs.GetConfig(ctx).Links,
 		prefix:      prefix,
 		prefixSlash: prefix + "/",
 	}
@@ -138,6 +140,7 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 	}
 	dt := dirtree.New()
 	skipped := 0
+	skippedLinks := 0
 	for _, file := range zr.File {
 		// Skip entries whose name escapes the archive's own namespace
 		remote, err := sanitize.Path(file.Name)
@@ -152,6 +155,15 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 			skipped++
 			continue
 		}
+		if !isDir && file.Mode()&os.ModeSymlink != 0 {
+			if !f.links {
+				skippedLinks++
+				continue
+			}
+			// The member holds the link target, so it reads back
+			// like a .rclonelink file
+			remote += fs.LinkSuffix
+		}
 		remote = path.Join(f.prefix, remote)
 		if f.root != "" {
 			// Ignore all files outside the root, requiring a path
@@ -191,6 +203,9 @@ func (f *Fs) readZip() (singleObject bool, err error) {
 	if skipped > 0 {
 		fs.Logf(f, "Skipped %d zip entries which escape the archive", skipped)
 	}
+	if skippedLinks > 0 {
+		fs.Logf(f, "Skipped %d symlinks - use -l/--links to read them", skippedLinks)
+	}
 	dt.CheckParents("")
 	dt.Sort()
 	f.dt = dt

backend/archive/zip/zip_internal_test.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"sort"
@@ -36,6 +37,26 @@ func writeZip(t *testing.T, dir, name string, names ...string) string {
 	return name
 }
 
+// writeZipSymlink builds a zip holding file and a symlink entry link
+// pointing at target, and returns its leaf name.
+func writeZipSymlink(t *testing.T, dir, name, file, link, target string) string {
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create(file)
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data for " + file))
+	require.NoError(t, err)
+	hdr := &zip.FileHeader{Name: link}
+	hdr.SetMode(os.ModeSymlink | 0777)
+	w, err = zw.CreateHeader(hdr)
+	require.NoError(t, err)
+	_, err = w.Write([]byte(target))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	require.NoError(t, os.WriteFile(filepath.Join(dir, name), buf.Bytes(), 0600))
+	return name
+}
+
 // allRemotes returns every Object remote in the mounted archive's dirtree.
 func allRemotes(t *testing.T, f fs.Fs) []string {
 	zf, ok := f.(*Fs)
@@ -170,3 +191,37 @@ func TestListCacheNotAliased(t *testing.T) {
 	}
 	assert.Equal(t, []string{"a.txt", "b.txt", "c.txt"}, remotes)
 }
+
+// A symlink stored in the archive is skipped unless -l/--links is in
+// use, as it is on the other backends.  With the flag it is exposed
+// under its name with the link suffix and reads back as its target.
+func TestReadZipSymlink(t *testing.T) {
+	ctx := context.Background()
+	dir := t.TempDir()
+	name := writeZipSymlink(t, dir, "links.zip", "file.txt", "link.txt", "file.txt")
+
+	localFs, err := cache.Get(ctx, dir)
+	require.NoError(t, err)
+
+	f, err := New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt"}, allRemotes(t, f))
+
+	ctx, ci := fs.AddConfig(ctx)
+	ci.Links = true
+
+	f, err = New(ctx, localFs, name, "", "")
+	require.NoError(t, err)
+	assert.Equal(t, []string{"file.txt", "link.txt" + fs.LinkSuffix}, allRemotes(t, f))
+
+	o, err := f.NewObject(ctx, "link.txt"+fs.LinkSuffix)
+	require.NoError(t, err)
+	assert.Equal(t, int64(len("file.txt")), o.Size())
+
+	rc, err := o.Open(ctx)
+	require.NoError(t, err)
+	target, err := io.ReadAll(rc)
+	require.NoError(t, err)
+	require.NoError(t, rc.Close())
+	assert.Equal(t, "file.txt", string(target))
+}

backend/azureblob/arrow.go

@@ -18,24 +18,20 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/fs"
 	"golang.org/x/sync/errgroup"
 )
 
+// arrowContentType is the Content-Type of an Apache Arrow IPC stream listing
+// response, used to tell whether the service honoured an Arrow request.
+const arrowContentType = "application/vnd.apache.arrow.stream"
+
 // listArrowParallel lists (containerName, directory) by splitting the blob name
 // keyspace into shards and listing them concurrently. opts must already request
 // the Arrow format; it is cloned per shard with StartFrom/EndBefore set. It
 // returns the number of raw items seen. directory/prefix are already normalised
 // by the caller.
-func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
-	// Parallel listing needs the arrowlist client (server-side endBefore is
-	// only honoured on the Arrow path) - without it fall back to sequential.
-	if _, err := f.arrowCntSVC(containerName); err != nil {
-		fs.Debugf(f, "Not using parallel Arrow listing: %v", err)
-		return f.listBlobsPager(ctx, containerName, directory, prefix, addContainer, opts, delimiter, fn)
-	}
-
+func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (int, error) {
 	// Split the keyspace by a single-case character ladder (see
 	// arrowLadderBoundaries for why single-case matters).
 	points := arrowLadderBoundaries(directory, arrowShardTarget(f.opt.ListParallelism))
@@ -115,16 +111,13 @@ func (f *Fs) listArrowParallel(ctx context.Context, containerName, directory, pr
 // isEndBeforeUnsupported reports whether err means the service can't honour
 // the endBefore listing bound, so sharded parallel listing must not be used.
 //
-// This happens two ways: the arrowlist pager's ErrEndBeforeXMLFallback (the
-// server answered XML with endBefore set), or the service rejecting the
-// endBefore parameter with 400 OperationNotSupportedWithFeatureMissing "The
-// requested operation is not allowed as EndBefore parameter support is
-// missing on this account" - endBefore is only supported on the Arrow
-// listing path of accounts with Blob Listing with Apache Arrow enabled.
+// The service rejects the endBefore parameter with 400
+// OperationNotSupportedWithFeatureMissing "The requested operation is not
+// allowed as EndBefore parameter support is missing on this account" -
+// endBefore is only supported on the Arrow listing path of accounts with
+// Blob Listing with Apache Arrow enabled. (The SDK pager would silently
+// accept an XML page with endBefore set, but the service never sends one.)
 func isEndBeforeUnsupported(err error) bool {
-	if errors.Is(err, arrowlist.ErrEndBeforeXMLFallback) {
-		return true
-	}
 	var respErr *azcore.ResponseError
 	return errors.As(err, &respErr) && respErr.ErrorCode == "OperationNotSupportedWithFeatureMissing"
 }

backend/azureblob/arrow_internal_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -212,9 +211,8 @@ func TestArrowLadderBoundaries(t *testing.T) {
 }
 
 func TestIsEndBeforeUnsupported(t *testing.T) {
-	assert.True(t, isEndBeforeUnsupported(arrowlist.ErrEndBeforeXMLFallback))
-	assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", arrowlist.ErrEndBeforeXMLFallback)))
 	assert.True(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"}))
+	assert.True(t, isEndBeforeUnsupported(fmt.Errorf("wrapped: %w", &azcore.ResponseError{ErrorCode: "OperationNotSupportedWithFeatureMissing"})))
 	assert.False(t, isEndBeforeUnsupported(&azcore.ResponseError{ErrorCode: "ContainerNotFound"}))
 	assert.False(t, isEndBeforeUnsupported(errors.New("some other error")))
 	assert.False(t, isEndBeforeUnsupported(nil))

backend/azureblob/arrowlist/arrow.go

@@ -1,421 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost), retyped onto the SDK's
-// public type aliases so it compiles against the released azblob module.
-
-package arrowlist
-
-import (
-	"encoding/base64"
-	"fmt"
-	"io"
-	"net/http"
-	"strconv"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease"
-	"github.com/apache/arrow-go/v18/arrow"
-	"github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/apache/arrow-go/v18/arrow/ipc"
-	"github.com/apache/arrow-go/v18/arrow/memory"
-)
-
-const (
-	// ArrowContentType is the Content-Type for Apache Arrow IPC stream responses.
-	ArrowContentType = "application/vnd.apache.arrow.stream"
-
-	// ArrowAcceptHeader is the Accept header value to request Arrow format.
-	ArrowAcceptHeader = ArrowContentType
-
-	// resourceTypeBlobPrefix is the ResourceType value that identifies a virtual directory prefix
-	// in hierarchy listing responses. In XML, prefixes are separate <BlobPrefix> elements;
-	// in Arrow, all rows share the same schema and are distinguished by this field value.
-	resourceTypeBlobPrefix = "blobprefix"
-)
-
-// HandleFlatListResponse parses an Arrow IPC stream response into a ContainerClientListBlobFlatSegmentResponse.
-func HandleFlatListResponse(resp *http.Response) (container.ListBlobsFlatResponse, error) {
-	result := container.ListBlobsFlatResponse{}
-
-	// Extract response headers
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
-	// Parse Arrow IPC stream
-	items, nextMarker, err := parseArrowStream(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
-	}
-
-	result.ListBlobsFlatSegmentResponse = container.ListBlobsFlatSegmentResponse{
-		Segment: &container.BlobFlatListSegment{
-			BlobItems: items,
-		},
-		NextMarker: nextMarker,
-	}
-
-	return result, nil
-}
-
-// HandleHierarchyListResponse parses an Arrow IPC stream response into a ContainerClientListBlobHierarchySegmentResponse.
-func HandleHierarchyListResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
-	result := container.ListBlobsHierarchyResponse{}
-
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-
-	items, nextMarker, err := parseArrowStream(resp.Body)
-	if err != nil {
-		return result, fmt.Errorf("failed to parse Arrow IPC response: %w", err)
-	}
-
-	// Separate blob items from blob prefixes based on ResourceType field.
-	var blobItems []*container.BlobItem
-	var blobPrefixes []*container.BlobPrefix
-	for _, item := range items {
-		if item.Properties != nil && item.Properties.ResourceType != nil && *item.Properties.ResourceType == resourceTypeBlobPrefix {
-			blobPrefixes = append(blobPrefixes, &container.BlobPrefix{
-				Name:       item.Name,
-				Properties: item.Properties,
-			})
-		} else {
-			blobItems = append(blobItems, item)
-		}
-	}
-
-	result.ListBlobsHierarchySegmentResponse = container.ListBlobsHierarchySegmentResponse{
-		Segment: &container.BlobHierarchyListSegment{
-			BlobItems:    blobItems,
-			BlobPrefixes: blobPrefixes,
-		},
-		NextMarker: nextMarker,
-	}
-
-	return result, nil
-}
-
-func extractResponseHeaders(resp *http.Response, clientRequestID, contentType **string, date **time.Time, requestID, version **string) {
-	if val := resp.Header.Get("x-ms-client-request-id"); val != "" {
-		*clientRequestID = &val
-	}
-	if val := resp.Header.Get("Content-Type"); val != "" {
-		*contentType = &val
-	}
-	if val := resp.Header.Get("Date"); val != "" {
-		if t, err := time.Parse(time.RFC1123, val); err == nil {
-			*date = &t
-		}
-	}
-	if val := resp.Header.Get("x-ms-request-id"); val != "" {
-		*requestID = &val
-	}
-	if val := resp.Header.Get("x-ms-version"); val != "" {
-		*version = &val
-	}
-}
-
-// parseArrowStream reads an Arrow IPC stream and converts it to BlobItems.
-func parseArrowStream(body io.Reader) ([]*container.BlobItem, *string, error) {
-	reader, err := ipc.NewReader(body, ipc.WithAllocator(memory.DefaultAllocator))
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to create Arrow IPC reader: %w", err)
-	}
-	defer reader.Release()
-
-	// Extract NextMarker from schema metadata
-	var nextMarker *string
-	var numRecords int
-	if md := reader.Schema().Metadata(); md.Len() > 0 {
-		if idx := md.FindKey("NextMarker"); idx >= 0 {
-			val := md.Values()[idx]
-			if val != "" {
-				nextMarker = &val
-			}
-		}
-		if idx := md.FindKey("NumberOfRecords"); idx >= 0 {
-			if n, err := strconv.Atoi(md.Values()[idx]); err == nil {
-				numRecords = n
-			}
-		}
-	}
-
-	// Build column index for the schema
-	schema := reader.Schema()
-	colIndex := buildColumnIndex(schema)
-
-	// Pre-allocate if we know the count
-	items := make([]*container.BlobItem, 0, numRecords)
-
-	// Iterate over record batches
-	for reader.Next() {
-		rec := reader.RecordBatch()
-		rows := int(rec.NumRows())
-
-		for row := range rows {
-			item := &container.BlobItem{
-				Properties: &container.BlobProperties{},
-			}
-			populateBlobItem(item, rec, row, colIndex)
-			items = append(items, item)
-		}
-	}
-
-	if err := reader.Err(); err != nil {
-		return nil, nil, fmt.Errorf("error reading Arrow record batches: %w", err)
-	}
-
-	return items, nextMarker, nil
-}
-
-// columnIndex maps column names to their indices in the record batch for O(1) lookup.
-type columnIndex map[string]int
-
-func buildColumnIndex(schema *arrow.Schema) columnIndex {
-	idx := make(columnIndex, len(schema.Fields()))
-	for i, f := range schema.Fields() {
-		idx[f.Name] = i
-	}
-	return idx
-}
-
-// populateBlobItem fills a BlobItem from a single row across all columns.
-func populateBlobItem(item *container.BlobItem, rec arrow.RecordBatch, row int, colIdx columnIndex) {
-	// Top-level BlobItem fields
-	item.Name = getStringField(rec, row, colIdx, "Name")
-	item.Deleted = getBoolField(rec, row, colIdx, "Deleted")
-	item.Snapshot = getStringField(rec, row, colIdx, "Snapshot")
-	item.VersionID = getStringField(rec, row, colIdx, "VersionId")
-	item.IsCurrentVersion = getBoolField(rec, row, colIdx, "IsCurrentVersion")
-	item.HasVersionsOnly = getBoolField(rec, row, colIdx, "HasVersionsOnly")
-
-	// Metadata maps
-	item.Metadata = getMapField(rec, row, colIdx, "Metadata")
-	item.OrMetadata = getMapField(rec, row, colIdx, "OrMetadata")
-
-	// Tags
-	if tags := getMapField(rec, row, colIdx, "Tags"); tags != nil {
-		var blobTags []*container.BlobTag
-		for k, v := range tags {
-			key := k
-			blobTags = append(blobTags, &container.BlobTag{Key: &key, Value: v})
-		}
-		item.BlobTags = &container.BlobTags{BlobTagSet: blobTags}
-	}
-
-	// BlobProperties fields
-	p := item.Properties
-	p.CreationTime = getTimestampField(rec, row, colIdx, "Creation-Time")
-	p.LastModified = getTimestampField(rec, row, colIdx, "Last-Modified")
-	p.ETag = getETagField(rec, row, colIdx, "Etag")
-	p.ContentLength = getInt64Field(rec, row, colIdx, "Content-Length")
-	p.ContentType = getStringField(rec, row, colIdx, "Content-Type")
-	p.ContentEncoding = getStringField(rec, row, colIdx, "Content-Encoding")
-	p.ContentLanguage = getStringField(rec, row, colIdx, "Content-Language")
-	p.ContentDisposition = getStringField(rec, row, colIdx, "Content-Disposition")
-	p.CacheControl = getStringField(rec, row, colIdx, "Cache-Control")
-	p.ContentMD5 = getBase64BytesField(rec, row, colIdx, "Content-MD5")
-	p.BlobType = getEnumField[container.BlobType](rec, row, colIdx, "BlobType")
-	p.AccessTier = getEnumField[container.AccessTier](rec, row, colIdx, "AccessTier")
-	p.AccessTierInferred = getBoolField(rec, row, colIdx, "AccessTierInferred")
-	p.AccessTierChangeTime = getTimestampField(rec, row, colIdx, "AccessTierChangeTime")
-	p.LeaseState = getEnumField[lease.StateType](rec, row, colIdx, "LeaseState")
-	p.LeaseStatus = getEnumField[lease.StatusType](rec, row, colIdx, "LeaseStatus")
-	p.LeaseDuration = getEnumField[lease.DurationType](rec, row, colIdx, "LeaseDuration")
-	p.ServerEncrypted = getBoolField(rec, row, colIdx, "ServerEncrypted")
-	p.CustomerProvidedKeySHA256 = getStringField(rec, row, colIdx, "CustomerProvidedKeySha256")
-	p.EncryptionScope = getStringField(rec, row, colIdx, "EncryptionScope")
-	p.IncrementalCopy = getBoolField(rec, row, colIdx, "IncrementalCopy")
-	p.IsSealed = getBoolField(rec, row, colIdx, "Sealed")
-	p.ArchiveStatus = getEnumField[container.ArchiveStatus](rec, row, colIdx, "ArchiveStatus")
-	p.RehydratePriority = getEnumField[blob.RehydratePriority](rec, row, colIdx, "RehydratePriority")
-	p.CopyID = getStringField(rec, row, colIdx, "CopyId")
-	p.CopyStatus = getEnumField[blob.CopyStatusType](rec, row, colIdx, "CopyStatus")
-	p.CopySource = getStringField(rec, row, colIdx, "CopySource")
-	p.CopyProgress = getStringField(rec, row, colIdx, "CopyProgress")
-	p.CopyCompletionTime = getTimestampField(rec, row, colIdx, "CopyCompletionTime")
-	p.CopyStatusDescription = getStringField(rec, row, colIdx, "CopyStatusDescription")
-	p.DestinationSnapshot = getStringField(rec, row, colIdx, "CopyDestinationSnapshot")
-	p.ImmutabilityPolicyExpiresOn = getTimestampField(rec, row, colIdx, "ImmutabilityPolicyUntilDate")
-	p.ImmutabilityPolicyMode = getEnumField[container.ImmutabilityPolicyMode](rec, row, colIdx, "ImmutabilityPolicyMode")
-	p.LegalHold = getBoolField(rec, row, colIdx, "LegalHold")
-	p.DeletedTime = getTimestampField(rec, row, colIdx, "DeletedTime")
-	p.RemainingRetentionDays = getInt32Field(rec, row, colIdx, "RemainingRetentionDays")
-	p.LastAccessedOn = getTimestampField(rec, row, colIdx, "LastAccessTime")
-	p.TagCount = getInt32Field(rec, row, colIdx, "TagCount")
-	p.BlobSequenceNumber = getInt64Field(rec, row, colIdx, "x-ms-blob-sequence-number")
-	p.ResourceType = getStringField(rec, row, colIdx, "ResourceType")
-
-	// Content-CRC64 is a string field in Arrow but not directly mapped to BlobProperties.
-	// The field exists in the Arrow schema but has no corresponding field in the Go SDK BlobProperties.
-	// It is silently ignored for forward compatibility.
-}
-
-// Field accessor helpers that safely handle missing columns and null values.
-
-func getStringField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *string {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.String); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getBoolField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *bool {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Boolean); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getTimestampField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *time.Time {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Timestamp); ok {
-		val := arr.Value(row)
-		// Arrow timestamps are stored as int64 with a unit; convert to time.Time
-		dt := val.ToTime(arr.DataType().(*arrow.TimestampType).Unit)
-		return &dt
-	}
-	return nil
-}
-
-func getInt64Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int64 {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Uint64); ok {
-		val := int64(arr.Value(row))
-		return &val
-	}
-	if arr, ok := col.(*array.Int64); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getInt32Field(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *int32 {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	if arr, ok := col.(*array.Uint64); ok {
-		val := int32(arr.Value(row))
-		return &val
-	}
-	if arr, ok := col.(*array.Int32); ok {
-		val := arr.Value(row)
-		return &val
-	}
-	return nil
-}
-
-func getETagField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *azcore.ETag {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	etag := azcore.ETag(*s)
-	return &etag
-}
-
-func getBase64BytesField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) []byte {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	decoded, err := base64.StdEncoding.DecodeString(*s)
-	if err != nil {
-		return nil
-	}
-	return decoded
-}
-
-// getEnumField returns a pointer to a string-typed enum value.
-type stringEnum interface {
-	~string
-}
-
-func getEnumField[T stringEnum](rec arrow.RecordBatch, row int, colIdx columnIndex, name string) *T {
-	s := getStringField(rec, row, colIdx, name)
-	if s == nil {
-		return nil
-	}
-	val := T(*s)
-	return &val
-}
-
-func getMapField(rec arrow.RecordBatch, row int, colIdx columnIndex, name string) map[string]*string {
-	idx, ok := colIdx[name]
-	if !ok {
-		return nil
-	}
-	col := rec.Column(idx)
-	if col.IsNull(row) {
-		return nil
-	}
-	mapArr, ok := col.(*array.Map)
-	if !ok {
-		return nil
-	}
-
-	// Get the offsets for this row's map entries
-	start := int(mapArr.Offsets()[row])
-	end := int(mapArr.Offsets()[row+1])
-	if start == end {
-		return nil
-	}
-
-	keys := mapArr.Keys().(*array.String)
-	values := mapArr.Items().(*array.String)
-
-	result := make(map[string]*string, end-start)
-	for i := start; i < end; i++ {
-		k := keys.Value(i)
-		if values.IsNull(i) {
-			result[k] = nil
-		} else {
-			v := values.Value(i)
-			result[k] = &v
-		}
-	}
-	return result
-}

backend/azureblob/arrowlist/arrow_test.go

@@ -1,374 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/arrow/arrow_test.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at commit
-// c6fa341ca22b (branch feature/storage/bifrost).
-
-package arrowlist
-
-import (
-	"bytes"
-	"io"
-	"net/http"
-	"testing"
-
-	"github.com/apache/arrow-go/v18/arrow"
-	arrowArray "github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/apache/arrow-go/v18/arrow/ipc"
-	"github.com/apache/arrow-go/v18/arrow/memory"
-	"github.com/stretchr/testify/require"
-)
-
-// coreSchema returns a basic Arrow schema for testing with common blob fields.
-func coreSchema(md *arrow.Metadata) *arrow.Schema {
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Creation-Time", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
-		{Name: "Last-Modified", Type: &arrow.TimestampType{Unit: arrow.Microsecond}, Nullable: true},
-		{Name: "BlobType", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Etag", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
-		{Name: "Content-Type", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "AccessTier", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "LeaseState", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "LeaseStatus", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	return arrow.NewSchema(fields, md)
-}
-
-// buildArrowStream constructs a synthetic Arrow IPC stream from a schema and a populate function.
-func buildArrowStream(t *testing.T, schema *arrow.Schema, populate func(builder *arrowArray.RecordBuilder)) []byte {
-	t.Helper()
-	alloc := memory.DefaultAllocator
-
-	var buf bytes.Buffer
-	w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
-	builder := arrowArray.NewRecordBuilder(alloc, schema)
-	defer builder.Release()
-
-	populate(builder)
-
-	rec := builder.NewRecordBatch()
-	defer rec.Release()
-
-	require.NoError(t, w.Write(rec))
-	require.NoError(t, w.Close())
-
-	return buf.Bytes()
-}
-
-func makeHTTPResponse(data []byte, headers map[string]string) *http.Response {
-	h := http.Header{}
-	h.Set("Content-Type", ArrowContentType)
-	for k, v := range headers {
-		h.Set(k, v)
-	}
-	return &http.Response{
-		StatusCode: http.StatusOK,
-		Header:     h,
-		Body:       io.NopCloser(bytes.NewReader(data)),
-	}
-}
-
-func TestHandleFlatListResponse_BasicParsing(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"marker123", "2"})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob1.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
-		b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000000000000))
-		b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
-		b.Field(4).(*arrowArray.StringBuilder).Append("0x1234")
-		b.Field(5).(*arrowArray.Uint64Builder).Append(1024)
-		b.Field(6).(*arrowArray.StringBuilder).Append("application/octet-stream")
-		b.Field(7).(*arrowArray.BooleanBuilder).Append(true)
-		b.Field(8).(*arrowArray.StringBuilder).Append("Hot")
-		b.Field(9).(*arrowArray.StringBuilder).Append("available")
-		b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob2.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
-		b.Field(2).(*arrowArray.TimestampBuilder).Append(arrow.Timestamp(1700000001000000))
-		b.Field(3).(*arrowArray.StringBuilder).Append("BlockBlob")
-		b.Field(4).(*arrowArray.StringBuilder).Append("0x5678")
-		b.Field(5).(*arrowArray.Uint64Builder).Append(2048)
-		b.Field(6).(*arrowArray.StringBuilder).Append("text/plain")
-		b.Field(7).(*arrowArray.BooleanBuilder).Append(false)
-		b.Field(8).(*arrowArray.StringBuilder).Append("Cool")
-		b.Field(9).(*arrowArray.StringBuilder).Append("available")
-		b.Field(10).(*arrowArray.StringBuilder).Append("unlocked")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	require.NotNil(t, result.NextMarker)
-	require.Equal(t, "marker123", *result.NextMarker)
-	require.Len(t, result.Segment.BlobItems, 2)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "blob1.txt", *item.Name)
-	require.Equal(t, "BlockBlob", string(*item.Properties.BlobType))
-	require.Equal(t, int64(1024), *item.Properties.ContentLength)
-	require.Equal(t, "Hot", string(*item.Properties.AccessTier))
-}
-
-func TestHandleFlatListResponse_EmptyRecordBatch(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "0"})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		// No rows added
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Nil(t, result.NextMarker)
-	require.Empty(t, result.Segment.BlobItems)
-}
-
-func TestHandleFlatListResponse_NullFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	schema := coreSchema(&md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.TimestampBuilder).AppendNull()
-		b.Field(2).(*arrowArray.TimestampBuilder).AppendNull()
-		b.Field(3).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(4).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(5).(*arrowArray.Uint64Builder).AppendNull()
-		b.Field(6).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(7).(*arrowArray.BooleanBuilder).AppendNull()
-		b.Field(8).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(9).(*arrowArray.StringBuilder).AppendNull()
-		b.Field(10).(*arrowArray.StringBuilder).AppendNull()
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "blob.txt", *item.Name)
-	require.Nil(t, item.Properties.CreationTime)
-	require.Nil(t, item.Properties.BlobType)
-	require.Nil(t, item.Properties.ETag)
-	require.Nil(t, item.Properties.ContentLength)
-	require.Nil(t, item.Properties.ContentType)
-	require.Nil(t, item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_UnknownColumnsIgnored(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "FutureField", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.StringBuilder).Append("some-future-value")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-	require.Equal(t, "blob.txt", *result.Segment.BlobItems[0].Name)
-}
-
-func TestHandleFlatListResponse_MultiRecordBatch(t *testing.T) {
-	alloc := memory.DefaultAllocator
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{"", "3"})
-	schema := coreSchema(&md)
-
-	var buf bytes.Buffer
-	w := ipc.NewWriter(&buf, ipc.WithSchema(schema), ipc.WithAllocator(alloc))
-
-	// First batch: 2 rows
-	builder := arrowArray.NewRecordBuilder(alloc, schema)
-	for range 2 {
-		builder.Field(0).(*arrowArray.StringBuilder).Append("batch1_blob")
-		for j := 1; j < 11; j++ {
-			builder.Field(j).AppendNull()
-		}
-	}
-	rec := builder.NewRecordBatch()
-	require.NoError(t, w.Write(rec))
-	rec.Release()
-	builder.Release()
-
-	// Second batch: 1 row
-	builder = arrowArray.NewRecordBuilder(alloc, schema)
-	builder.Field(0).(*arrowArray.StringBuilder).Append("batch2_blob")
-	for j := 1; j < 11; j++ {
-		builder.Field(j).AppendNull()
-	}
-	rec = builder.NewRecordBatch()
-	require.NoError(t, w.Write(rec))
-	rec.Release()
-	builder.Release()
-
-	require.NoError(t, w.Close())
-
-	resp := makeHTTPResponse(buf.Bytes(), nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 3)
-}
-
-func TestHandleHierarchyListResponse_SeparatesBlobsAndPrefixes(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "ResourceType", Type: arrow.BinaryTypes.String, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		// A regular blob
-		b.Field(0).(*arrowArray.StringBuilder).Append("folder/file.txt")
-		b.Field(1).(*arrowArray.StringBuilder).AppendNull()
-
-		// A blob prefix (virtual directory)
-		b.Field(0).(*arrowArray.StringBuilder).Append("folder/subfolder/")
-		b.Field(1).(*arrowArray.StringBuilder).Append("blobprefix")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleHierarchyListResponse(resp)
-	require.NoError(t, err)
-
-	require.Len(t, result.Segment.BlobItems, 1)
-	require.Equal(t, "folder/file.txt", *result.Segment.BlobItems[0].Name)
-
-	require.Len(t, result.Segment.BlobPrefixes, 1)
-	require.Equal(t, "folder/subfolder/", *result.Segment.BlobPrefixes[0].Name)
-}
-
-func TestHandleFlatListResponse_ResponseHeaders(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-	})
-
-	resp := makeHTTPResponse(data, map[string]string{
-		"x-ms-request-id":        "req-123",
-		"x-ms-version":           "2026-10-06",
-		"x-ms-client-request-id": "client-req-456",
-		"Date":                   "Mon, 01 Jan 2024 00:00:00 GMT",
-	})
-
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Equal(t, "req-123", *result.RequestID)
-	require.Equal(t, "2026-10-06", *result.Version)
-	require.Equal(t, "client-req-456", *result.ClientRequestID)
-	require.NotNil(t, result.Date)
-}
-
-func TestHandleFlatListResponse_MapFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Metadata", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
-		{Name: "Tags", Type: arrow.MapOf(arrow.BinaryTypes.String, arrow.BinaryTypes.String), Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-
-		metaBuilder := b.Field(1).(*arrowArray.MapBuilder)
-		metaBuilder.Append(true)
-		metaBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("key1")
-		metaBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("value1")
-
-		tagBuilder := b.Field(2).(*arrowArray.MapBuilder)
-		tagBuilder.Append(true)
-		tagBuilder.KeyBuilder().(*arrowArray.StringBuilder).Append("env")
-		tagBuilder.ItemBuilder().(*arrowArray.StringBuilder).Append("prod")
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-	require.Len(t, result.Segment.BlobItems, 1)
-
-	item := result.Segment.BlobItems[0]
-	require.NotNil(t, item.Metadata)
-	require.Equal(t, "value1", *item.Metadata["key1"])
-
-	require.NotNil(t, item.BlobTags)
-	require.Len(t, item.BlobTags.BlobTagSet, 1)
-}
-
-func TestHandleFlatListResponse_BooleanFields(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Deleted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-		{Name: "ServerEncrypted", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("blob.txt")
-		b.Field(1).(*arrowArray.BooleanBuilder).Append(true)
-		b.Field(2).(*arrowArray.BooleanBuilder).Append(false)
-		b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	item := result.Segment.BlobItems[0]
-	require.True(t, *item.Deleted)
-	require.False(t, *item.IsCurrentVersion)
-	require.True(t, *item.Properties.ServerEncrypted)
-}
-
-func TestHandleFlatListResponse_VersionAndSnapshot(t *testing.T) {
-	md := arrow.NewMetadata([]string{"NextMarker"}, []string{""})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "VersionId", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "Snapshot", Type: arrow.BinaryTypes.String, Nullable: true},
-		{Name: "IsCurrentVersion", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-
-	data := buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append("versioned-blob.txt")
-		b.Field(1).(*arrowArray.StringBuilder).Append("2024-01-01T00:00:00.0000000Z")
-		b.Field(2).(*arrowArray.StringBuilder).Append("snap-123")
-		b.Field(3).(*arrowArray.BooleanBuilder).Append(true)
-	})
-
-	resp := makeHTTPResponse(data, nil)
-	result, err := HandleFlatListResponse(resp)
-	require.NoError(t, err)
-
-	item := result.Segment.BlobItems[0]
-	require.Equal(t, "2024-01-01T00:00:00.0000000Z", *item.VersionID)
-	require.Equal(t, "snap-123", *item.Snapshot)
-	require.True(t, *item.IsCurrentVersion)
-}

backend/azureblob/arrowlist/arrowlist.go

@@ -1,259 +0,0 @@
-//go:build !plan9 && !js
-
-// Package arrowlist implements the experimental "Blob Listing with Apache
-// Arrow" feature on top of the released Azure azblob SDK.
-//
-// The Azure SDK for Go has support for Arrow listing on the unreleased
-// feature/storage/bifrost branch (commit c6fa341ca22b) where the listing
-// pagers decode Arrow IPC stream responses into the ordinary ListBlobs
-// response types. Until that ships in a tagged azblob release this package
-// fills the gap: it exposes the same options and pager interface as the
-// experimental SDK, built only on the public API of the released SDK plus
-// copies of two small internal auth policies (see sharedkey.go and
-// challenge_policy.go).
-//
-// This package is temporary. When Arrow listing ships in a released azblob
-// the whole package should be deleted and its callers pointed back at
-// container.Client.NewListBlobsHierarchyPager - the option and response
-// types here are deliberately source compatible with the SDK's to make that
-// a mechanical change.
-package arrowlist
-
-import (
-	"context"
-	"errors"
-	"net/http"
-	"strconv"
-	"strings"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-)
-
-const (
-	// moduleName and moduleVersion identify this package in the azcore
-	// telemetry policy's User-Agent fragment.
-	moduleName    = "github.com/rclone/rclone/backend/azureblob/arrowlist"
-	moduleVersion = "v0.0.1"
-
-	// serviceVersion is the x-ms-version sent with listing requests. Arrow
-	// responses and the startFrom/endBefore parameters need at least
-	// 2026-06-06.
-	serviceVersion = "2026-06-06"
-
-	// tokenScope is the OAuth scope for Azure Storage, as used by the SDK's
-	// own clients.
-	tokenScope = "https://storage.azure.com/.default"
-)
-
-// ErrEndBeforeXMLFallback is returned by the pager when the server answered
-// with XML instead of Arrow while EndBefore was set. The service only honours
-// endBefore on the Arrow listing path, so an XML page could silently contain
-// blobs outside the requested range and cannot be used.
-var ErrEndBeforeXMLFallback = errors.New("arrow listing fell back to XML with endBefore set")
-
-// ListBlobsHierarchyOptions provides the configuration for
-// Client.NewListBlobsHierarchyPager.
-//
-// It mirrors the experimental SDK's container.ListBlobsHierarchyOptions:
-// the released SDK's options are embedded (providing Include, Marker,
-// MaxResults, Prefix and StartFrom by promotion) with the two fields the
-// released SDK lacks added alongside.
-type ListBlobsHierarchyOptions struct {
-	container.ListBlobsHierarchyOptions
-	// EndBefore limits listing to blobs whose full container path is
-	// lexically before it (exclusive). Only honoured on Arrow responses.
-	EndBefore *string
-	// UseArrowFormat requests the response as an Apache Arrow IPC stream.
-	UseArrowFormat *bool
-}
-
-// ClientOptions contains the optional parameters for client creation.
-type ClientOptions struct {
-	azcore.ClientOptions
-}
-
-// Client issues Arrow listing requests to a single container.
-type Client struct {
-	endpoint string
-	azClient *azcore.Client
-}
-
-// newClient creates a Client for the container at containerURL sending
-// requests through a pipeline with the given auth policy (which may be nil
-// for anonymous or SAS access).
-func newClient(containerURL string, authPolicy policy.Policy, options *ClientOptions) (*Client, error) {
-	if options == nil {
-		options = &ClientOptions{}
-	}
-	plOpts := runtime.PipelineOptions{}
-	if authPolicy != nil {
-		plOpts.PerRetry = []policy.Policy{authPolicy}
-	}
-	azClient, err := azcore.NewClient(moduleName, moduleVersion, plOpts, &options.ClientOptions)
-	if err != nil {
-		return nil, err
-	}
-	return &Client{
-		endpoint: containerURL,
-		azClient: azClient,
-	}, nil
-}
-
-// NewClient creates a Client authenticating with a token credential.
-func NewClient(containerURL string, cred azcore.TokenCredential, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, NewStorageChallengePolicy(cred, tokenScope, false), options)
-}
-
-// NewClientWithNoCredential creates a Client with no authentication, for
-// anonymous access or a containerURL carrying a SAS token in its query.
-func NewClientWithNoCredential(containerURL string, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, nil, options)
-}
-
-// NewClientWithSharedKeyCredential creates a Client signing requests with the
-// account's shared key.
-func NewClientWithSharedKeyCredential(containerURL string, cred *SharedKeyCredential, options *ClientOptions) (*Client, error) {
-	return newClient(containerURL, NewSharedKeyCredPolicy(cred), options)
-}
-
-// NewListBlobsHierarchyPager returns a pager over the container's blobs,
-// requesting Arrow responses when o.UseArrowFormat is set and returning the
-// same response type as container.Client.NewListBlobsHierarchyPager.
-//
-// If the service does not support Arrow listing (the feature not enabled on
-// the account, or an HNS account) it answers with XML, which is decoded
-// transparently - unless EndBefore is set, in which case the fetcher returns
-// ErrEndBeforeXMLFallback (see there).
-func (c *Client) NewListBlobsHierarchyPager(delimiter string, o *ListBlobsHierarchyOptions) *runtime.Pager[container.ListBlobsHierarchyResponse] {
-	// Copy the options so advancing the Marker doesn't mutate the caller's struct.
-	opts := ListBlobsHierarchyOptions{}
-	if o != nil {
-		opts = *o
-	}
-	useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
-	return runtime.NewPager(runtime.PagingHandler[container.ListBlobsHierarchyResponse]{
-		More: func(page container.ListBlobsHierarchyResponse) bool {
-			return page.NextMarker != nil && len(*page.NextMarker) > 0
-		},
-		Fetcher: func(ctx context.Context, page *container.ListBlobsHierarchyResponse) (container.ListBlobsHierarchyResponse, error) {
-			if page != nil {
-				opts.Marker = page.NextMarker
-			}
-			req, err := c.listCreateRequest(ctx, delimiter, &opts, useArrow)
-			if err != nil {
-				return container.ListBlobsHierarchyResponse{}, err
-			}
-			resp, err := c.azClient.Pipeline().Do(req)
-			if err != nil {
-				return container.ListBlobsHierarchyResponse{}, err
-			}
-			if !runtime.HasStatusCode(resp, http.StatusOK) {
-				return container.ListBlobsHierarchyResponse{}, runtime.NewResponseError(resp)
-			}
-			if useArrow && resp.Header.Get("Content-Type") == ArrowContentType {
-				return HandleHierarchyListResponse(resp)
-			}
-			if opts.EndBefore != nil {
-				return container.ListBlobsHierarchyResponse{}, ErrEndBeforeXMLFallback
-			}
-			return handleXMLResponse(resp)
-		},
-	})
-}
-
-// listCreateRequest builds the ListBlobs request, mirroring the generated
-// request builders in the SDK.
-func (c *Client) listCreateRequest(ctx context.Context, delimiter string, opts *ListBlobsHierarchyOptions, useArrow bool) (*policy.Request, error) {
-	req, err := runtime.NewRequest(ctx, http.MethodGet, c.endpoint)
-	if err != nil {
-		return nil, err
-	}
-	// Reading the existing query first preserves any SAS token in the endpoint.
-	reqQP := req.Raw().URL.Query()
-	reqQP.Set("comp", "list")
-	reqQP.Set("restype", "container")
-	reqQP.Set("delimiter", delimiter)
-	if include := formatInclude(opts.Include); include != "" {
-		reqQP.Set("include", include)
-	}
-	if opts.Marker != nil {
-		reqQP.Set("marker", *opts.Marker)
-	}
-	if opts.MaxResults != nil {
-		reqQP.Set("maxresults", strconv.FormatInt(int64(*opts.MaxResults), 10))
-	}
-	if opts.Prefix != nil {
-		reqQP.Set("prefix", *opts.Prefix)
-	}
-	if opts.StartFrom != nil {
-		reqQP.Set("startFrom", *opts.StartFrom)
-	}
-	if useArrow && opts.EndBefore != nil {
-		reqQP.Set("endBefore", *opts.EndBefore)
-	}
-	req.Raw().URL.RawQuery = strings.ReplaceAll(reqQP.Encode(), "+", "%20")
-	if useArrow {
-		// Stream the Arrow body rather than buffering it in the pipeline.
-		runtime.SkipBodyDownload(req)
-		req.Raw().Header["Accept"] = []string{ArrowAcceptHeader}
-	} else {
-		req.Raw().Header["Accept"] = []string{"application/xml"}
-	}
-	req.Raw().Header["x-ms-version"] = []string{serviceVersion}
-	return req, nil
-}
-
-// handleXMLResponse decodes an XML ListBlobs response, mirroring the SDK's
-// generated ListBlobHierarchySegmentHandleResponse.
-func handleXMLResponse(resp *http.Response) (container.ListBlobsHierarchyResponse, error) {
-	result := container.ListBlobsHierarchyResponse{}
-	extractResponseHeaders(resp, &result.ClientRequestID, &result.ContentType, &result.Date, &result.RequestID, &result.Version)
-	if err := runtime.UnmarshalAsXML(resp, &result.ListBlobsHierarchySegmentResponse); err != nil {
-		return container.ListBlobsHierarchyResponse{}, err
-	}
-	return result, nil
-}
-
-// formatInclude renders the include datasets as the comma separated query
-// parameter value, in the same order as the SDK's unexported
-// ListBlobsInclude.format.
-func formatInclude(l container.ListBlobsInclude) string {
-	var include []string
-	if l.Copy {
-		include = append(include, "copy")
-	}
-	if l.Deleted {
-		include = append(include, "deleted")
-	}
-	if l.DeletedWithVersions {
-		include = append(include, "deletedwithversions")
-	}
-	if l.ImmutabilityPolicy {
-		include = append(include, "immutabilitypolicy")
-	}
-	if l.LegalHold {
-		include = append(include, "legalhold")
-	}
-	if l.Metadata {
-		include = append(include, "metadata")
-	}
-	if l.Snapshots {
-		include = append(include, "snapshots")
-	}
-	if l.Tags {
-		include = append(include, "tags")
-	}
-	if l.UncommittedBlobs {
-		include = append(include, "uncommittedblobs")
-	}
-	if l.Versions {
-		include = append(include, "versions")
-	}
-	if l.Permissions {
-		include = append(include, "permissions")
-	}
-	return strings.Join(include, ",")
-}

backend/azureblob/arrowlist/arrowlist_test.go

@@ -1,198 +0,0 @@
-//go:build !plan9 && !js
-
-package arrowlist
-
-import (
-	"context"
-	"encoding/base64"
-	"net/http"
-	"net/http/httptest"
-	"strings"
-	"testing"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
-	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
-	"github.com/apache/arrow-go/v18/arrow"
-	arrowArray "github.com/apache/arrow-go/v18/arrow/array"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// arrowPage builds a one row Arrow IPC listing page with the given blob name
-// and NextMarker.
-func arrowPage(t *testing.T, name, nextMarker string) []byte {
-	md := arrow.NewMetadata([]string{"NextMarker", "NumberOfRecords"}, []string{nextMarker, "1"})
-	fields := []arrow.Field{
-		{Name: "Name", Type: arrow.BinaryTypes.String, Nullable: false},
-		{Name: "Content-Length", Type: arrow.PrimitiveTypes.Uint64, Nullable: true},
-	}
-	schema := arrow.NewSchema(fields, &md)
-	return buildArrowStream(t, schema, func(b *arrowArray.RecordBuilder) {
-		b.Field(0).(*arrowArray.StringBuilder).Append(name)
-		b.Field(1).(*arrowArray.Uint64Builder).Append(42)
-	})
-}
-
-func TestPagerArrow(t *testing.T) {
-	var requests []*http.Request
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		requests = append(requests, r.Clone(context.Background()))
-		w.Header().Set("Content-Type", ArrowContentType)
-		if r.URL.Query().Get("marker") == "" {
-			_, _ = w.Write(arrowPage(t, "blob1.txt", "marker1"))
-		} else {
-			_, _ = w.Write(arrowPage(t, "blob2.txt", ""))
-		}
-	}))
-	defer srv.Close()
-
-	// SAS style query on the endpoint must be preserved in requests.
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer?sv=fakesas", nil)
-	require.NoError(t, err)
-
-	opts := &ListBlobsHierarchyOptions{
-		ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
-			Include:    container.ListBlobsInclude{Metadata: true, Tags: true},
-			Prefix:     new("dir/"),
-			MaxResults: new(int32(1000)),
-			StartFrom:  new("testcontainer/dir/a"),
-		},
-		EndBefore:      new("testcontainer/dir/n"),
-		UseArrowFormat: new(true),
-	}
-	pager := client.NewListBlobsHierarchyPager("/", opts)
-
-	var names []string
-	for pager.More() {
-		page, err := pager.NextPage(context.Background())
-		require.NoError(t, err)
-		assert.Equal(t, ArrowContentType, *page.ContentType)
-		for _, item := range page.Segment.BlobItems {
-			names = append(names, *item.Name)
-			assert.Equal(t, int64(42), *item.Properties.ContentLength)
-		}
-	}
-	assert.Equal(t, []string{"blob1.txt", "blob2.txt"}, names)
-
-	require.Len(t, requests, 2)
-	q := requests[0].URL.Query()
-	assert.Equal(t, "list", q.Get("comp"))
-	assert.Equal(t, "container", q.Get("restype"))
-	assert.Equal(t, "/", q.Get("delimiter"))
-	assert.Equal(t, "metadata,tags", q.Get("include"))
-	assert.Equal(t, "dir/", q.Get("prefix"))
-	assert.Equal(t, "1000", q.Get("maxresults"))
-	assert.Equal(t, "testcontainer/dir/a", q.Get("startFrom"))
-	assert.Equal(t, "testcontainer/dir/n", q.Get("endBefore"))
-	assert.Equal(t, "fakesas", q.Get("sv"))
-	assert.Empty(t, q.Get("marker"))
-	assert.Equal(t, ArrowAcceptHeader, requests[0].Header.Get("Accept"))
-	assert.Equal(t, serviceVersion, requests[0].Header.Get("x-ms-version"))
-	assert.Equal(t, "marker1", requests[1].URL.Query().Get("marker"))
-
-	// The caller's options must not have been mutated by paging.
-	assert.Nil(t, opts.Marker)
-}
-
-const xmlListResponse = `<?xml version="1.0" encoding="utf-8"?>
-<EnumerationResults ServiceEndpoint="https://example.com/" ContainerName="testcontainer">
-  <Blobs>
-    <Blob>
-      <Name>file.txt</Name>
-      <Properties>
-        <Last-Modified>Mon, 01 Jan 2024 00:00:00 GMT</Last-Modified>
-        <Etag>0x8D1</Etag>
-        <Content-Length>7</Content-Length>
-        <BlobType>BlockBlob</BlobType>
-      </Properties>
-    </Blob>
-    <BlobPrefix>
-      <Name>subdir/</Name>
-    </BlobPrefix>
-  </Blobs>
-  <NextMarker />
-</EnumerationResults>`
-
-func TestPagerXMLFallback(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/xml")
-		_, _ = w.Write([]byte(xmlListResponse))
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: new(true),
-	})
-	page, err := pager.NextPage(context.Background())
-	require.NoError(t, err)
-	assert.False(t, pager.More())
-	assert.Equal(t, "application/xml", *page.ContentType)
-	require.Len(t, page.Segment.BlobItems, 1)
-	assert.Equal(t, "file.txt", *page.Segment.BlobItems[0].Name)
-	assert.Equal(t, int64(7), *page.Segment.BlobItems[0].Properties.ContentLength)
-	require.Len(t, page.Segment.BlobPrefixes, 1)
-	assert.Equal(t, "subdir/", *page.Segment.BlobPrefixes[0].Name)
-}
-
-func TestPagerXMLFallbackWithEndBefore(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/xml")
-		_, _ = w.Write([]byte(xmlListResponse))
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		EndBefore:      new("testcontainer/n"),
-		UseArrowFormat: new(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.ErrorIs(t, err, ErrEndBeforeXMLFallback)
-}
-
-func TestPagerSharedKeyAuth(t *testing.T) {
-	var gotAuth, gotDate string
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		gotAuth = r.Header.Get("Authorization")
-		gotDate = r.Header.Get("x-ms-date")
-		w.Header().Set("Content-Type", ArrowContentType)
-		_, _ = w.Write(arrowPage(t, "blob.txt", ""))
-	}))
-	defer srv.Close()
-
-	cred, err := NewSharedKeyCredential("testaccount", base64.StdEncoding.EncodeToString([]byte("testkey")))
-	require.NoError(t, err)
-	client, err := NewClientWithSharedKeyCredential(srv.URL+"/testcontainer", cred, nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: new(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.NoError(t, err)
-	assert.True(t, strings.HasPrefix(gotAuth, "SharedKey testaccount:"), "Authorization = %q", gotAuth)
-	assert.NotEmpty(t, gotDate)
-}
-
-func TestPagerResponseError(t *testing.T) {
-	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("x-ms-error-code", string(bloberror.ContainerNotFound))
-		w.WriteHeader(http.StatusNotFound)
-	}))
-	defer srv.Close()
-
-	client, err := NewClientWithNoCredential(srv.URL+"/testcontainer", nil)
-	require.NoError(t, err)
-
-	pager := client.NewListBlobsHierarchyPager("/", &ListBlobsHierarchyOptions{
-		UseArrowFormat: new(true),
-	})
-	_, err = pager.NextPage(context.Background())
-	require.Error(t, err)
-	assert.True(t, bloberror.HasCode(err, bloberror.ContainerNotFound), "expected ContainerNotFound, got %v", err)
-}

backend/azureblob/arrowlist/challenge_policy.go

@@ -1,121 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/shared/challenge_policy.go in the Azure SDK for Go
-// github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0. The
-// released SDK does not export its storage bearer-challenge policy, which is
-// needed for token authentication with the same semantics as the SDK's own
-// clients.
-
-package arrowlist
-
-import (
-	"errors"
-	"net/http"
-	"strings"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
-)
-
-type storageAuthorizer struct {
-	scopes   []string
-	tenantID string
-}
-
-// NewStorageChallengePolicy returns a bearer token policy for cred which
-// handles the storage service's authentication challenges.
-func NewStorageChallengePolicy(cred azcore.TokenCredential, audience string, allowHTTP bool) policy.Policy {
-	s := storageAuthorizer{scopes: []string{audience}}
-	return runtime.NewBearerTokenPolicy(cred, []string{audience}, &policy.BearerTokenOptions{
-		AuthorizationHandler: policy.AuthorizationHandler{
-			OnRequest:   s.onRequest,
-			OnChallenge: s.onChallenge,
-		},
-		InsecureAllowCredentialWithHTTP: allowHTTP,
-	})
-}
-
-func (s *storageAuthorizer) onRequest(req *policy.Request, authNZ func(policy.TokenRequestOptions) error) error {
-	return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-func (s *storageAuthorizer) onChallenge(req *policy.Request, resp *http.Response, authNZ func(policy.TokenRequestOptions) error) error {
-	// parse the challenge
-	err := s.parseChallenge(resp)
-	if err != nil {
-		return err
-	}
-	// TODO: Set tenantID when policy.TokenRequestOptions supports it. https://github.com/Azure/azure-sdk-for-go/issues/19841
-	return authNZ(policy.TokenRequestOptions{Scopes: s.scopes})
-}
-
-type challengePolicyError struct {
-	err error
-}
-
-func (c *challengePolicyError) Error() string {
-	return c.err.Error()
-}
-
-func (*challengePolicyError) NonRetriable() {
-	// marker method
-}
-
-func (c *challengePolicyError) Unwrap() error {
-	return c.err
-}
-
-// parses Tenant ID from auth challenge
-// https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/authorize
-func parseTenant(url string) string {
-	if url == "" {
-		return ""
-	}
-	parts := strings.Split(url, "/")
-	if len(parts) >= 3 {
-		tenant := parts[3]
-		tenant = strings.ReplaceAll(tenant, ",", "")
-		return tenant
-	}
-	return ""
-}
-
-func (s *storageAuthorizer) parseChallenge(resp *http.Response) error {
-	authHeader := resp.Header.Get("WWW-Authenticate")
-	if authHeader == "" {
-		return &challengePolicyError{err: errors.New("response has no WWW-Authenticate header for challenge authentication")}
-	}
-
-	// Strip down to auth and resource
-	// Format is "Bearer authorization_uri=\"<site>\" resource_id=\"<site>\""
-	authHeader = strings.ReplaceAll(authHeader, "Bearer ", "")
-
-	parts := strings.Split(authHeader, " ")
-
-	vals := map[string]string{}
-	for _, part := range parts {
-		subParts := strings.Split(part, "=")
-		if len(subParts) == 2 {
-			stripped := strings.ReplaceAll(subParts[1], "\"", "")
-			stripped = strings.TrimSuffix(stripped, ",")
-			vals[subParts[0]] = stripped
-		}
-	}
-
-	s.tenantID = parseTenant(vals["authorization_uri"])
-
-	scope := vals["resource_id"]
-	if scope == "" {
-		return &challengePolicyError{err: errors.New("could not find a valid resource in the WWW-Authenticate header")}
-	}
-
-	if !strings.HasSuffix(scope, "/.default") {
-		scope += "/.default"
-	}
-	s.scopes = []string{scope}
-	return nil
-}

backend/azureblob/arrowlist/sharedkey.go

@@ -1,318 +0,0 @@
-//go:build !plan9 && !js
-
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License. See License.txt in the project root for license information.
-//
-// Adapted from internal/exported/shared_key_credential.go in the Azure SDK
-// for Go github.com/Azure/azure-sdk-for-go/sdk/storage/azblob at v1.8.0,
-// with the internal header constants inlined. The released SDK does not
-// export its shared key signing policy, so a copy is needed to sign the
-// Arrow listing requests this package sends through its own pipeline.
-
-package arrowlist
-
-import (
-	"bytes"
-	"crypto/hmac"
-	"crypto/sha256"
-	"encoding/base64"
-	"fmt"
-	"net/http"
-	"net/url"
-	"sort"
-	"strings"
-	"sync/atomic"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/rclone/rclone/fs"
-)
-
-// NewSharedKeyCredential creates an immutable SharedKeyCredential containing the
-// storage account's name and either its primary or secondary key.
-func NewSharedKeyCredential(accountName string, accountKey string) (*SharedKeyCredential, error) {
-	c := SharedKeyCredential{accountName: accountName}
-	if err := c.SetAccountKey(accountKey); err != nil {
-		return nil, err
-	}
-	return &c, nil
-}
-
-// SharedKeyCredential contains an account's name and its primary or secondary key.
-type SharedKeyCredential struct {
-	// Only the NewSharedKeyCredential method should set these; all other methods should treat them as read-only
-	accountName string
-	accountKey  atomic.Value // []byte
-}
-
-// AccountName returns the Storage account's name.
-func (c *SharedKeyCredential) AccountName() string {
-	return c.accountName
-}
-
-// SetAccountKey replaces the existing account key with the specified account key.
-func (c *SharedKeyCredential) SetAccountKey(accountKey string) error {
-	_bytes, err := base64.StdEncoding.DecodeString(accountKey)
-	if err != nil {
-		return fmt.Errorf("decode account key: %w", err)
-	}
-	c.accountKey.Store(_bytes)
-	return nil
-}
-
-// ComputeHMACSHA256 generates a hash signature for an HTTP request or for a SAS.
-func (c *SharedKeyCredential) computeHMACSHA256(message string) (string, error) {
-	h := hmac.New(sha256.New, c.accountKey.Load().([]byte))
-	_, err := h.Write([]byte(message))
-	return base64.StdEncoding.EncodeToString(h.Sum(nil)), err
-}
-
-func (c *SharedKeyCredential) buildStringToSign(req *http.Request) (string, error) {
-	// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
-	headers := req.Header
-	contentLength := getHeader("Content-Length", headers)
-	if contentLength == "0" {
-		contentLength = ""
-	}
-
-	canonicalizedResource, err := c.buildCanonicalizedResource(req.URL)
-	if err != nil {
-		return "", err
-	}
-
-	stringToSign := strings.Join([]string{
-		req.Method,
-		getHeader("Content-Encoding", headers),
-		getHeader("Content-Language", headers),
-		contentLength,
-		getHeader("Content-MD5", headers),
-		getHeader("Content-Type", headers),
-		"", // Empty date because x-ms-date is expected (as per web page above)
-		getHeader("If-Modified-Since", headers),
-		getHeader("If-Match", headers),
-		getHeader("If-None-Match", headers),
-		getHeader("If-Unmodified-Since", headers),
-		getHeader("Range", headers),
-		c.buildCanonicalizedHeader(headers),
-		canonicalizedResource,
-	}, "\n")
-	return stringToSign, nil
-}
-
-func getHeader(key string, headers map[string][]string) string {
-	if headers == nil {
-		return ""
-	}
-	if v, ok := headers[key]; ok {
-		if len(v) > 0 {
-			return v[0]
-		}
-	}
-
-	return ""
-}
-
-func getWeightTables() [][]int {
-	tableLv0 := [...]int{
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x71c, 0x0, 0x71f, 0x721, 0x723, 0x725,
-		0x0, 0x0, 0x0, 0x72d, 0x803, 0x0, 0x0, 0x733, 0x0, 0xd03, 0xd1a, 0xd1c, 0xd1e,
-		0xd20, 0xd22, 0xd24, 0xd26, 0xd28, 0xd2a, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25, 0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51,
-		0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99, 0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9,
-		0x0, 0x0, 0x0, 0x743, 0x744, 0x748, 0xe02, 0xe09, 0xe0a, 0xe1a, 0xe21, 0xe23, 0xe25,
-		0xe2c, 0xe32, 0xe35, 0xe36, 0xe48, 0xe51, 0xe70, 0xe7c, 0xe7e, 0xe89, 0xe8a, 0xe91, 0xe99,
-		0xe9f, 0xea2, 0xea4, 0xea6, 0xea7, 0xea9, 0x0, 0x74c, 0x0, 0x750, 0x0,
-	}
-	tableLv2 := [...]int{
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8012, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8212, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-		0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0,
-	}
-	tables := [][]int{tableLv0[:], tableLv2[:]}
-	return tables
-}
-
-// NewHeaderStringComparer performs a multi-level, weight-based comparison of two strings
-func compareHeaders(lhs, rhs string, tables [][]int) int {
-	currLevel, i, j := 0, 0, 0
-	n := len(tables)
-	lhsLen := len(lhs)
-	rhsLen := len(rhs)
-
-	for currLevel < n {
-		if currLevel == (n-1) && i != j {
-			if i > j {
-				return -1
-			}
-			if i < j {
-				return 1
-			}
-			return 0
-		}
-
-		var w1, w2 int
-
-		// Check bounds before accessing lhs[i]
-		if i < lhsLen {
-			w1 = tables[currLevel][lhs[i]]
-		} else {
-			w1 = 0x1
-		}
-
-		// Check bounds before accessing rhs[j]
-		if j < rhsLen {
-			w2 = tables[currLevel][rhs[j]]
-		} else {
-			w2 = 0x1
-		}
-
-		if w1 == 0x1 && w2 == 0x1 {
-			i = 0
-			j = 0
-			currLevel++
-		} else if w1 == w2 {
-			i++
-			j++
-		} else if w1 == 0 {
-			i++
-		} else if w2 == 0 {
-			j++
-		} else {
-			if w1 < w2 {
-				return -1
-			}
-			if w1 > w2 {
-				return 1
-			}
-			return 0
-		}
-	}
-	return 0
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedHeader(headers http.Header) string {
-	cm := map[string][]string{}
-	for k, v := range headers {
-		headerName := strings.TrimSpace(strings.ToLower(k))
-		if strings.HasPrefix(headerName, "x-ms-") {
-			cm[headerName] = v // NOTE: the value must not have any whitespace around it.
-		}
-	}
-	if len(cm) == 0 {
-		return ""
-	}
-
-	keys := make([]string, 0, len(cm))
-	for key := range cm {
-		keys = append(keys, key)
-	}
-	tables := getWeightTables()
-	// Sort the keys using the custom comparator
-	sort.Slice(keys, func(i, j int) bool {
-		return compareHeaders(keys[i], keys[j], tables) < 0
-	})
-	ch := bytes.NewBufferString("")
-	for i, key := range keys {
-		if i > 0 {
-			ch.WriteRune('\n')
-		}
-		ch.WriteString(key)
-		ch.WriteRune(':')
-		ch.WriteString(strings.Join(cm[key], ","))
-	}
-	return ch.String()
-}
-
-func (c *SharedKeyCredential) buildCanonicalizedResource(u *url.URL) (string, error) {
-	// https://docs.microsoft.com/en-us/rest/api/storageservices/authentication-for-the-azure-storage-services
-	cr := bytes.NewBufferString("/")
-	cr.WriteString(c.accountName)
-
-	if len(u.Path) > 0 {
-		// Any portion of the CanonicalizedResource string that is derived from
-		// the resource's URI should be encoded exactly as it is in the URI.
-		// -- https://msdn.microsoft.com/en-gb/library/azure/dd179428.aspx
-		cr.WriteString(u.EscapedPath())
-	} else {
-		// a slash is required to indicate the root path
-		cr.WriteString("/")
-	}
-
-	// params is a map[string][]string; param name is key; params values is []string
-	params, err := url.ParseQuery(u.RawQuery) // Returns URL decoded values
-	if err != nil {
-		return "", fmt.Errorf("failed to parse query params: %w", err)
-	}
-
-	if len(params) > 0 { // There is at least 1 query parameter
-		var paramNames []string // We use this to sort the parameter key names
-		for paramName := range params {
-			paramNames = append(paramNames, paramName) // paramNames must be lowercase
-		}
-		sort.Strings(paramNames)
-
-		for _, paramName := range paramNames {
-			paramValues := params[paramName]
-			sort.Strings(paramValues)
-
-			// Join the sorted key values separated by ','
-			// Then prepend "keyName:"; then add this string to the buffer
-			cr.WriteString("\n" + strings.ToLower(paramName) + ":" + strings.Join(paramValues, ","))
-		}
-	}
-	return cr.String(), nil
-}
-
-// ComputeHMACSHA256 is a helper for computing the signed string outside of this package.
-func ComputeHMACSHA256(cred *SharedKeyCredential, message string) (string, error) {
-	return cred.computeHMACSHA256(message)
-}
-
-// SharedKeyCredPolicy is a pipeline policy that signs each request with the
-// account's shared key.
-type SharedKeyCredPolicy struct {
-	cred *SharedKeyCredential
-}
-
-// NewSharedKeyCredPolicy creates a SharedKeyCredPolicy signing with cred.
-func NewSharedKeyCredPolicy(cred *SharedKeyCredential) *SharedKeyCredPolicy {
-	return &SharedKeyCredPolicy{cred: cred}
-}
-
-// Do signs the request and sends it via the next policy in the pipeline.
-func (s *SharedKeyCredPolicy) Do(req *policy.Request) (*http.Response, error) {
-	// skip adding the authorization header if no SharedKeyCredential was provided.
-	// this prevents a panic that might be hard to diagnose and allows testing
-	// against http endpoints that don't require authentication.
-	if s.cred == nil {
-		return req.Next()
-	}
-
-	if d := getHeader("x-ms-date", req.Raw().Header); d == "" {
-		req.Raw().Header.Set("x-ms-date", time.Now().UTC().Format(http.TimeFormat))
-	}
-	stringToSign, err := s.cred.buildStringToSign(req.Raw())
-	if err != nil {
-		return nil, err
-	}
-	signature, err := s.cred.computeHMACSHA256(stringToSign)
-	if err != nil {
-		return nil, err
-	}
-	authHeader := strings.Join([]string{"SharedKey ", s.cred.AccountName(), ":", signature}, "")
-	req.Raw().Header.Set("Authorization", authHeader)
-
-	response, err := req.Next()
-	if err != nil && response != nil && response.StatusCode == http.StatusForbidden {
-		// Service failed to authenticate request, log it
-		fs.Debugf(nil, "azureblob arrowlist: HTTP Forbidden status, String-to-Sign: %q", stringToSign)
-	}
-	return response, err
-}

backend/azureblob/azureblob.go

@@ -19,6 +19,7 @@ import (
 	"path"
 	"slices"
 	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -26,14 +27,12 @@ import (
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
-	"github.com/rclone/rclone/backend/azureblob/arrowlist"
 	"github.com/rclone/rclone/backend/azureblob/auth"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/chunksize"
@@ -241,40 +240,38 @@ avoid the time out.`,
 			Advanced: true,
 		}, {
 			Name: "use_arrow_list",
-			Help: `Use the experimental Apache Arrow listing format.
+			Help: `Use the Apache Arrow listing format.
 
 If set, directory listings are fetched using the ListBlobs Apache
-Arrow response format instead of XML. This can be faster for very
-large containers.
+Arrow response format instead of XML. Arrow responses are smaller and
+much cheaper to parse, making listings of large containers several
+times faster. Combine with "list_parallelism" for the biggest gains.
 
-This is EXPERIMENTAL and requires the "Blob Listing with Apache Arrow"
-preview feature to be enabled on the storage account. It is NOT
-supported on accounts with a hierarchical namespace (ADLS Gen2) -
-those return a 409 error. If the feature is not enabled the server
-returns XML and the listing transparently falls back to the normal XML
-path (logged at debug level).
-
-Not supported with connection_string auth - falls back to normal
-listing.`,
+"Blob Listing with Apache Arrow" is in public preview at Microsoft and
+is only supported on flat namespace accounts. On accounts with a
+hierarchical namespace (ADLS Gen2), or where the feature is otherwise
+unavailable, the server returns XML and the listing transparently
+falls back to the normal XML path (logged at debug level).`,
 			Default:  false,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
 		}, {
 			Name: "list_parallelism",
 			Help: `Number of parallel shards to list a directory with.
 
-EXPERIMENTAL. If set greater than 1, the blob name keyspace of each
-directory is split into this many ranges which are listed concurrently
-using the Arrow startFrom/endBefore range parameters. This can
-dramatically speed up listing containers with millions of objects, for
-both recursive (ListR) and single directory listings.
+If set greater than 1, the blob name keyspace of each directory is
+split into ranges which are listed concurrently using the Arrow
+startFrom/endBefore range parameters. This can dramatically speed up
+listing containers with millions of objects, for both recursive
+(ListR) and single directory listings. Speed keeps improving up to a
+parallelism of around 30.
 
 This has no effect unless "use_arrow_list" is also set, as Arrow is the
-only listing path that supports server-side name ranges. The default of
-0 (or 1) lists sequentially.`,
+only listing path that supports server-side name ranges. If the
+account does not support range listing (e.g. it has a hierarchical
+namespace) the listing falls back to sequential. The default of 0 (or
+1) lists sequentially.`,
 			Default:  0,
 			Advanced: true,
-			Hide:     fs.OptionHideBoth,
 		}, {
 			Name: "access_tier",
 			Help: `Access tier of blob: hot, cool, cold or archive.
@@ -368,7 +365,22 @@ creates an empty object ending with "/", to persist the folder.
 
 This object also has the metadata "` + dirMetaKey + ` = ` + dirMetaValue + `" to conform to
 the Microsoft standard.
+
+This is not needed on storage accounts with a hierarchical namespace
+(see the hns option) as these always support empty directories.
  `,
+		}, {
+			Name:     "hns",
+			Default:  fs.Tristate{},
+			Advanced: true,
+			Help: `Set if the storage account has a hierarchical namespace.
+
+Storage accounts with hierarchical namespace enabled (Azure Data Lake
+Storage Gen2) have real directories which can be empty.
+
+If this is unset then rclone will read it from the storage account
+and save it in the config file.
+`,
 		}, {
 			Name: "no_check_container",
 			Help: `If set, don't attempt to check the container exists or create it.
@@ -436,6 +448,7 @@ type Options struct {
 	Enc                  encoder.MultiEncoder `config:"encoding"`
 	PublicAccess         string               `config:"public_access"`
 	DirectoryMarkers     bool                 `config:"directory_markers"`
+	HNS                  fs.Tristate          `config:"hns"`
 	NoCheckContainer     bool                 `config:"no_check_container"`
 	NoHeadObject         bool                 `config:"no_head_object"`
 	DeleteSnapshots      string               `config:"delete_snapshots"`
@@ -449,10 +462,8 @@ type Fs struct {
 	opt                Options                      // parsed config options
 	ci                 *fs.ConfigInfo               // global config
 	features           *fs.Features                 // optional features
-	cntSVCcacheMu      sync.Mutex                   // mutex to protect cntSVCcache and arrowCntSVCcache
+	cntSVCcacheMu      sync.Mutex                   // mutex to protect cntSVCcache
 	cntSVCcache        map[string]*container.Client // reference to containerClient per container
-	arrowCntSVCcache   map[string]*arrowlist.Client // reference to arrowlist client per container
-	arrowClientOpts    *arrowlist.ClientOptions     // client options for the arrowlist clients
 	arrowXMLFallback   atomic.Bool                  // set once the server answers XML so parallel listing is not retried
 	svc                *service.Client              // client to access azblob
 	cred               azcore.TokenCredential       // how to generate tokens (may be nil)
@@ -461,6 +472,7 @@ type Fs struct {
 	rootContainer      string                       // container part of root (if any)
 	rootDirectory      string                       // directory part of root (if any)
 	isLimited          bool                         // if limited to one container
+	hns                bool                         // if the storage account has a hierarchical namespace
 	cache              *bucket.Cache                // cache for container creation status
 	pacer              *fs.Pacer                    // To pace and retry the API calls
 	uploadToken        *pacer.TokenDispenser        // control concurrency
@@ -527,7 +539,7 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
 	containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
-	if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
+	if f.dirMarkers() && strings.HasSuffix(containerPath, "//") {
 		containerPath = containerPath[:len(containerPath)-1]
 	}
 	return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
@@ -670,15 +682,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:             name,
-		opt:              *opt,
-		ci:               ci,
-		pacer:            fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		uploadToken:      pacer.NewTokenDispenser(ci.Transfers),
-		copyToken:        pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
-		cache:            bucket.NewCache(),
-		cntSVCcache:      make(map[string]*container.Client, 1),
-		arrowCntSVCcache: make(map[string]*arrowlist.Client, 1),
+		name:        name,
+		opt:         *opt,
+		ci:          ci,
+		pacer:       fs.NewPacer(ctx, pacer.NewS3(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+		copyToken:   pacer.NewTokenDispenser(opt.CopyTotalConcurrency),
+		cache:       bucket.NewCache(),
+		cntSVCcache: make(map[string]*container.Client, 1),
 	}
 	f.publicAccess = container.PublicAccessType(opt.PublicAccess)
 	f.setRoot(root)
@@ -695,10 +706,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ServerSideAcrossConfigs: true,
 		DoubleSlash:             true,
 	}).Fill(ctx, f)
-	if opt.DirectoryMarkers {
-		f.features.CanHaveEmptyDirectories = true
-		fs.Debugf(f, "Using directory markers")
-	}
 
 	conf := auth.NewClientOpts[service.Client, service.ClientOptions, service.SharedKeyCredential]{
 		DefaultBaseURL:                   storageDefaultBaseURL,
@@ -725,21 +732,34 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	f.usingSharedKeyCred = res.UsingSharedKeyCred
 	f.anonymous = res.Anonymous
 
-	// Client options for the arrowlist clients, using the same transport and
-	// gzip policy as the SDK clients built above.
-	f.arrowClientOpts = &arrowlist.ClientOptions{
-		ClientOptions: azcore.ClientOptions{
-			Transport:       auth.Transporter(ctx),
-			PerCallPolicies: []policy.Policy{setAcceptEncodingGzip{}},
-		},
-	}
-
 	// if using Container level SAS put the container client into the cache
 	if opt.SASURL != "" && res.Container != "" {
 		_ = f.cntSVC(res.Container)
 		f.isLimited = true
 	}
 
+	if opt.HNS.Valid {
+		f.hns = opt.HNS.Value
+	} else if !f.anonymous {
+		containerName := f.rootContainer
+		if containerName == "" {
+			containerName = res.Container
+		}
+		f.hns, err = f.readHNS(ctx, containerName)
+		if err != nil {
+			fs.Debugf(f, "Failed to read whether storage account has a hierarchical namespace - assuming not: %v", err)
+		} else {
+			m.Set("hns", strconv.FormatBool(f.hns))
+		}
+	}
+	if f.hns {
+		fs.Debugf(f, "Storage account has a hierarchical namespace")
+	}
+	if f.dirMarkers() {
+		f.features.CanHaveEmptyDirectories = true
+		fs.Debugf(f, "Using directory markers")
+	}
+
 	if f.rootContainer != "" && f.rootDirectory != "" {
 		// Check to see if the (container,directory) is actually an existing file
 		oldRoot := f.root
@@ -760,6 +780,39 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
+// readHNS reads whether the storage account has a hierarchical namespace
+//
+// If containerName is set then the container client is used which works
+// with credentials limited to that container.
+func (f *Fs) readHNS(ctx context.Context, containerName string) (hns bool, err error) {
+	var isHNS *bool
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		if containerName != "" {
+			var resp container.GetAccountInfoResponse
+			resp, err = f.cntSVC(containerName).GetAccountInfo(ctx, nil)
+			isHNS = resp.IsHierarchicalNamespaceEnabled
+		} else {
+			var resp service.GetAccountInfoResponse
+			resp, err = f.svc.GetAccountInfo(ctx, nil)
+			isHNS = resp.IsHierarchicalNamespaceEnabled
+		}
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return false, err
+	}
+	if isHNS == nil {
+		return false, errors.New("not returned by server")
+	}
+	return *isHNS, nil
+}
+
+// dirMarkers returns true if directories are persisted with directory markers
+func (f *Fs) dirMarkers() bool {
+	return f.opt.DirectoryMarkers || f.hns
+}
+
 // return the container client for the container passed in
 func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
 	f.cntSVCcacheMu.Lock()
@@ -772,44 +825,6 @@ func (f *Fs) cntSVC(containerName string) (containerClient *container.Client) {
 	return containerClient
 }
 
-// errArrowAuthUnsupported is returned by arrowCntSVC when the configured
-// credentials can't be used for Arrow listing.
-var errArrowAuthUnsupported = errors.New("credentials not supported for Arrow listing (connection_string auth is not supported)")
-
-// return the arrowlist client for the container passed in
-//
-// Returns errArrowAuthUnsupported if the configured credentials can't be
-// reused for the arrowlist client's pipeline.
-func (f *Fs) arrowCntSVC(containerName string) (client *arrowlist.Client, err error) {
-	// The container URL includes any SAS token in its query
-	url := f.cntSVC(containerName).URL()
-	f.cntSVCcacheMu.Lock()
-	defer f.cntSVCcacheMu.Unlock()
-	if client, ok := f.arrowCntSVCcache[containerName]; ok {
-		return client, nil
-	}
-	switch {
-	case f.usingSharedKeyCred:
-		// Covers account+key and the emulator (auth fills in Account/Key)
-		var cred *arrowlist.SharedKeyCredential
-		cred, err = arrowlist.NewSharedKeyCredential(f.opt.Account, f.opt.Key)
-		if err == nil {
-			client, err = arrowlist.NewClientWithSharedKeyCredential(url, cred, f.arrowClientOpts)
-		}
-	case f.cred != nil:
-		client, err = arrowlist.NewClient(url, f.cred, f.arrowClientOpts)
-	case f.anonymous || f.opt.SASURL != "":
-		client, err = arrowlist.NewClientWithNoCredential(url, f.arrowClientOpts)
-	default:
-		return nil, errArrowAuthUnsupported
-	}
-	if err != nil {
-		return nil, err
-	}
-	f.arrowCntSVCcache[containerName] = client
-	return client, nil
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -835,7 +850,12 @@ func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *contain
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(ctx, remote, nil)
+	o, err := f.newObjectWithInfo(ctx, remote, nil)
+	if err == fs.ErrorNotAFile {
+		// The blob is a directory
+		return nil, fs.ErrorObjectNotFound
+	}
+	return o, err
 }
 
 // getBlobSVC creates a blob client
@@ -1149,18 +1169,33 @@ func isDirectoryMarker(size int64, metadata map[string]*string, remote string) b
 		if endsWithSlash || remote == "" {
 			return true
 		}
-		// Note that metadata with hdi_isfolder = true seems to be a
-		// defacto standard for marking blobs as directories.
-		// Note also that the metadata hasn't been normalised to lower case yet
-		for k, v := range metadata {
-			if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
-				return true
-			}
+		return hasDirMetadata(metadata)
+	}
+	return false
+}
+
+// Returns whether the metadata marks the blob as a directory
+func hasDirMetadata(metadata map[string]*string) bool {
+	// Note that metadata with hdi_isfolder = true seems to be a
+	// defacto standard for marking blobs as directories.
+	// Note also that the metadata hasn't been normalised to lower case yet
+	for k, v := range metadata {
+		if v != nil && strings.EqualFold(k, dirMetaKey) && *v == dirMetaValue {
+			return true
 		}
 	}
 	return false
 }
 
+// Returns whether props are for a directory on a storage account with
+// a hierarchical namespace
+//
+// These ignore a trailing "/" on the name so a file can be returned
+// when looking for a directory.
+func isHNSDirectory(props *blob.GetPropertiesResponse) bool {
+	return props.ContentLength != nil && *props.ContentLength == 0 && hasDirMetadata(props.Metadata)
+}
+
 // listFn is called from list to handle an object
 type listFn func(remote string, object *container.BlobItem, isDirectory bool) error
 
@@ -1186,27 +1221,25 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 		delimiter = "/"
 	}
 
-	opts := &arrowlist.ListBlobsHierarchyOptions{
-		ListBlobsHierarchyOptions: container.ListBlobsHierarchyOptions{
-			// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
-			Include: container.ListBlobsInclude{
-				Copy:             false,
-				Metadata:         true,
-				Snapshots:        false,
-				UncommittedBlobs: false,
-				Deleted:          false,
-			},
-			Prefix:     &directory,
-			MaxResults: &maxResults,
+	opts := &container.ListBlobsHierarchyOptions{
+		// Copy, Metadata, Snapshots, UncommittedBlobs, Deleted, Tags, Versions, LegalHold, ImmutabilityPolicy, DeletedWithVersions bool
+		Include: container.ListBlobsInclude{
+			Copy:             false,
+			Metadata:         true,
+			Snapshots:        false,
+			UncommittedBlobs: false,
+			Deleted:          false,
 		},
+		Prefix:     &directory,
+		MaxResults: &maxResults,
 	}
-	// Experimental: request the Apache Arrow listing format. The arrowlist
-	// pager requests an Arrow IPC stream and decodes it, falling back to XML
-	// if the account doesn't have Arrow listing enabled. Skip the
-	// maxResults==1 probe (isEmpty) which doesn't benefit.
+	// Request the Apache Arrow listing format. The SDK pager requests an
+	// Arrow IPC stream and decodes it, falling back to XML if the account
+	// doesn't have Arrow listing enabled. Skip the maxResults==1 probe
+	// (isEmpty) which doesn't benefit.
 	useArrow := f.opt.UseArrowList && maxResults != 1
 	if useArrow {
-		opts.UseArrowFormat = new(true)
+		opts.ResponseFormat = container.StorageResponseFormatArrow
 	}
 
 	var foundItems int
@@ -1221,15 +1254,18 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 	if err != nil {
 		return err
 	}
-	if f.opt.DirectoryMarkers && foundItems == 0 && directory != "" {
+	if f.dirMarkers() && foundItems == 0 && directory != "" {
 		// Determine whether the directory exists or not by whether it has a marker
-		_, err := f.readMetaData(ctx, containerName, directory)
+		props, err := f.readMetaData(ctx, containerName, directory)
 		if err != nil {
 			if err == fs.ErrorObjectNotFound {
 				return fs.ErrorDirNotFound
 			}
 			return err
 		}
+		if f.hns && !isHNSDirectory(props) {
+			return fs.ErrorDirNotFound
+		}
 	}
 	return nil
 }
@@ -1237,25 +1273,11 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 // listBlobsPager runs the hierarchy listing described by opts, calling fn for
 // each blob and subdirectory, and returns the number of raw items seen.
 // delimiter selects flat (recurse) vs hierarchical listing. If opts requests
-// the Apache Arrow format the listing goes through the arrowlist pager
-// (falling back to the SDK pager if the credentials don't support it); if the
-// service then answers with XML (Arrow listing not enabled) a debug message
-// is logged.
-func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *arrowlist.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
-	useArrow := opts.UseArrowFormat != nil && *opts.UseArrowFormat
-	var pager *runtime.Pager[container.ListBlobsHierarchyResponse]
-	if useArrow {
-		arrowSVC, err := f.arrowCntSVC(containerName)
-		if err != nil {
-			fs.Debugf(f, "Not using Arrow listing: %v", err)
-			useArrow = false
-		} else {
-			pager = arrowSVC.NewListBlobsHierarchyPager(delimiter, opts)
-		}
-	}
-	if pager == nil {
-		pager = f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, &opts.ListBlobsHierarchyOptions)
-	}
+// the Apache Arrow format and the service answers with XML (Arrow listing
+// not enabled) a debug message is logged.
+func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefix string, addContainer bool, opts *container.ListBlobsHierarchyOptions, delimiter string, fn listFn) (foundItems int, err error) {
+	useArrow := opts.ResponseFormat == container.StorageResponseFormatArrow
+	pager := f.cntSVC(containerName).NewListBlobsHierarchyPager(delimiter, opts)
 	checkedArrow := false
 	for pager.More() {
 		var response container.ListBlobsHierarchyResponse
@@ -1271,7 +1293,7 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
 		// but not accelerated.
 		if useArrow && !checkedArrow && err == nil {
 			checkedArrow = true
-			if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowlist.ArrowContentType) {
+			if response.ContentType == nil || !strings.HasPrefix(*response.ContentType, arrowContentType) {
 				fs.Debugf(f, "Apache Arrow listing requested but server returned XML - Blob Listing with Apache Arrow may not be enabled on this account")
 			}
 		}
@@ -1296,6 +1318,13 @@ func (f *Fs) listBlobsPager(ctx context.Context, containerName, directory, prefi
 				fs.Debugf(f, "Nil name received")
 				continue
 			}
+			// Storage accounts with a hierarchical namespace return
+			// the file "dir" when listing "dir/"
+			if !strings.HasPrefix(*file.Name, directory) {
+				fs.Debugf(f, "Ignoring %q not in directory %q", *file.Name, directory)
+				foundItems--
+				continue
+			}
 			remote := f.opt.Enc.ToStandardPath(*file.Name)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Debugf(f, "Odd name received %q", remote)
@@ -1557,7 +1586,7 @@ type listContainerFn func(Name string, LastModified time.Time) error
 func (f *Fs) listContainersToFn(fn listContainerFn) error {
 	max := int32(f.opt.ListChunkSize)
 	pager := f.svc.NewListContainersPager(&service.ListContainersOptions{
-		Include:    service.ListContainersInclude{Metadata: true, Deleted: true},
+		Include:    service.ListContainersInclude{Metadata: true},
 		MaxResults: &max,
 	})
 	ctx := context.Background()
@@ -1607,7 +1636,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // Create directory marker file and parents
 func (f *Fs) createDirectoryMarker(ctx context.Context, container, dir string) error {
-	if !f.opt.DirectoryMarkers || container == "" {
+	if !f.dirMarkers() || container == "" {
 		return nil
 	}
 
@@ -1664,6 +1693,11 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
+	if f.hns {
+		// Parent directories are created by the upload
+		container, _ := f.split(remote)
+		return f.makeContainer(ctx, container)
+	}
 	remote, _ = strings.CutSuffix(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
@@ -1769,6 +1803,9 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	container, directory := f.split(dir)
+	if f.hns && container != "" && directory != "" {
+		return f.rmdirHNS(ctx, container, directory)
+	}
 	// Remove directory marker file
 	if f.opt.DirectoryMarkers && container != "" && directory != "" {
 		o := &Object{
@@ -1791,6 +1828,39 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return f.deleteContainer(ctx, container)
 }
 
+// rmdirHNS removes the directory on a storage account with a
+// hierarchical namespace
+//
+// The directory must be deleted without a trailing "/" and the server
+// refuses to delete it if it isn't empty.
+func (f *Fs) rmdirHNS(ctx context.Context, container, directory string) error {
+	directory, _ = strings.CutSuffix(directory, "/")
+	// Check it is a directory as deleting a file would succeed
+	props, err := f.readMetaData(ctx, container, directory)
+	if err == fs.ErrorObjectNotFound {
+		return fs.ErrorDirNotFound
+	} else if err != nil {
+		return err
+	}
+	if !isHNSDirectory(props) {
+		return fs.ErrorIsFile
+	}
+	blb := f.getBlobSVC(container, directory)
+	err = f.pacer.Call(func() (bool, error) {
+		_, err := blb.Delete(ctx, nil)
+		return f.shouldRetry(ctx, err)
+	})
+	if storageErr, ok := errors.AsType[*azcore.ResponseError](err); ok {
+		if storageErr.ErrorCode == "DirectoryIsNotEmpty" {
+			return fs.ErrorDirectoryNotEmpty
+		}
+		if storageErr.StatusCode == http.StatusNotFound {
+			return fs.ErrorDirNotFound
+		}
+	}
+	return err
+}
+
 // Precision of the remote
 func (f *Fs) Precision() time.Duration {
 	return time.Nanosecond

backend/azureblob/azureblob_internal_test.go

@@ -325,6 +325,14 @@ func assertHeadersAndMetadata(t *testing.T, props *blob.GetPropertiesResponse, w
 }
 
 // helper to read blob tags for an object
+// skipIfHNS skips the test if the storage account has a hierarchical
+// namespace as these don't support blob tags
+func skipIfHNS(t *testing.T, f *Fs) {
+	if f.hns {
+		t.Skip("blob tags not supported with hierarchical namespace")
+	}
+}
+
 func getTagsMap(ctx context.Context, t *testing.T, o fs.Object) map[string]string {
 	ao := o.(*Object)
 	blb := ao.getBlobSVC()
@@ -410,6 +418,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 		// Tags: Singlepart upload
 		t.Run("PutSinglepartTags", func(t *testing.T) {
+			skipIfHNS(t, f)
 			contents := random.String(int(f.opt.ChunkSize / 2))
 			item := fstest.NewItem("tags-single.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 			meta := fs.Metadata{
@@ -425,6 +434,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 		// Tags: Multipart upload
 		t.Run("PutMultipartTags", func(t *testing.T) {
+			skipIfHNS(t, f)
 			contents := random.String(int(f.opt.ChunkSize + 2048))
 			item := fstest.NewItem("tags-multipart.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
 			meta := fs.Metadata{
@@ -612,6 +622,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 	// Tags: Singlepart copy
 	t.Run("CopySinglepartTags", func(t *testing.T) {
+		skipIfHNS(t, f)
 		// create small source
 		contents := random.String(int(f.opt.ChunkSize / 2))
 		srcItem := fstest.NewItem("tags-copy-single-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
@@ -637,6 +648,7 @@ func (f *Fs) testMetadataPaths(t *testing.T) {
 
 	// Tags: Multipart copy
 	t.Run("CopyMultipartTags", func(t *testing.T) {
+		skipIfHNS(t, f)
 		// create large source to force multipart
 		contents := random.String(int(f.opt.CopyCutoff + 4096))
 		srcItem := fstest.NewItem("tags-copy-multi-src.txt", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))

backend/azurefiles/azurefiles.go

@@ -79,7 +79,7 @@ This is required and is the name of the share to access.
 			Help: `Upload chunk size.
 
 Note that this is stored in memory and there may be up to
-"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
+"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
 in memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
@@ -95,7 +95,7 @@ links and these uploads do not fully utilize your bandwidth, then
 increasing this may help to speed up the transfers.
 
 Note that chunks are stored in memory and there may be up to
-"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
+"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
 in memory.`,
 			Default:  16,
 			Advanced: true,

backend/b2/b2.go

@@ -191,6 +191,11 @@ If the custom endpoint rewrites the requests for authentication,
 e.g., in Cloudflare Workers, this header needs to be handled properly.
 Leave blank if you want to use the endpoint provided by Backblaze.
 
+Rclone will not follow a redirect from this endpoint which downgrades
+HTTPS to HTTP, and will not send the "Authorization" header or the
+SSE-C key headers once a redirect has left the host and port of this
+endpoint.
+
 The URL provided here SHOULD have the protocol and SHOULD NOT have
 a trailing slash or specify the /file/bucket subpath as rclone will
 request files with "{download_url}/file/{bucket_name}/{path}".
@@ -272,7 +277,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
 			Name: "sse_customer_key",
 			Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
 
-Alternatively you can provide --sse-customer-key-base64.`,
+Alternatively you can provide --b2-sse-customer-key-base64.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -283,7 +288,7 @@ Alternatively you can provide --sse-customer-key-base64.`,
 			Name: "sse_customer_key_base64",
 			Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
 
-Alternatively you can provide --sse-customer-key.`,
+Alternatively you can provide --b2-sse-customer-key.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -529,6 +534,25 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
+// redirectSecretHeaders are the request headers carrying secrets that
+// must not be forwarded when a redirect leaves the original host.
+var redirectSecretHeaders = []string{
+	"Authorization",
+	sseAlgorithmHeader,
+	sseKeyHeader,
+	sseMd5Header,
+}
+
+// newClient makes the http client used for all B2 requests.
+//
+// The download_url option sends the requests to a host the user does
+// not control so it strips headers on cross host redirects.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(redirectSecretHeaders...)
+	return client
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -581,7 +605,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		name:        name,
 		opt:         *opt,
 		ci:          ci,
-		srv:         rest.NewClient(fshttp.NewClient(ctx)).SetErrorHandler(errorHandler),
+		srv:         rest.NewClient(newClient(ctx)).SetErrorHandler(errorHandler),
 		cache:       bucket.NewCache(),
 		_bucketID:   make(map[string]string, 1),
 		_bucketType: make(map[string]string, 1),

backend/b2/b2_internal_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"crypto/sha1"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"path"
 	"sort"
 	"strings"
@@ -617,3 +619,60 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// redirectTestHeaders is a deliberately literal copy of
+// redirectSecretHeaders so that a header dropped from the production
+// list fails TestRedirectSecretHeaders rather than silently losing
+// test coverage.
+var redirectTestHeaders = map[string]string{
+	"Authorization": "secret-token",
+	"X-Bz-Server-Side-Encryption-Customer-Algorithm": "AES256",
+	"X-Bz-Server-Side-Encryption-Customer-Key":       "secret-key",
+	"X-Bz-Server-Side-Encryption-Customer-Key-Md5":   "secret-md5",
+}
+
+func TestRedirectSecretHeaders(t *testing.T) {
+	want := make([]string, 0, len(redirectTestHeaders))
+	for header := range redirectTestHeaders {
+		want = append(want, header)
+	}
+	sort.Strings(want)
+	got := append([]string(nil), redirectSecretHeaders...)
+	sort.Strings(got)
+	assert.Equal(t, want, got)
+}
+
+// redirectTestRequest makes a GET request to rawURL carrying the
+// secret headers plus a harmless marker header.
+func redirectTestRequest(t *testing.T, rawURL string) *http.Request {
+	req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, rawURL, nil)
+	require.NoError(t, err)
+	for header, value := range redirectTestHeaders {
+		req.Header.Set(header, value)
+	}
+	req.Header.Set("X-Bz-Test-Mode", "marker")
+	return req
+}
+
+func TestClientStripsSecretHeadersOnCrossHostRedirect(t *testing.T) {
+	client := newClient(context.Background())
+
+	redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for header := range redirectTestHeaders {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+		assert.Equal(t, "marker", r.Header.Get("X-Bz-Test-Mode"))
+		w.WriteHeader(http.StatusOK)
+	}))
+	defer redirectServer.Close()
+
+	initialServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, redirectServer.URL, http.StatusTemporaryRedirect)
+	}))
+	defer initialServer.Close()
+
+	resp, err := client.Do(redirectTestRequest(t, initialServer.URL))
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+	assert.NoError(t, resp.Body.Close())
+}

backend/box/box.go

@@ -27,7 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/golang-jwt/jwt/v4"
+	"github.com/golang-jwt/jwt/v5"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -77,7 +77,10 @@ var (
 )
 
 type boxCustomClaims struct {
-	jwt.StandardClaims
+	jwt.RegisteredClaims
+	// Audience overrides RegisteredClaims.Audience so it is sent as
+	// a single string rather than an array of strings
+	Audience   string `json:"aud,omitempty"`
 	BoxSubType string `json:"box_sub_type,omitempty"`
 }
 
@@ -223,7 +226,7 @@ func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
 
 	if configFileCredentials == "" {
 		configFile, _ := m.Get("box_config_file")
-		configFileBytes, err = os.ReadFile(configFile)
+		configFileBytes, err = os.ReadFile(env.ShellExpand(configFile))
 		if err != nil {
 			return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 		}
@@ -243,15 +246,13 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	}
 
 	claims = &boxCustomClaims{
-		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
-		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
-		StandardClaims: jwt.StandardClaims{
-			Id:        val,
+		RegisteredClaims: jwt.RegisteredClaims{
+			ID:        val,
 			Issuer:    boxConfig.BoxAppSettings.ClientID,
 			Subject:   boxConfig.EnterpriseID,
-			Audience:  tokenURL,
-			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
 		},
+		Audience:   tokenURL,
 		BoxSubType: boxSubType,
 	}
 	return claims, nil

backend/box/box_internal_test.go

@@ -0,0 +1,118 @@
+package box
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/box/api"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/lib/jwtutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// decodeJWTPart decodes one base64url encoded part of a JWT into a map
+func decodeJWTPart(t *testing.T, part string) map[string]any {
+	data, err := base64.RawURLEncoding.DecodeString(part)
+	require.NoError(t, err)
+	var result map[string]any
+	require.NoError(t, json.Unmarshal(data, &result))
+	return result
+}
+
+// Test the JWT assertion sent to the token endpoint is in the format Box expects
+func TestJWTAssertion(t *testing.T) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+	boxConfig := &api.ConfigJSON{
+		BoxAppSettings: api.AppSettings{
+			ClientID:     "client-id",
+			ClientSecret: "client-secret",
+			AppAuth: api.AppAuth{
+				PublicKeyID: "public-key-id",
+			},
+		},
+		EnterpriseID: "enterprise-id",
+	}
+
+	var form map[string][]string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "POST", r.Method)
+		assert.NoError(t, r.ParseForm())
+		form = r.PostForm
+		_, _ = w.Write([]byte(`{"access_token":"access-token","token_type":"bearer","expires_in":3600}`))
+	}))
+	defer ts.Close()
+
+	claims, err := getClaims(boxConfig, "enterprise")
+	require.NoError(t, err)
+	m := configmap.Simple{}
+	start := time.Now()
+	err = jwtutil.Config("box", "test", ts.URL, *claims, getSigningHeaders(boxConfig), getQueryParams(boxConfig), privateKey, m, ts.Client())
+	require.NoError(t, err)
+
+	// Check the token was stored
+	assert.Contains(t, m["token"], `"access_token":"access-token"`)
+
+	// Check the form parameters
+	assert.Equal(t, []string{"urn:ietf:params:oauth:grant-type:jwt-bearer"}, form["grant_type"])
+	assert.Equal(t, []string{"client-id"}, form["client_id"])
+	assert.Equal(t, []string{"client-secret"}, form["client_secret"])
+	require.Len(t, form["assertion"], 1)
+	parts := strings.Split(form["assertion"][0], ".")
+	require.Len(t, parts, 3)
+
+	// Check the signature independently of the JWT library
+	signature, err := base64.RawURLEncoding.DecodeString(parts[2])
+	require.NoError(t, err)
+	hash := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
+	require.NoError(t, rsa.VerifyPKCS1v15(&privateKey.PublicKey, crypto.SHA256, hash[:], signature))
+
+	// Check the header
+	assert.Equal(t, map[string]any{
+		"alg": "RS256",
+		"typ": "JWT",
+		"kid": "public-key-id",
+	}, decodeJWTPart(t, parts[0]))
+
+	// Check the claims - aud must be a string, not an array
+	payload := decodeJWTPart(t, parts[1])
+	exp, ok := payload["exp"].(float64)
+	require.True(t, ok, "exp should be a number")
+	assert.InDelta(t, float64(start.Add(45*time.Second).Unix()), exp, 5)
+	jti, ok := payload["jti"].(string)
+	require.True(t, ok, "jti should be a string")
+	assert.Len(t, jti, 40)
+	delete(payload, "exp")
+	delete(payload, "jti")
+	assert.Equal(t, map[string]any{
+		"aud":          tokenURL,
+		"iss":          "client-id",
+		"sub":          "enterprise-id",
+		"box_sub_type": "enterprise",
+	}, payload)
+}
+
+// Test box_config_file has ~ and environment variables expanded
+func TestGetBoxConfigShellExpand(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"enterpriseID":"enterprise-id"}`), 0600))
+	t.Setenv("RCLONE_TEST_BOX_CONFIG_DIR", dir)
+	m := configmap.Simple{
+		"box_config_file": "$RCLONE_TEST_BOX_CONFIG_DIR/config.json",
+	}
+	boxConfig, err := getBoxConfig(m)
+	require.NoError(t, err)
+	assert.Equal(t, "enterprise-id", boxConfig.EnterpriseID)
+}

backend/chunker/chunker.go

@@ -801,9 +801,12 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 				fs.Debugf(f, "skip orphan data chunk %q", remote)
 				break
 			}
-			if mainObject == nil && !f.useMeta {
+			if mainObject == nil && !f.useMeta && !isSpecial {
 				// this is the "nometa" case
 				// create dummy chunked object without metadata
+				//
+				// not for temporary or control chunks as they may be all
+				// that is left of an interrupted upload
 				mainObject = f.newObject(mainRemote, nil, nil)
 				byRemote[mainRemote] = mainObject
 				if !badEntry[mainRemote] {
@@ -1499,11 +1502,21 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
 func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
 	oldFsObject, err := f.NewObject(ctx, remote)
 	if err == nil {
-		oldObject := oldFsObject.(*Object)
-		for _, chunk := range oldObject.chunks {
-			if err := chunk.Remove(ctx); err != nil {
-				fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
-			}
+		removeStaleChunks(ctx, oldFsObject.(*Object), nil)
+	}
+}
+
+func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
+	currentRemotes := make(map[string]struct{}, len(currentChunks))
+	for _, chunk := range currentChunks {
+		currentRemotes[chunk.Remote()] = struct{}{}
+	}
+	for _, chunk := range oldObject.chunks {
+		if _, found := currentRemotes[chunk.Remote()]; found {
+			continue
+		}
+		if err := chunk.Remove(ctx); err != nil {
+			fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
 		}
 	}
 }
@@ -1684,12 +1697,20 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		// metadata format which might involve unsupported chunk types.
 		return nil, fmt.Errorf("can't %s this file: %w", opName, err)
 	}
+	// Snapshot destination chunks before its metadata is replaced.
+	var oldObject *Object
+	if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
+		oldObject = oldFsObject.(*Object)
+	}
 	if !o.isComposite() {
 		fs.Debugf(o, "%s non-chunked object...", opName)
 		oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
 		if err != nil {
 			return nil, err
 		}
+		if oldObject != nil {
+			removeStaleChunks(ctx, oldObject, nil)
+		}
 		return f.newObject("", oResult, nil), nil
 	}
 
@@ -1755,6 +1776,9 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		silentlyRemove(ctx, newObj)
 		return nil, err
 	}
+	if oldObject != nil {
+		removeStaleChunks(ctx, oldObject, newChunks)
+	}
 	return newObj, nil
 }
 

backend/chunker/chunker_internal_test.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	_ "github.com/rclone/rclone/backend/memory"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/fspath"
@@ -627,6 +628,22 @@ func testChunkNumberOverflow(t *testing.T, f *Fs) {
 	_ = file.Remove(ctx)
 }
 
+// testOrphanTempChunks checks that temporary chunks left behind by an
+// interrupted upload are not listed and don't break the listing
+func testOrphanTempChunks(t *testing.T, f *Fs) {
+	const dir = "orphan"
+	ctx := context.Background()
+	defer func() {
+		_ = operations.Purge(ctx, f.base, dir)
+	}()
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	item := fstest.Item{Path: f.makeChunkName(path.Join(dir, "interrupted"), 0, "", "abc123"), ModTime: modTime}
+	fstests.PutTestContents(ctx, t, f.base, &item, "data", true)
+
+	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
+}
+
 func testMetadataInput(t *testing.T, f *Fs) {
 	const minChunkForTest = 50
 	if f.opt.ChunkSize < minChunkForTest {
@@ -861,6 +878,104 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	_ = operations.Purge(ctx, f.base, dir)
 }
 
+func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
+	ctx := context.Background()
+	const dir = "serveroverwrite"
+	remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
+	fsResult, err := fs.NewFs(ctx, remote)
+	require.NoError(t, err)
+	copyFs, ok := fsResult.(*Fs)
+	require.True(t, ok)
+	oldMoveChunkSize := moveFs.opt.ChunkSize
+	moveFs.opt.ChunkSize = fs.SizeSuffix(3)
+	defer func() {
+		moveFs.opt.ChunkSize = oldMoveChunkSize
+		_ = operations.Purge(ctx, copyFs.base, "")
+		_ = operations.Purge(ctx, moveFs.base, dir)
+	}()
+
+	type operation struct {
+		name    string
+		fs      *Fs
+		enabled bool
+		do      copyMoveFn
+	}
+	operations := []operation{
+		{name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
+		{name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
+	}
+	testCases := []struct {
+		name           string
+		sourceContents string
+		targetContents string
+	}{
+		{name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
+		{name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
+	}
+	transactionModes := []struct {
+		name        string
+		useNoRename bool
+	}{
+		{name: "Rename"},
+		{name: "NoRename", useNoRename: true},
+	}
+
+	for _, operation := range operations {
+		t.Run(operation.name, func(t *testing.T) {
+			if !operation.enabled {
+				t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
+			}
+			for _, transactionMode := range transactionModes {
+				t.Run(transactionMode.name, func(t *testing.T) {
+					f := operation.fs
+					if transactionMode.useNoRename && !f.useMeta {
+						t.Skip("Can't test norename transactions without metadata")
+					}
+					oldUseNoRename := f.useNoRename
+					f.useNoRename = transactionMode.useNoRename
+					defer func() { f.useNoRename = oldUseNoRename }()
+					for _, testCase := range testCases {
+						t.Run(testCase.name, func(t *testing.T) {
+							prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
+							source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
+							target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
+							targetObject := target.(*Object)
+							require.NotEmpty(t, targetObject.chunks)
+
+							oldChunks := append([]fs.Object(nil), targetObject.chunks...)
+							result, err := operation.do(ctx, source, target.Remote())
+							require.NoError(t, err)
+
+							reader, err := result.Open(ctx)
+							require.NoError(t, err)
+							contents, err := io.ReadAll(reader)
+							require.NoError(t, err)
+							require.NoError(t, reader.Close())
+							assert.Equal(t, testCase.sourceContents, string(contents))
+
+							resultObject := result.(*Object)
+							currentChunks := make(map[string]struct{}, len(resultObject.chunks))
+							for _, chunk := range resultObject.chunks {
+								currentChunks[chunk.Remote()] = struct{}{}
+							}
+							removed := 0
+							for _, chunk := range oldChunks {
+								if _, found := currentChunks[chunk.Remote()]; found {
+									continue
+								}
+								_, err := f.base.NewObject(ctx, chunk.Remote())
+								assert.Equal(t, fs.ErrorObjectNotFound, err)
+								removed++
+							}
+							assert.Positive(t, removed)
+						})
+					}
+				})
+			}
+		})
+	}
+}
+
 // Test that md5all creates metadata even for small files
 func testMD5AllSlow(t *testing.T, f *Fs) {
 	ctx := context.Background()
@@ -928,9 +1043,15 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("BackwardsCompatibility", func(t *testing.T) {
 		testBackwardsCompatibility(t, f)
 	})
+	t.Run("OrphanTempChunks", func(t *testing.T) {
+		testOrphanTempChunks(t, f)
+	})
 	t.Run("ChunkerServerSideMove", func(t *testing.T) {
 		testChunkerServerSideMove(t, f)
 	})
+	t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
+		testChunkerServerSideOverwrite(t, f)
+	})
 	t.Run("MD5AllSlow", func(t *testing.T) {
 		testMD5AllSlow(t, f)
 	})

backend/crypt/cipher.go

@@ -118,8 +118,8 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-// fileNameEncoding are the encoding methods dealing with encrypted file names
-type fileNameEncoding interface {
+// FileNameEncoding is the interface for the encoding methods dealing with encrypted file names
+type FileNameEncoding interface {
 	EncodeToString(src []byte) string
 	DecodeString(s string) ([]byte, error)
 }
@@ -154,7 +154,7 @@ func (caseInsensitiveBase32Encoding) DecodeString(s string) ([]byte, error) {
 }
 
 // NewNameEncoding creates a NameEncoding from a string
-func NewNameEncoding(s string) (enc fileNameEncoding, err error) {
+func NewNameEncoding(s string) (enc FileNameEncoding, err error) {
 	s = strings.ToLower(s)
 	switch s {
 	case "base32":
@@ -176,7 +176,7 @@ type Cipher struct {
 	nameTweak       [nameCipherBlockSize]byte // used to tweak the name crypto
 	block           gocipher.Block
 	mode            NameEncryptionMode
-	fileNameEnc     fileNameEncoding
+	fileNameEnc     FileNameEncoding
 	buffers         sync.Pool // encrypt/decrypt buffers
 	cryptoRand      io.Reader // read crypto random numbers from here
 	dirNameEncrypt  bool
@@ -185,7 +185,7 @@ type Cipher struct {
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc fileNameEncoding) (*Cipher, error) {
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool, enc FileNameEncoding) (*Cipher, error) {
 	c := &Cipher{
 		mode:            mode,
 		fileNameEnc:     enc,

backend/crypt/crypt.go

@@ -1197,7 +1197,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash
-	if srcObj.Fs().Features().IsLocal {
+	if srcObj != nil && srcObj.Fs().Features().IsLocal {
 		// Read the data and encrypt it to calculate the hash
 		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
 		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)

backend/crypt/crypt_internal_test.go

@@ -93,6 +93,16 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
 }
 
+// Test ObjectInfo.Hash doesn't panic when the source is an
+// fs.OverrideRemote which wraps an fs.ObjectInfo that isn't an fs.Object
+func TestObjectInfoHashOverrideRemoteNotObject(t *testing.T) {
+	src := object.NewStaticObjectInfo("file", time.Now(), 3, true, nil, nil)
+	oi := (&Fs{}).newObjectInfo(fs.NewOverrideRemote(src, "new_remote"), nonce{})
+	gotHash, err := oi.Hash(context.Background(), hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, "", gotHash)
+}
+
 func testComputeHash(t *testing.T, f *Fs) {
 	var (
 		contents = random.String(100)

backend/dosya/api/types.go

@@ -0,0 +1,259 @@
+// Package api has type definitions for dosya.dev
+package api
+
+import "fmt"
+
+// Response is the base response wrapper
+type Response struct {
+	OK    bool   `json:"ok"`
+	Error string `json:"error,omitempty"`
+}
+
+// Error is an error response from the API.
+//
+// StatusCode and Status are filled in from the HTTP response. The API
+// sets ErrorCode on the errors a client is expected to act on rather
+// than just report, e.g. "concurrent_upload_limit".
+type Error struct {
+	StatusCode           int    `json:"-"`
+	Status               string `json:"-"`
+	OK                   bool   `json:"ok"`
+	Message              string `json:"error"`
+	ErrorCode            string `json:"error_code,omitempty"`
+	MaxConcurrentUploads int    `json:"max_concurrent_uploads,omitempty"`
+}
+
+// Error satisfies the error interface
+func (e *Error) Error() string {
+	out := fmt.Sprintf("HTTP error %d (%s)", e.StatusCode, e.Status)
+	if e.Message != "" {
+		out += ": " + e.Message
+	}
+	if e.ErrorCode != "" {
+		out += " (" + e.ErrorCode + ")"
+	}
+	return out
+}
+
+// FileItem represents a file from the dosya API
+type FileItem struct {
+	ID        string  `json:"id"`
+	Name      string  `json:"name"`
+	SizeBytes int64   `json:"size_bytes"`
+	MimeType  string  `json:"mime_type"`
+	Extension string  `json:"extension"`
+	Region    string  `json:"region"`
+	CreatedAt float64 `json:"created_at"`
+	UpdatedAt float64 `json:"updated_at"`
+	FolderID  *string `json:"folder_id"`
+	IsSynced  int     `json:"is_synced"`
+	// ContentHash is the lowercase-hex SHA-256 of the file's bytes, or "" when
+	// the server has none (e.g. a file uploaded as an R2 multipart, whose parts
+	// are never re-read to hash the whole object - the same limitation S3 has).
+	ContentHash string `json:"content_hash"`
+}
+
+// FolderItem represents a folder from the dosya API
+type FolderItem struct {
+	ID        string  `json:"id"`
+	Name      string  `json:"name"`
+	ParentID  *string `json:"parent_id"`
+	CreatedAt float64 `json:"created_at"`
+	FileCount int     `json:"file_count"`
+	IsSynced  int     `json:"is_synced"`
+}
+
+// ListResponse is the response from GET /api/files
+type ListResponse struct {
+	Response
+	Files      []FileItem   `json:"files"`
+	Folders    []FolderItem `json:"folders"`
+	Pagination Pagination   `json:"pagination"`
+}
+
+// Pagination describes which page of files a ListResponse holds
+type Pagination struct {
+	Page       int `json:"page"`
+	PerPage    int `json:"per_page"`
+	TotalFiles int `json:"total_files"`
+	TotalPages int `json:"total_pages"`
+}
+
+// CreateFolderRequest is the request body for POST /api/folders
+type CreateFolderRequest struct {
+	WorkspaceID string  `json:"workspace_id"`
+	ParentID    *string `json:"parent_id"`
+	Name        string  `json:"name"`
+}
+
+// CreateFolderResponse is the response from POST /api/folders
+type CreateFolderResponse struct {
+	Response
+	Folder struct {
+		ID        string  `json:"id"`
+		Name      string  `json:"name"`
+		ParentID  *string `json:"parent_id"`
+		Workspace string  `json:"workspace_id"`
+	} `json:"folder"`
+}
+
+// DeleteFolderResponse is the response from DELETE /api/folders/:id
+type DeleteFolderResponse struct {
+	Response
+	Permanent      bool `json:"permanent"`
+	Complete       bool `json:"complete"`
+	Remaining      int  `json:"remaining"`
+	FilesAffected  int  `json:"files_affected"`
+	FoldersRemoved int  `json:"folders_removed"`
+}
+
+// RenameFolderRequest is the request body for PUT /api/folders/:id/rename
+type RenameFolderRequest struct {
+	Name string `json:"name"`
+}
+
+// MoveFolderRequest is the request body for PUT /api/folders/:id/move
+type MoveFolderRequest struct {
+	ParentID *string `json:"parent_id"`
+	Name     string  `json:"name,omitempty"`
+}
+
+// MoveFolderResponse is the response from PUT /api/folders/:id/move
+type MoveFolderResponse struct {
+	Response
+	Name string `json:"name"`
+}
+
+// UploadInitRequest is the request body for POST /api/upload/init
+type UploadInitRequest struct {
+	WorkspaceID string  `json:"workspace_id"`
+	FileName    string  `json:"file_name"`
+	FileSize    int64   `json:"file_size"`
+	MimeType    string  `json:"mime_type"`
+	Region      string  `json:"region,omitempty"`
+	FolderID    *string `json:"folder_id"`
+	FileID      *string `json:"file_id,omitempty"`
+}
+
+// UploadInitResponse is the response from POST /api/upload/init
+type UploadInitResponse struct {
+	Response
+	SessionID   string `json:"session_id"`
+	UploadURL   string `json:"upload_url"`
+	WorkspaceID string `json:"workspace_id"`
+	FileName    string `json:"file_name"`
+	FileSize    int64  `json:"file_size"`
+	MimeType    string `json:"mime_type"`
+	Extension   string `json:"extension"`
+	Region      string `json:"region"`
+	Resumable   *struct {
+		PartSize      int64  `json:"part_size"`
+		TotalParts    int    `json:"total_parts"`
+		PartUploadURL string `json:"part_upload_url"`
+		CompleteURL   string `json:"complete_url"`
+		StatusURL     string `json:"status_url"`
+	} `json:"resumable"`
+}
+
+// UploadCompleteResponse is the response from PUT /api/upload/:sessionId or POST /api/upload/:sessionId/complete
+type UploadCompleteResponse struct {
+	Response
+	File struct {
+		ID        string  `json:"id"`
+		Name      string  `json:"name"`
+		SizeBytes int64   `json:"size_bytes"`
+		MimeType  string  `json:"mime_type"`
+		Extension string  `json:"extension"`
+		Region    string  `json:"region"`
+		Version   int     `json:"version"`
+		CreatedAt float64 `json:"created_at"`
+		// SHA-256 of the uploaded bytes for a single-shot (small-file) upload;
+		// "" for a multipart upload, which the server does not hash.
+		ContentHash string `json:"content_hash"`
+	} `json:"file"`
+}
+
+// UploadPartResponse is the response from PUT /api/upload/:sessionId/part/:partNumber
+type UploadPartResponse struct {
+	Response
+	PartNumber     int    `json:"part_number"`
+	ETag           string `json:"etag"`
+	BytesUploaded  int64  `json:"bytes_uploaded"`
+	PartsCompleted int    `json:"parts_completed"`
+	TotalParts     int    `json:"total_parts"`
+}
+
+// DeleteFileResponse is the response from DELETE /api/files/:id
+type DeleteFileResponse struct {
+	Response
+	Permanent bool `json:"permanent"`
+}
+
+// RenameFileRequest is the request body for PUT /api/files/:id/rename
+type RenameFileRequest struct {
+	Name string `json:"name"`
+}
+
+// RenameFileResponse is the response from PUT /api/files/:id/rename
+type RenameFileResponse struct {
+	Response
+	Name string `json:"name"`
+}
+
+// MoveFileRequest is the request body for PUT /api/files/:id/move
+type MoveFileRequest struct {
+	FolderID *string `json:"folder_id"`
+	Name     *string `json:"name,omitempty"`
+}
+
+// CopyFileRequest is the request body for POST /api/files/:id/copy
+type CopyFileRequest struct {
+	FolderID *string `json:"folder_id"`
+	Name     string  `json:"name,omitempty"`
+}
+
+// CopyFileResponse is the response from POST /api/files/:id/copy
+type CopyFileResponse struct {
+	Response
+	FileID string `json:"file_id"`
+	Name   string `json:"name"`
+}
+
+// ShareLinkRequest is the request body for POST /api/files/:id/share
+type ShareLinkRequest struct {
+	ExpiresInDays *int `json:"expires_in_days,omitempty"`
+}
+
+// ShareLinkResponse is the response from POST /api/files/:id/share
+type ShareLinkResponse struct {
+	Response
+	Link struct {
+		ID        string  `json:"id"`
+		Token     string  `json:"token"`
+		URL       string  `json:"url"`
+		ExpiresAt *int64  `json:"expires_at"`
+		CreatedAt float64 `json:"created_at"`
+	} `json:"link"`
+}
+
+// Workspace is one entry of the workspace list
+type Workspace struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+// WorkspaceListResponse is the response from GET /api/workspaces
+type WorkspaceListResponse struct {
+	Response
+	Workspaces []Workspace `json:"workspaces"`
+}
+
+// WorkspaceInfoResponse is the response from GET /api/workspaces/:id
+type WorkspaceInfoResponse struct {
+	Response
+	Storage struct {
+		Used  int64 `json:"used"`
+		Total int64 `json:"total"`
+		Free  int64 `json:"free"`
+	} `json:"storage"`
+}

backend/dosya/dosya.go

@@ -0,0 +1,772 @@
+// Package dosya provides an interface to the dosya.dev cloud storage system.
+package dosya
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+const (
+	rootID         = "root"
+	defaultBaseURL = "https://api.dosya.dev"
+	minSleep       = 100 * time.Millisecond
+	maxSleep       = 2 * time.Second
+	decayConstant  = 2
+	attackConstant = 0
+)
+
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "dosya",
+		Description: "dosya.dev",
+		NewFs:       NewFs,
+		Options: []fs.Option{{
+			Help:      "Your API Key, get it from https://dosya.dev/profile under \"API keys\".",
+			Name:      "api_key",
+			Sensitive: true,
+			Required:  true,
+		}, {
+			Name: "workspace_id",
+			Help: `Your workspace ID.
+
+Leave this blank to use the one workspace the API key can reach, which
+is what a key restricted to a single workspace does. If the key can
+reach several, rclone will refuse and list their IDs for you to choose
+from.`,
+		}, {
+			Help:     "API base URL.",
+			Name:     "api_url",
+			Default:  defaultBaseURL,
+			Advanced: true,
+		}, {
+			Name: "upload_concurrency",
+			Help: `Concurrency for multipart uploads.
+
+This is the number of parts of the same file that are uploaded
+concurrently. The first part is always uploaded on its own, because
+the server creates the upload when it receives it.
+
+Each part in flight is held in memory, so memory use grows with
+this value times the part size the server chooses.
+
+If you are uploading large files over high-speed links and these
+uploads do not fully utilize your bandwidth, then increasing this
+may help to speed up the transfers.`,
+			Default:  4,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeSingleQuote |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeLtGt |
+				encoder.EncodeCtl |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
+		}},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	APIKey            string               `config:"api_key"`
+	WorkspaceID       string               `config:"workspace_id"`
+	APIURL            string               `config:"api_url"`
+	UploadConcurrency int                  `config:"upload_concurrency"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents a remote dosya.dev filesystem
+type Fs struct {
+	name     string
+	root     string
+	opt      Options
+	features *fs.Features
+	rest     *rest.Client
+	pacer    *fs.Pacer
+	dirCache *dircache.DirCache
+}
+
+// Object represents a dosya.dev file
+type Object struct {
+	fs     *Fs
+	remote string
+	file   api.FileItem
+}
+
+// findWorkspace fills in the workspace ID when the config leaves it blank.
+//
+// The ID is not shown anywhere in the dosya.dev web interface, so requiring
+// it meant asking people to find it in a developer console. A key restricted
+// to one workspace can only mean that one, and an account with a single
+// workspace has only one possible answer; anything else genuinely has to be
+// chosen, so list the choices in the error.
+func (f *Fs) findWorkspace(ctx context.Context) error {
+	workspaces, err := f.listWorkspaces(ctx)
+	if err != nil {
+		return err
+	}
+	switch len(workspaces) {
+	case 0:
+		return errors.New("no workspaces found for this API key")
+	case 1:
+		f.opt.WorkspaceID = workspaces[0].ID
+		fs.Debugf(f, "Using workspace %q (%s)", workspaces[0].Name, workspaces[0].ID)
+		return nil
+	}
+	var choices strings.Builder
+	for _, w := range workspaces {
+		_, _ = fmt.Fprintf(&choices, "\n    %s (%s)", w.ID, w.Name)
+	}
+	return fmt.Errorf("this API key can reach %d workspaces, so workspace_id must be set to one of:%s",
+		len(workspaces), choices.String())
+}
+
+// FindLeaf finds a directory of name leaf in the folder with ID pathID
+func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
+	listing, err := f.listFilesAndFolders(ctx, pathID)
+	if err != nil {
+		return "", false, err
+	}
+	for _, folder := range listing.Folders {
+		if f.opt.Enc.ToStandardName(folder.Name) == leaf {
+			return folder.ID, true, nil
+		}
+	}
+	return "", false, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+	name := f.opt.Enc.FromStandardName(leaf)
+	resp, err := f.createFolder(ctx, name, pathID)
+	if err != nil {
+		return "", err
+	}
+	return resp.Folder.ID, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("dosya.dev root '%s'", f.root)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	// Modification times are stored and returned to the second (the API
+	// keeps them as Unix seconds, sent on upload via the source-mtime header).
+	return time.Second
+}
+
+// Hashes returns the supported hash types of the filesystem
+func (f *Fs) Hashes() hash.Set {
+	// The server computes and stores the SHA-256 of a single-shot upload's
+	// bytes; multipart uploads have none (see Object.Hash).
+	return hash.NewHashSet(hash.SHA256)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// NewFs makes a new Fs object from the path
+func NewFs(ctx context.Context, name string, root string, config configmap.Mapper) (fs.Fs, error) {
+	opt := new(Options)
+	err := configstruct.Set(config, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	root = strings.Trim(root, "/")
+
+	f := &Fs{
+		name:  name,
+		root:  root,
+		opt:   *opt,
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
+	}
+
+	// No DuplicateFiles: an upload that names no existing file but whose
+	// (workspace, folder, name) is already taken becomes a new VERSION of
+	// that file rather than a second row, so a name is unique within a
+	// folder and the same upload twice returns the same file id.
+	f.features = (&fs.Features{
+		CaseInsensitive:         false,
+		CanHaveEmptyDirectories: true,
+		ReadMimeType:            true,
+	}).Fill(ctx, f)
+
+	client := fshttp.NewClient(ctx)
+	f.rest = rest.NewClient(client).SetRoot(strings.TrimSuffix(opt.APIURL, "/")).SetErrorHandler(errorHandler)
+	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
+
+	if f.opt.WorkspaceID == "" {
+		err = f.findWorkspace(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	f.dirCache = dircache.New(root, rootID, f)
+
+	// Find the current root
+	err = f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		// Assume it is a file
+		newRoot, remote := dircache.SplitPath(root)
+		tempF := *f
+		tempF.dirCache = dircache.New(newRoot, rootID, &tempF)
+		tempF.root = newRoot
+		// Make new Fs which is the parent
+		err = tempF.dirCache.FindRoot(ctx, false)
+		if err != nil {
+			// No root so return old f
+			return f, nil
+		}
+		_, err := tempF.NewObject(ctx, remote)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound {
+				// File doesn't exist so return old f
+				return f, nil
+			}
+			return nil, err
+		}
+		f.features.Fill(ctx, &tempF)
+		f.dirCache = tempF.dirCache
+		f.root = tempF.root
+		return f, fs.ErrorIsFile
+	}
+	return f, nil
+}
+
+// List the objects and directories in dir into entries
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return nil, err
+	}
+
+	listing, err := f.listFilesAndFolders(ctx, directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, folder := range listing.Folders {
+		folderName := f.opt.Enc.ToStandardName(folder.Name)
+		fullPath := getRemote(dir, folderName)
+		modTime := time.Unix(int64(folder.CreatedAt), 0)
+		d := fs.NewDir(fullPath, modTime).SetID(folder.ID)
+		entries = append(entries, d)
+		f.dirCache.Put(fullPath, folder.ID)
+	}
+
+	for _, file := range listing.Files {
+		fileName := f.opt.Enc.ToStandardName(file.Name)
+		remote := getRemote(dir, fileName)
+		o := &Object{
+			fs:     f,
+			remote: remote,
+			file:   file,
+		}
+		entries = append(entries, o)
+	}
+
+	return entries, nil
+}
+
+// NewObject finds the Object at remote
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
+	if err != nil {
+		if err == fs.ErrorDirNotFound {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
+	}
+
+	listing, err := f.listFilesAndFolders(ctx, directoryID)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, file := range listing.Files {
+		if f.opt.Enc.ToStandardName(file.Name) == leaf {
+			return &Object{
+				fs:     f,
+				remote: remote,
+				file:   file,
+			}, nil
+		}
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
+// Put uploads a new file
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	existingObj, err := f.NewObject(ctx, src.Remote())
+	switch err {
+	case nil:
+		return existingObj, existingObj.Update(ctx, in, src, options...)
+	case fs.ErrorObjectNotFound:
+		return f.put(ctx, in, src, options...)
+	default:
+		return nil, err
+	}
+}
+
+// put uploads src as a new file
+func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	remote := src.Remote()
+
+	_, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	modTime := src.ModTime(ctx)
+	resp, err := f.uploadFile(ctx, in, remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file:   fileFromUpload(resp, modTime),
+	}, nil
+}
+
+// fileFromUpload builds the FileItem for the object an upload just created.
+// The single-shot upload response omits updated_at, so the modification time
+// is taken from what was sent (modTime) rather than defaulting to the create
+// time - it is the time the server stored via the source-mtime header. The
+// SHA-256 comes straight from the response (empty for a multipart upload).
+func fileFromUpload(resp *api.UploadCompleteResponse, modTime time.Time) api.FileItem {
+	updatedAt := resp.File.CreatedAt
+	if !modTime.IsZero() {
+		updatedAt = float64(modTime.Unix())
+	}
+	return api.FileItem{
+		ID:          resp.File.ID,
+		Name:        resp.File.Name,
+		SizeBytes:   resp.File.SizeBytes,
+		MimeType:    resp.File.MimeType,
+		Extension:   resp.File.Extension,
+		Region:      resp.File.Region,
+		CreatedAt:   resp.File.CreatedAt,
+		UpdatedAt:   updatedAt,
+		ContentHash: resp.File.ContentHash,
+	}
+}
+
+// Mkdir makes the directory (container, bucket)
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	_, err := f.dirCache.FindDir(ctx, dir, true)
+	return err
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+
+	// The server checks emptiness itself (?if_empty=1) and answers
+	// fs.ErrorDirectoryNotEmpty's 409 without a separate listing call.
+	err = f.removeFolder(ctx, directoryID, true)
+	if err != nil {
+		return err
+	}
+
+	f.dirCache.FlushDir(dir)
+	return nil
+}
+
+// Move src to this remote using server-side move operations
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	srcLeaf := f.opt.Enc.ToStandardName(srcObj.file.Name)
+	_, srcDirID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
+	if err != nil {
+		return nil, err
+	}
+
+	// Find destination directory
+	dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	needsMove := srcDirID != dstDirID
+	needsRename := srcLeaf != dstLeaf
+
+	// Moving and renaming in the same request (rather than a moveFile call
+	// followed by a separate renameFile call) matters when both are needed:
+	// a two-step move-then-rename can transiently place the file's CURRENT
+	// name into a destination folder that already holds an unrelated file
+	// with that name, even though the FINAL name is free - the server only
+	// allows one active name per folder, so that intermediate state 409s.
+	if needsMove || needsRename {
+		newName := ""
+		if needsRename {
+			newName = f.opt.Enc.FromStandardName(dstLeaf)
+		}
+		err = f.moveFile(ctx, srcObj.file.ID, dstDirID, newName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Return new object. file.Name must reflect the destination leaf, not
+	// the source's - a caller that reuses this object as the src of a
+	// SECOND Move (e.g. operations.RemoveExisting's restore-on-error path)
+	// recomputes srcLeaf from file.Name; a stale name there makes that
+	// second move see srcLeaf == dstLeaf and skip the rename entirely,
+	// silently leaving the object under the wrong name.
+	newFile := srcObj.file
+	newFile.Name = f.opt.Enc.FromStandardName(dstLeaf)
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file:   newFile,
+	}, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	srcID, srcDirectoryID, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+	if err != nil {
+		return err
+	}
+
+	srcLeaf := srcRemote
+	if idx := strings.LastIndex(srcRemote, "/"); idx >= 0 {
+		srcLeaf = srcRemote[idx+1:]
+	}
+	needsRename := srcLeaf != dstLeaf
+	newName := ""
+	if needsRename {
+		newName = f.opt.Enc.FromStandardName(dstLeaf)
+	}
+
+	// Move folder to new parent only if parent changed. The new name goes
+	// in the same request, as for files in Move: a move followed by a
+	// separate rename can collide with an unrelated folder sharing the
+	// CURRENT name in the destination even though the final name is free.
+	if srcDirectoryID != dstDirectoryID {
+		renamed, err := f.moveFolder(ctx, srcID, dstDirectoryID, newName)
+		if err != nil {
+			return fmt.Errorf("couldn't move directory: %w", err)
+		}
+		if renamed {
+			needsRename = false
+		}
+	}
+
+	if needsRename {
+		err = f.renameFolder(ctx, srcID, newName)
+		if err != nil {
+			return fmt.Errorf("couldn't rename directory: %w", err)
+		}
+	}
+
+	srcFs.dirCache.FlushDir(srcRemote)
+	return nil
+}
+
+// Copy src to this remote using server-side copy operations
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+
+	dstLeaf, dstDirID, err := f.dirCache.FindPath(ctx, remote, true)
+	if err != nil {
+		return nil, err
+	}
+
+	// The copy API creates a new row named after the source file when
+	// copying across folders, which collides with one already at the
+	// destination. Copy under a temporary name, then delete the old
+	// destination, then move the copy onto the name - so a copy that fails
+	// leaves the destination as it was. Deleting it up front lost it
+	// (prior art: backend/box).
+	existingObj, err := f.NewObject(ctx, remote)
+	switch err {
+	case nil:
+		tempRemote := remote + "-rclone-copy-" + random.String(8)
+		fs.Debugf(remote, "dst already exists, copying to temp name %v", tempRemote)
+		tempObj, err := f.Copy(ctx, src, tempRemote)
+		if err != nil {
+			return nil, err
+		}
+		fs.Debugf(tempRemote, "moving to real name %v", remote)
+		if err := existingObj.Remove(ctx); err != nil {
+			return nil, fmt.Errorf("couldn't remove existing destination file: %w", err)
+		}
+		return f.Move(ctx, tempObj, remote)
+	case fs.ErrorObjectNotFound:
+		// nothing at the destination, so copy straight onto the name
+	default:
+		return nil, err
+	}
+
+	dstName := f.opt.Enc.FromStandardName(dstLeaf)
+
+	resp, err := f.copyFile(ctx, srcObj.file.ID, dstDirID, dstName)
+	if err != nil {
+		return nil, fmt.Errorf("couldn't copy file: %w", err)
+	}
+
+	// Servers that predate the name field ignore it and name the copy after
+	// the source, or "Copy of <original>" in the same folder. Rename to the
+	// requested destination name if it differs.
+	if resp.Name != dstName {
+		err = f.renameFile(ctx, resp.FileID, dstName)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't rename copied file: %w", err)
+		}
+		resp.Name = dstName
+	}
+
+	// A server-side copy is byte-identical to the source and the API
+	// preserves the source's modification time and content hash onto the
+	// copy, so the returned object must report the SOURCE's modtime and hash,
+	// not the copy's wall-clock time. Reading "now" here made the object
+	// rclone holds in memory disagree with the one read back from the remote
+	// (its Fingerprint check compares the two exactly).
+	return &Object{
+		fs:     f,
+		remote: remote,
+		file: api.FileItem{
+			ID:          resp.FileID,
+			Name:        resp.Name,
+			SizeBytes:   srcObj.file.SizeBytes,
+			MimeType:    srcObj.file.MimeType,
+			Extension:   srcObj.file.Extension,
+			Region:      srcObj.file.Region,
+			CreatedAt:   srcObj.file.CreatedAt,
+			UpdatedAt:   srcObj.file.UpdatedAt,
+			ContentHash: srcObj.file.ContentHash,
+		},
+	}, nil
+}
+
+// Purge deletes all the files and the container
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+
+	// DELETE /api/folders/:id?permanent=1 recursively deletes all contents
+	err = f.removeFolder(ctx, directoryID, false)
+	if err != nil {
+		return err
+	}
+
+	f.dirCache.FlushDir(dir)
+	return nil
+}
+
+// PublicLink generates a public link to the remote path
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	o, err := f.NewObject(ctx, remote)
+	if err != nil {
+		return "", err
+	}
+	obj := o.(*Object)
+
+	resp, err := f.createShareLink(ctx, obj.file.ID, expire)
+	if err != nil {
+		return "", err
+	}
+
+	return resp.Link.URL, nil
+}
+
+// DirCacheFlush resets the directory cache - used in testing as an
+// optional interface
+func (f *Fs) DirCacheFlush() {
+	f.dirCache.ResetRoot()
+}
+
+// About gets quota information from the Fs
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+	info, err := f.getWorkspaceInfo(ctx)
+	if err != nil {
+		return nil, err
+	}
+	usage := &fs.Usage{
+		Used:  fs.NewUsageValue(info.Storage.Used),
+		Total: fs.NewUsageValue(info.Storage.Total),
+		Free:  fs.NewUsageValue(info.Storage.Free),
+	}
+	return usage, nil
+}
+
+// ------------------------------------------------------------
+// Object methods
+// ------------------------------------------------------------
+
+// String returns a description of the Object
+func (o *Object) String() string {
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification date of the file
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	if o.file.UpdatedAt > 0 {
+		return time.Unix(int64(o.file.UpdatedAt), 0)
+	}
+	return time.Unix(int64(o.file.CreatedAt), 0)
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return o.file.SizeBytes
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns the selected checksum of the file
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.SHA256 {
+		return "", hash.ErrUnsupported
+	}
+	// May be "" - a multipart-uploaded object carries no server-side hash,
+	// which rclone treats as "hash unavailable for this object" (as it does
+	// for S3 multipart objects).
+	return o.file.ContentHash, nil
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+// Open opens the file for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	fs.FixRangeOption(options, o.file.SizeBytes)
+	return o.fs.downloadFile(ctx, o.file.ID, options)
+}
+
+// Update replaces the file content
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	// Find the directory ID for this file
+	_, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
+	if err != nil {
+		return err
+	}
+
+	// Upload as new version of existing file
+	fileID := o.file.ID
+	modTime := src.ModTime(ctx)
+	resp, err := o.fs.uploadFile(ctx, in, o.remote, src.Size(), modTime, fs.MimeType(ctx, src), directoryID, &fileID)
+	if err != nil {
+		return err
+	}
+
+	// Update object metadata
+	o.file = fileFromUpload(resp, modTime)
+
+	return nil
+}
+
+// Remove removes this object
+func (o *Object) Remove(ctx context.Context) error {
+	return o.fs.deleteFile(ctx, o.file.ID)
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.file.MimeType
+}
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.file.ID
+}
+
+func getRemote(dir, fileName string) string {
+	if dir == "" {
+		return fileName
+	}
+	return dir + "/" + fileName
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs              = (*Fs)(nil)
+	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Purger          = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
+	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ dircache.DirCacher = (*Fs)(nil)
+	_ fs.Object          = (*Object)(nil)
+	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
+)

backend/dosya/dosya_api.go

@@ -0,0 +1,880 @@
+package dosya
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
+)
+
+// sourceMTimeHeader is the request header the upload endpoints read to set a
+// file's modification time (Unix seconds); see lib/upload/source-timestamps.ts.
+const sourceMTimeHeader = "x-dosya-source-mtime"
+
+// modTimeHeaders returns the ExtraHeaders that carry a source modification time
+// on an upload, or nil when there is nothing plausible to send. The server
+// clamps implausible values itself, but a zero time is never worth a header.
+func modTimeHeaders(modTime time.Time) map[string]string {
+	if modTime.IsZero() {
+		return nil
+	}
+	return map[string]string{sourceMTimeHeader: strconv.FormatInt(modTime.Unix(), 10)}
+}
+
+// workspaceHintHeader names the workspace a by-id request is about. The API
+// maps an id to its workspace through an index the workspace fills about a
+// second after a write, so an id this client was handed moments ago can still
+// miss; the hint lets the server look for it in the named workspace instead.
+// It only locates: every route still checks membership against the workspace
+// it resolved, and a workspace-pinned key's hint is ignored unless it matches.
+const workspaceHintHeader = "X-Dosya-Workspace"
+
+// byIDHeaders returns the ExtraHeaders every /api/files/:id and
+// /api/folders/:id call carries, or nil when no workspace is configured.
+func (f *Fs) byIDHeaders() map[string]string {
+	if f.opt.WorkspaceID == "" {
+		return nil
+	}
+	return map[string]string{workspaceHintHeader: f.opt.WorkspaceID}
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	429, // Too Many Requests
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+}
+
+// shouldRetry returns a boolean as to whether this resp and err
+// deserve to be retried
+func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	retry := fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes)
+	if retry {
+		if d, ok := retryAfter(resp); ok {
+			// The API answers 503 + Retry-After while a workspace is being
+			// moved between storage homes, and Cloudflare sends it with 429.
+			// The pacer's own backoff tops out at maxSleep, far short of that.
+			fs.Debugf(nil, "Server asked to retry after %v", d)
+			err = pacer.RetryAfterError(err, d)
+		}
+	}
+	return retry, err
+}
+
+// maxRetryAfter caps how long a single Retry-After is honoured for
+const maxRetryAfter = 5 * time.Minute
+
+// retryAfter reads a 429 or 503 response's Retry-After header, which the API
+// sends as whole seconds
+func retryAfter(resp *http.Response) (time.Duration, bool) {
+	if resp == nil || (resp.StatusCode != http.StatusTooManyRequests && resp.StatusCode != http.StatusServiceUnavailable) {
+		return 0, false
+	}
+	seconds, err := strconv.Atoi(strings.TrimSpace(resp.Header.Get("Retry-After")))
+	if err != nil || seconds <= 0 {
+		return 0, false
+	}
+	return min(time.Duration(seconds)*time.Second, maxRetryAfter), true
+}
+
+// errorHandler decodes a non-2xx response into an *api.Error so callers can
+// act on the machine-readable error_code rather than the message text.
+func errorHandler(resp *http.Response) error {
+	body, err := rest.ReadBody(resp)
+	if err != nil {
+		return fmt.Errorf("error reading error out of body: %w", err)
+	}
+	apiErr := &api.Error{
+		StatusCode: resp.StatusCode,
+		Status:     resp.Status,
+	}
+	if json.Unmarshal(body, apiErr) != nil || apiErr.Message == "" {
+		// Not the API's JSON shape (e.g. an edge response that never
+		// reached the API) - keep the body text so it isn't lost.
+		apiErr.ErrorCode = ""
+		apiErr.Message = strings.TrimSpace(string(body))
+	}
+	return apiErr
+}
+
+// The workspace caps the number of in-flight uploads per user. The API
+// refuses an upload/init past that cap with a 400 carrying
+// error_code "concurrent_upload_limit", and documents it as an error to
+// wait on rather than give up on: slots free as uploads finish, and the
+// server marks sessions idle for 10 minutes as failed, so waiting up to
+// that long guarantees progress unless the workspace really is saturated.
+// A 400 is deliberately outside the pacer's retry codes, so the wait
+// lives in initUpload instead of shouldRetry.
+var (
+	concurrentUploadRetryDelay = 5 * time.Second
+	concurrentUploadWaitMax    = 10 * time.Minute
+)
+
+// abortUploadTimeout bounds the abort of a session the client is giving up on,
+// which runs outside the transfer's context so a cancelled transfer still
+// sends it
+const abortUploadTimeout = 10 * time.Second
+
+// isConcurrentUploadLimit reports whether err is the API refusing an
+// upload because the workspace's concurrent upload limit is reached
+func isConcurrentUploadLimit(err error) bool {
+	var apiErr *api.Error
+	return errors.As(err, &apiErr) && apiErr.ErrorCode == "concurrent_upload_limit"
+}
+
+// listPageSize is the largest page GET /api/files will serve
+const listPageSize = 500
+
+// listFilesAndFolders lists files and folders in a directory
+//
+// The API pages files (at most listPageSize per page) but returns every
+// folder on each page, so the files from all pages are gathered and the
+// folders are taken from the first page only. Pages are ordered by name,
+// which the API tie-breaks on id, so offsets are stable between requests.
+func (f *Fs) listFilesAndFolders(ctx context.Context, folderID string) (*api.ListResponse, error) {
+	var listing *api.ListResponse
+	for page := 1; ; page++ {
+		opts := rest.Opts{
+			Method: "GET",
+			Path:   "/api/files",
+			Parameters: map[string][]string{
+				"workspace_id": {f.opt.WorkspaceID},
+				"per_page":     {strconv.Itoa(listPageSize)},
+				"page":         {strconv.Itoa(page)},
+				"sort":         {"name"},
+				"dir":          {"asc"},
+			},
+		}
+		if folderID != "" && folderID != rootID {
+			opts.Parameters["folder_id"] = []string{folderID}
+		}
+
+		var result api.ListResponse
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("couldn't list files: %w", err)
+		}
+		if !result.OK {
+			return nil, fmt.Errorf("API error: %s", result.Error)
+		}
+		if listing == nil {
+			listing = &result
+		} else {
+			listing.Files = append(listing.Files, result.Files...)
+		}
+		if page >= result.Pagination.TotalPages || len(result.Files) == 0 {
+			return listing, nil
+		}
+	}
+}
+
+// createFolder creates a folder
+func (f *Fs) createFolder(ctx context.Context, name string, parentID string) (*api.CreateFolderResponse, error) {
+	request := api.CreateFolderRequest{
+		WorkspaceID: f.opt.WorkspaceID,
+		Name:        name,
+	}
+	if parentID != "" && parentID != rootID {
+		request.ParentID = &parentID
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/api/folders",
+	}
+
+	var result api.CreateFolderResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create folder: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// removeFolder deletes a folder
+//
+// ?permanent=1 trashes and purges a live folder in one request on today's
+// API. ifEmpty also sends ?if_empty=1, which asks the server to refuse with
+// HTTP 409 "Folder is not empty" instead of purging a folder that still
+// holds a live file or a live child folder; that 409 is turned into
+// fs.ErrorDirectoryNotEmpty here rather than treated as a generic API
+// error, and it is never retried (409 is not one of retryErrorCodes). A big
+// subtree is purged in bounded passes (202, complete=false), so keep
+// calling until the purge is complete. An older server that ignores
+// permanent=1 only trashes on the first call (answering permanent=false),
+// so the loop's second pass sends the plain DELETE that purges it.
+func (f *Fs) removeFolder(ctx context.Context, folderID string, ifEmpty bool) error {
+	parameters := map[string][]string{"permanent": {"1"}}
+	if ifEmpty {
+		parameters["if_empty"] = []string{"1"}
+	}
+	opts := rest.Opts{
+		Method:       "DELETE",
+		Path:         "/api/folders/" + folderID,
+		Parameters:   parameters,
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	for pass := 0; ; pass++ {
+		var result api.DeleteFolderResponse
+		var statusCode int
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+			if resp != nil {
+				statusCode = resp.StatusCode
+			}
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			if ifEmpty && statusCode == http.StatusConflict {
+				return fs.ErrorDirectoryNotEmpty
+			}
+			if pass == 0 {
+				return fmt.Errorf("couldn't remove folder: %w", err)
+			}
+			return fmt.Errorf("couldn't permanently remove folder: %w", err)
+		}
+		if !result.OK {
+			return fmt.Errorf("API error: %s", result.Error)
+		}
+		if result.Permanent && result.Complete {
+			return nil
+		}
+		if pass >= maxFolderPurgePasses {
+			return fmt.Errorf("couldn't permanently remove folder: purge still incomplete after %d passes", pass)
+		}
+	}
+}
+
+// maxFolderPurgePasses bounds the DELETE calls spent purging one folder
+const maxFolderPurgePasses = 1000
+
+// renameFolder renames a folder
+func (f *Fs) renameFolder(ctx context.Context, folderID string, newName string) error {
+	request := api.RenameFolderRequest{Name: newName}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/folders/" + folderID + "/rename",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.Response
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't rename folder: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+	return nil
+}
+
+// moveFolder moves a folder to a new parent
+func (f *Fs) moveFolder(ctx context.Context, folderID string, newParentID string, newName string) (renamed bool, err error) {
+	request := api.MoveFolderRequest{Name: newName}
+	if newParentID != "" && newParentID != rootID {
+		request.ParentID = &newParentID
+	}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/folders/" + folderID + "/move",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.MoveFolderResponse
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return false, fmt.Errorf("couldn't move folder: %w", err)
+	}
+	if !result.OK {
+		return false, fmt.Errorf("API error: %s", result.Error)
+	}
+	// Servers that predate the name field ignore it and answer without a
+	// name, leaving the caller to rename separately
+	return newName != "" && result.Name == newName, nil
+}
+
+// deleteFile deletes a file
+//
+// ?permanent=1 trashes and purges it in one request on today's API. An
+// older server that ignores the flag only soft-deletes it, answering
+// permanent=false, so a second DELETE on the now soft-deleted file purges
+// it from the database and R2, as it always has.
+func (f *Fs) deleteFile(ctx context.Context, fileID string) error {
+	opts := rest.Opts{
+		Method:       "DELETE",
+		Path:         "/api/files/" + fileID,
+		Parameters:   map[string][]string{"permanent": {"1"}},
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.DeleteFileResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't delete file: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+
+	// Answered into its own value so a field the second response omits
+	// can't be read off the first.
+	if !result.Permanent {
+		var permanent api.DeleteFileResponse
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, nil, &permanent)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return fmt.Errorf("couldn't permanently delete file: %w", err)
+		}
+		if !permanent.OK {
+			return fmt.Errorf("API error: %s", permanent.Error)
+		}
+	}
+	return nil
+}
+
+// renameFile renames a file
+func (f *Fs) renameFile(ctx context.Context, fileID string, newName string) error {
+	request := api.RenameFileRequest{Name: newName}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/files/" + fileID + "/rename",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.RenameFileResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't rename file: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+	return nil
+}
+
+// moveFile moves a file to a new folder and, if newName is non-empty,
+// renames it in the same request
+func (f *Fs) moveFile(ctx context.Context, fileID string, folderID string, newName string) error {
+	request := api.MoveFileRequest{}
+	if folderID != "" && folderID != rootID {
+		request.FolderID = &folderID
+	}
+	if newName != "" {
+		request.Name = &newName
+	}
+	opts := rest.Opts{
+		Method:       "PUT",
+		Path:         "/api/files/" + fileID + "/move",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.Response
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return fmt.Errorf("couldn't move file: %w", err)
+	}
+	if !result.OK {
+		return fmt.Errorf("API error: %s", result.Error)
+	}
+	return nil
+}
+
+// copyFile copies a file to a folder
+func (f *Fs) copyFile(ctx context.Context, fileID string, folderID string, name string) (*api.CopyFileResponse, error) {
+	request := api.CopyFileRequest{Name: name}
+	if folderID != "" && folderID != rootID {
+		request.FolderID = &folderID
+	}
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         "/api/files/" + fileID + "/copy",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.CopyFileResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't copy file: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// downloadFile downloads a file by its ID
+//
+// The download endpoint answers 302 with a presigned R2 URL in Location, so
+// this is two calls. Both go through the rest client, which is built on
+// fshttp, so --timeout, --contimeout, --ca-cert, --no-check-certificate,
+// --user-agent, --bind, --dump and the proxy settings apply to downloads as
+// they do to every other call, and errorHandler and shouldRetry decide the
+// errors and the retries uniformly.
+func (f *Fs) downloadFile(ctx context.Context, fileID string, options []fs.OpenOption) (io.ReadCloser, error) {
+	// NoRedirect stops the client following the 302 with the API's
+	// Authorization header attached. A 302 is not a 2xx, so Call reports it
+	// as an error (having read and closed the body); the Location header
+	// is what actually matters, and finding it clears the error, as in
+	// backend/linkbox.
+	opts := rest.Opts{
+		Method:       "GET",
+		Path:         "/api/files/" + fileID + "/download",
+		ExtraHeaders: f.byIDHeaders(),
+		NoRedirect:   true,
+	}
+	var downloadURL string
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.Call(ctx, &opts)
+		if resp != nil {
+			if location := resp.Header.Get("Location"); location != "" {
+				downloadURL, err = location, nil
+			}
+		}
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get download URL: %w", err)
+	}
+	if downloadURL == "" {
+		return nil, errors.New("no download URL in redirect response")
+	}
+
+	// Fetch the object from the presigned URL. It carries its own
+	// credentials and R2 refuses a request that also has an Authorization
+	// header, so the client-level bearer token is removed by setting the
+	// header empty: the rest client skips a header with an empty value.
+	opts = rest.Opts{
+		Method:       "GET",
+		RootURL:      downloadURL,
+		Options:      options,
+		ExtraHeaders: map[string]string{"Authorization": ""},
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		resp, err = f.rest.Call(ctx, &opts)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't download file: %w", err)
+	}
+	return resp.Body, nil
+}
+
+// uploadSmallFile uploads a file smaller than 10MB in a single request
+func (f *Fs) uploadSmallFile(ctx context.Context, in io.Reader, sessionID string, size int64, modTime time.Time) (*api.UploadCompleteResponse, error) {
+	opts := rest.Opts{
+		Method:        "PUT",
+		Path:          "/api/upload/" + sessionID,
+		Body:          in,
+		ContentLength: &size,
+		ContentType:   "application/octet-stream",
+		ExtraHeaders:  modTimeHeaders(modTime),
+	}
+
+	// The body is a stream which can't be rewound, so a low level retry
+	// would resend whatever is left of it - nothing - and the server
+	// would store that as an empty file. Don't retry here; rclone's high
+	// level retries redo the whole transfer instead.
+	var result api.UploadCompleteResponse
+	err := f.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't upload file: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// uploadPart uploads a single part of a multipart upload
+func (f *Fs) uploadPart(ctx context.Context, part []byte, sessionID string, partNumber int) (*api.UploadPartResponse, error) {
+	size := int64(len(part))
+	opts := rest.Opts{
+		Method:        "PUT",
+		Path:          "/api/upload/" + sessionID + "/part/" + strconv.Itoa(partNumber),
+		ContentLength: &size,
+		ContentType:   "application/octet-stream",
+	}
+
+	var result api.UploadPartResponse
+	err := f.pacer.Call(func() (bool, error) {
+		// A fresh reader per attempt so a retry resends the whole part
+		opts.Body = bytes.NewReader(part)
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't upload part %d: %w", partNumber, err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// completeUpload finalizes a multipart upload
+func (f *Fs) completeUpload(ctx context.Context, sessionID string, modTime time.Time) (*api.UploadCompleteResponse, error) {
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         "/api/upload/" + sessionID + "/complete",
+		ExtraHeaders: modTimeHeaders(modTime),
+	}
+
+	var result api.UploadCompleteResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't complete upload: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// initUpload initializes a new upload session
+func (f *Fs) initUpload(ctx context.Context, fileName string, fileSize int64, mimeType string, folderID string, fileID *string) (*api.UploadInitResponse, error) {
+	request := api.UploadInitRequest{
+		WorkspaceID: f.opt.WorkspaceID,
+		FileName:    fileName,
+		FileSize:    fileSize,
+		MimeType:    mimeType,
+		FileID:      fileID,
+	}
+	if folderID != "" && folderID != rootID {
+		request.FolderID = &folderID
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/api/upload/init",
+	}
+
+	var result api.UploadInitResponse
+	deadline := time.Now().Add(concurrentUploadWaitMax)
+	for {
+		result = api.UploadInitResponse{}
+		err := f.pacer.Call(func() (bool, error) {
+			resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err == nil {
+			break
+		}
+		if !isConcurrentUploadLimit(err) || time.Now().After(deadline) {
+			return nil, fmt.Errorf("couldn't init upload: %w", err)
+		}
+		fs.Debugf(f, "Upload slots in use, waiting %v before retrying %q: %v", concurrentUploadRetryDelay, fileName, err)
+		select {
+		case <-ctx.Done():
+			return nil, fmt.Errorf("couldn't init upload: %w", ctx.Err())
+		case <-time.After(concurrentUploadRetryDelay):
+		}
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// uploadParts reads the source part by part and uploads the parts.
+//
+// Part 1 goes on its own: the server creates the R2 multipart upload (and a
+// new file's id and key) when it receives the first part, so parts sent
+// before that has finished would each start an upload of their own. The
+// rest go out opt.UploadConcurrency at a time. A part is only read from the
+// source once a buffer is free, so at most that many parts are held in
+// memory, and a short source is still found by the reader before the part
+// it could not fill is sent.
+func (f *Fs) uploadParts(ctx context.Context, in *readers.CountingReader, sessionID string, size, partSize int64, totalParts int) error {
+	readPart := func(buf []byte, partNum int) ([]byte, error) {
+		currentPartSize := partSize
+		if partNum == totalParts {
+			currentPartSize = size - partSize*int64(totalParts-1)
+		}
+		part := buf[:currentPartSize]
+		if _, err := io.ReadFull(in, part); err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, in.BytesRead(), io.ErrUnexpectedEOF)
+			}
+			return nil, fmt.Errorf("failed to read part %d: %w", partNum, err)
+		}
+		return part, nil
+	}
+
+	concurrency := max(f.opt.UploadConcurrency, 1)
+	buffers := make(chan []byte, concurrency)
+	buffers <- make([]byte, partSize)
+	allocated := 1
+
+	first := <-buffers
+	part, err := readPart(first, 1)
+	if err != nil {
+		return err
+	}
+	if _, err := f.uploadPart(ctx, part, sessionID, 1); err != nil {
+		return err
+	}
+	buffers <- first
+
+	// A failed read has to take the parts already in flight down with it,
+	// or they upload to completion while the error waits on Wait.
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	g, gCtx := errgroup.WithContext(ctx)
+	var readErr error
+	for partNum := 2; partNum <= totalParts; partNum++ {
+		// Take a free buffer, allocating one while under the limit.
+		var buf []byte
+		select {
+		case buf = <-buffers:
+		default:
+			if allocated < concurrency {
+				buf = make([]byte, partSize)
+				allocated++
+			} else {
+				select {
+				case buf = <-buffers:
+				case <-gCtx.Done():
+				}
+			}
+		}
+		if buf == nil {
+			break // a part failed; its error comes back from Wait
+		}
+		part, err := readPart(buf, partNum)
+		if err != nil {
+			readErr = err
+			cancel()
+			break
+		}
+		g.Go(func() error {
+			defer func() { buffers <- buf }()
+			_, err := f.uploadPart(gCtx, part, sessionID, partNum)
+			return err
+		})
+	}
+	err = g.Wait()
+	if readErr != nil {
+		return readErr
+	}
+	return err
+}
+
+// abortUpload retires an upload session the client is giving up on.
+//
+// Best effort: the server reclaims an abandoned session itself, so a failure
+// here - including an older server with no such route - costs the slot the
+// wait it would have cost anyway and is not worth failing an upload over.
+// Sent on a context of its own so a cancelled transfer still gives its slot
+// back rather than leaving it to the sweep.
+func (f *Fs) abortUpload(ctx context.Context, sessionID string) {
+	ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), abortUploadTimeout)
+	defer cancel()
+	opts := rest.Opts{
+		Method:     "DELETE",
+		Path:       "/api/upload/" + sessionID,
+		NoResponse: true,
+	}
+	err := f.pacer.CallNoRetry(func() (bool, error) {
+		resp, err := f.rest.Call(ctx, &opts)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		fs.Debugf(f, "Couldn't abort upload session %q: %v", sessionID, err)
+	}
+}
+
+// uploadFile handles the full upload flow (small or multipart)
+func (f *Fs) uploadFile(ctx context.Context, in io.Reader, remote string, size int64, modTime time.Time, mimeType string, folderID string, fileID *string) (resp *api.UploadCompleteResponse, err error) {
+	// Both refusals live here so a new file and a new version of an existing
+	// one answer the same way: upload/init needs a byte count up front, and
+	// the server stores no zero-length object.
+	if size < 0 {
+		return nil, errors.New("can't upload files with unknown size")
+	}
+	if size == 0 {
+		return nil, fs.ErrorCantUploadEmptyFiles
+	}
+
+	leaf := f.opt.Enc.FromStandardName(path.Base(remote))
+
+	initResp, err := f.initUpload(ctx, leaf, size, mimeType, folderID, fileID)
+	if err != nil {
+		return nil, err
+	}
+	// A session abandoned here holds one of the workspace's upload slots
+	// until the server's sweep retires it ten minutes later, and rclone's
+	// high-level retry opens a fresh session rather than resuming this one -
+	// so without this, a few failed attempts use up every slot.
+	defer func() {
+		if err != nil {
+			f.abortUpload(ctx, initResp.SessionID)
+		}
+	}()
+
+	// Count what the source actually supplies, so a source which ends
+	// early is reported as an error rather than stored as a truncated
+	// file.
+	counter := readers.NewCountingReader(in)
+
+	// Small file upload (single PUT, no resumable)
+	if initResp.Resumable == nil {
+		resp, err = f.uploadSmallFile(ctx, counter, initResp.SessionID, size, modTime)
+		if err != nil {
+			return nil, err
+		}
+		if got := int64(counter.BytesRead()); got != size {
+			return nil, fmt.Errorf("expected %d bytes in input, but got %d: %w", size, got, io.ErrUnexpectedEOF)
+		}
+		return resp, nil
+	}
+
+	// Multipart upload. Each part is read fully into memory before it is
+	// sent, so a short source is caught before the part goes out and a
+	// part which has to be retried is resent with all of its bytes.
+	partSize := initResp.Resumable.PartSize
+	totalParts := initResp.Resumable.TotalParts
+	// Both bounds matter. partSize*(totalParts-1) >= size means a part would
+	// have nothing to carry; size > partSize*totalParts means the last part
+	// is bigger than partSize, and readPart would slice past its buffer and
+	// panic.
+	if partSize <= 0 || totalParts <= 0 || partSize*int64(totalParts-1) >= size || size > partSize*int64(totalParts) {
+		return nil, fmt.Errorf("invalid multipart layout from server: %d parts of %d bytes for %d bytes", totalParts, partSize, size)
+	}
+	if err = f.uploadParts(ctx, counter, initResp.SessionID, size, partSize, totalParts); err != nil {
+		return nil, err
+	}
+
+	return f.completeUpload(ctx, initResp.SessionID, modTime)
+}
+
+// createShareLink creates a public share link for a file
+func (f *Fs) createShareLink(ctx context.Context, fileID string, expire fs.Duration) (*api.ShareLinkResponse, error) {
+	request := api.ShareLinkRequest{}
+	if expire < fs.DurationOff {
+		days := int(time.Duration(expire).Hours() / 24)
+		if days < 1 {
+			days = 1
+		}
+		request.ExpiresInDays = &days
+	}
+
+	opts := rest.Opts{
+		Method:       "POST",
+		Path:         "/api/files/" + fileID + "/share",
+		ExtraHeaders: f.byIDHeaders(),
+	}
+
+	var result api.ShareLinkResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't create share link: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}
+
+// getWorkspaceInfo fetches workspace storage info
+// listWorkspaces lists the workspaces this credential can reach. An API key
+// pinned to a workspace sees exactly one: its own.
+func (f *Fs) listWorkspaces(ctx context.Context) ([]api.Workspace, error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/api/workspaces",
+	}
+
+	var result api.WorkspaceListResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't list workspaces: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return result.Workspaces, nil
+}
+
+func (f *Fs) getWorkspaceInfo(ctx context.Context) (*api.WorkspaceInfoResponse, error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/api/workspaces/" + f.opt.WorkspaceID,
+	}
+
+	var result api.WorkspaceInfoResponse
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't get workspace info: %w", err)
+	}
+	if !result.OK {
+		return nil, fmt.Errorf("API error: %s", result.Error)
+	}
+	return &result, nil
+}

backend/dosya/dosya_compat_test.go

@@ -0,0 +1,235 @@
+package dosya
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestShouldRetryHonoursRetryAfter(t *testing.T) {
+	ctx := context.Background()
+	for _, tc := range []struct {
+		name   string
+		status int
+		header string
+		want   time.Duration
+		ok     bool
+	}{
+		{"503 during a workspace move", http.StatusServiceUnavailable, "30", 30 * time.Second, true},
+		{"429 from the edge", http.StatusTooManyRequests, "7", 7 * time.Second, true},
+		{"capped", http.StatusServiceUnavailable, "86400", maxRetryAfter, true},
+		{"missing header", http.StatusServiceUnavailable, "", 0, false},
+		{"HTTP date is not seconds", http.StatusServiceUnavailable, "Wed, 21 Oct 2015 07:28:00 GMT", 0, false},
+		{"502 ignores the header", http.StatusBadGateway, "30", 0, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			resp := jsonResp(tc.status, `{"ok":false,"error":"busy"}`)
+			if tc.header != "" {
+				resp.Header.Set("Retry-After", tc.header)
+			}
+			retry, err := shouldRetry(ctx, resp, errorHandler(resp))
+			assert.True(t, retry)
+			d, ok := pacer.IsRetryAfter(err)
+			assert.Equal(t, tc.ok, ok)
+			if tc.ok {
+				assert.Equal(t, tc.want, d)
+			}
+		})
+	}
+}
+
+func TestShouldRetryDoesNotRetryConflict(t *testing.T) {
+	resp := jsonResp(http.StatusConflict, `{"ok":false,"error":"A folder with this name already exists"}`)
+	resp.Header.Set("Retry-After", "30")
+	retry, err := shouldRetry(context.Background(), resp, errorHandler(resp))
+	assert.False(t, retry)
+	_, ok := pacer.IsRetryAfter(err)
+	assert.False(t, ok)
+}
+
+func TestRemoveFolderPurgesPermanently(t *testing.T) {
+	// a big subtree purges in bounded passes: 202 complete=false repeats
+	// the same request until complete=true
+	answers := []struct {
+		status int
+		body   string
+	}{
+		{http.StatusAccepted, `{"ok":true,"permanent":true,"complete":false,"remaining":1}`},
+		{http.StatusOK, `{"ok":true,"permanent":true,"complete":true,"remaining":0}`},
+	}
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		require.Equal(t, "DELETE", r.Method)
+		require.Equal(t, "/api/folders/fld_1", r.URL.Path)
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "", r.URL.Query().Get("if_empty"))
+		require.Less(t, calls, len(answers), "DELETE sent after the purge completed")
+		a := answers[calls]
+		calls++
+		return jsonResp(a.status, a.body), nil
+	}))
+
+	require.NoError(t, f.removeFolder(context.Background(), "fld_1", false))
+	assert.Equal(t, len(answers), calls)
+}
+
+func TestRemoveFolderStopsOnPurgeError(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		return jsonResp(http.StatusForbidden, `{"ok":false,"error":"This folder contains a locked file and cannot be purged"}`), nil
+	}))
+
+	err := f.removeFolder(context.Background(), "fld_1", false)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "locked file")
+	assert.Equal(t, 1, calls)
+}
+
+func TestRemoveFolderIfEmptyRejectsNonEmptyWithoutRetry(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+		return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
+	}))
+
+	err := f.removeFolder(context.Background(), "fld_1", true)
+	assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
+	assert.Equal(t, 1, calls, "a 409 must not be retried by the pacer")
+}
+
+func TestRemoveFolderIfEmptyPurgesEmptyFolderInOneCall(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+		return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+	}))
+
+	require.NoError(t, f.removeFolder(context.Background(), "fld_1", true))
+	assert.Equal(t, 1, calls)
+}
+
+func TestRmdirMakesOneRequest(t *testing.T) {
+	calls := 0
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		calls++
+		assert.Equal(t, "DELETE", r.Method)
+		assert.Equal(t, "/api/folders/fld_1", r.URL.Path)
+		assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+		assert.Equal(t, "1", r.URL.Query().Get("if_empty"))
+		return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+	}))
+	f.dirCache = dircache.New("", rootID, f)
+	require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
+	f.dirCache.Put("sub", "fld_1")
+
+	require.NoError(t, f.Rmdir(context.Background(), "sub"))
+	assert.Equal(t, 1, calls, "Rmdir must not list the directory before removing it")
+}
+
+func TestRmdirMapsNonEmptyConflict(t *testing.T) {
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		return jsonResp(http.StatusConflict, `{"ok":false,"error":"Folder is not empty"}`), nil
+	}))
+	f.dirCache = dircache.New("", rootID, f)
+	require.NoError(t, f.dirCache.FindRoot(context.Background(), false))
+	f.dirCache.Put("sub", "fld_1")
+
+	err := f.Rmdir(context.Background(), "sub")
+	assert.ErrorIs(t, err, fs.ErrorDirectoryNotEmpty)
+}
+
+func TestDeleteFileSendsPermanentFlag(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		permanent bool
+		wantCalls int
+	}{
+		{"today's API purges in one call", true, 1},
+		{"an older server ignores the flag and still gets a second DELETE", false, 2},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			calls := 0
+			f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+				calls++
+				assert.Equal(t, "DELETE", r.Method)
+				assert.Equal(t, "/api/files/fil_1", r.URL.Path)
+				assert.Equal(t, "1", r.URL.Query().Get("permanent"))
+				return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"permanent":%t}`, tc.permanent)), nil
+			}))
+
+			require.NoError(t, f.deleteFile(context.Background(), "fil_1"))
+			assert.Equal(t, tc.wantCalls, calls)
+		})
+	}
+}
+
+// decodeBody reads a JSON request body into a map
+func decodeBody(t *testing.T, r *http.Request) map[string]any {
+	b, err := io.ReadAll(r.Body)
+	require.NoError(t, err)
+	var m map[string]any
+	require.NoError(t, json.Unmarshal(b, &m))
+	return m
+}
+
+func TestMoveFolderSendsName(t *testing.T) {
+	for _, tc := range []struct {
+		name        string
+		newName     string
+		respBody    string
+		wantRenamed bool
+	}{
+		{"server honours name", "dst", `{"ok":true,"name":"dst"}`, true},
+		{"older server ignores name", "dst", `{"ok":true}`, false},
+		{"no rename requested", "", `{"ok":true,"name":"src"}`, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+				assert.Equal(t, "/api/folders/fld_1/move", r.URL.Path)
+				body := decodeBody(t, r)
+				assert.Equal(t, "fld_parent", body["parent_id"])
+				if tc.newName == "" {
+					assert.NotContains(t, body, "name")
+				} else {
+					assert.Equal(t, tc.newName, body["name"])
+				}
+				return jsonResp(http.StatusOK, tc.respBody), nil
+			}))
+
+			renamed, err := f.moveFolder(context.Background(), "fld_1", "fld_parent", tc.newName)
+			require.NoError(t, err)
+			assert.Equal(t, tc.wantRenamed, renamed)
+		})
+	}
+}
+
+func TestCopyFileSendsName(t *testing.T) {
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		assert.Equal(t, "/api/files/fil_1/copy", r.URL.Path)
+		body := decodeBody(t, r)
+		assert.Equal(t, "fld_dst", body["folder_id"])
+		assert.Equal(t, "final.txt", body["name"])
+		return jsonResp(http.StatusCreated, `{"ok":true,"file_id":"fil_2","name":"final.txt"}`), nil
+	}))
+
+	resp, err := f.copyFile(context.Background(), "fil_1", "fld_dst", "final.txt")
+	require.NoError(t, err)
+	assert.Equal(t, "fil_2", resp.FileID)
+	assert.Equal(t, "final.txt", resp.Name)
+}

backend/dosya/dosya_hash_modtime_test.go

@@ -0,0 +1,171 @@
+package dosya
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFsAdvertisesSHA256AndSecondPrecision(t *testing.T) {
+	f := &Fs{}
+	assert.Equal(t, time.Second, f.Precision())
+	assert.True(t, f.Hashes().Contains(hash.SHA256), "SHA-256 must be advertised")
+	// Exactly SHA-256, nothing else.
+	assert.Equal(t, hash.NewHashSet(hash.SHA256), f.Hashes())
+}
+
+func TestObjectHashReturnsContentHash(t *testing.T) {
+	o := &Object{file: api.FileItem{ContentHash: "abc123"}}
+
+	got, err := o.Hash(context.Background(), hash.SHA256)
+	require.NoError(t, err)
+	assert.Equal(t, "abc123", got)
+
+	// A multipart object with no server-side hash yields "" and no error,
+	// exactly as an S3 multipart object does.
+	empty := &Object{file: api.FileItem{ContentHash: ""}}
+	got, err = empty.Hash(context.Background(), hash.SHA256)
+	require.NoError(t, err)
+	assert.Equal(t, "", got)
+
+	// Any other algorithm is unsupported.
+	_, err = o.Hash(context.Background(), hash.MD5)
+	assert.ErrorIs(t, err, hash.ErrUnsupported)
+}
+
+func TestObjectModTimeUsesUpdatedAt(t *testing.T) {
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+	o := &Object{file: api.FileItem{UpdatedAt: float64(mt.Unix()), CreatedAt: 1}}
+	assert.True(t, mt.Equal(o.ModTime(context.Background())))
+}
+
+func TestModTimeHeaders(t *testing.T) {
+	assert.Nil(t, modTimeHeaders(time.Time{}), "a zero time sends no header")
+
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 499, time.UTC)
+	h := modTimeHeaders(mt)
+	require.NotNil(t, h)
+	assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), h[sourceMTimeHeader])
+}
+
+func TestFileFromUploadKeepsModTimeAndHash(t *testing.T) {
+	resp := &api.UploadCompleteResponse{}
+	resp.File.ID = "fil_1"
+	resp.File.Name = "x.txt"
+	resp.File.SizeBytes = 10
+	resp.File.CreatedAt = 1_000
+	resp.File.ContentHash = "deadbeef"
+
+	mt := time.Date(2004, 3, 3, 4, 5, 6, 0, time.UTC)
+	item := fileFromUpload(resp, mt)
+	assert.Equal(t, float64(mt.Unix()), item.UpdatedAt, "modtime is what we sent, not the create time")
+	assert.Equal(t, "deadbeef", item.ContentHash)
+	assert.Equal(t, float64(1_000), item.CreatedAt)
+
+	// A zero modtime falls back to the create time.
+	item = fileFromUpload(resp, time.Time{})
+	assert.Equal(t, float64(1_000), item.UpdatedAt)
+}
+
+// TestUploadSendsSourceMTimeHeader drives a real single-PUT upload through a
+// fake transport and asserts the source-mtime header actually rides on the PUT.
+func TestUploadSendsSourceMTimeHeader(t *testing.T) {
+	var putMTime string
+	rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+		if r.Body != nil {
+			_, _ = io.ReadAll(r.Body) // drain so the upload's counting reader advances
+		}
+		switch {
+		case r.Method == "POST" && r.URL.Path == "/api/upload/init":
+			return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
+		case r.Method == "PUT" && r.URL.Path == "/api/upload/upl_1":
+			putMTime = r.Header.Get(sourceMTimeHeader)
+			return jsonResp(http.StatusCreated, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":5,"content_hash":"h"}}`), nil
+		}
+		t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		return nil, nil
+	})
+	f := newFakeFs(rt)
+
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+	resp, err := f.uploadFile(context.Background(), strings.NewReader("hello"), "file.txt", 5, mt, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "h", resp.File.ContentHash)
+	assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), putMTime, "the PUT must carry the source-mtime header")
+}
+
+// TestMultipartCompleteSendsSourceMTimeHeader asserts the header also rides on
+// the completion request for a multipart upload (the server stores the modtime
+// there, not on the individual parts).
+func TestMultipartCompleteSendsSourceMTimeHeader(t *testing.T) {
+	var completeMTime string
+	rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+		if r.Body != nil {
+			_, _ = io.ReadAll(r.Body)
+		}
+		switch {
+		case r.URL.Path == "/api/upload/init":
+			return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":{"part_size":4,"total_parts":3}}`), nil
+		case strings.HasPrefix(r.URL.Path, "/api/upload/upl_1/part/"):
+			return jsonResp(http.StatusOK, `{"ok":true}`), nil
+		case r.URL.Path == "/api/upload/upl_1/complete":
+			completeMTime = r.Header.Get(sourceMTimeHeader)
+			return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
+		}
+		t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		return nil, nil
+	})
+	f := newFakeFs(rt)
+
+	mt := time.Date(2003, 2, 3, 4, 5, 6, 0, time.UTC)
+	_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, mt, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, strconv.FormatInt(mt.Unix(), 10), completeMTime, "the complete call must carry the source-mtime header")
+}
+
+// TestCopyReturnsSourceModTimeAndHash guards the fingerprint of the object a
+// server-side Copy returns: it must report the SOURCE's modtime and hash (the
+// copy is byte-identical and the API preserves both), not the copy's wall
+// clock. Reading "now" here made rclone's in-memory object disagree with the
+// one read back from the remote and failed FsCopy's fingerprint check.
+func TestCopyReturnsSourceModTimeAndHash(t *testing.T) {
+	rt := rtFunc(func(r *http.Request) (*http.Response, error) {
+		switch {
+		case r.Method == "GET" && r.URL.Path == "/api/files":
+			// NewObject's existence probe on the destination: nothing there.
+			return jsonResp(http.StatusOK, `{"ok":true,"files":[],"folders":[]}`), nil
+		case r.Method == "POST" && strings.HasSuffix(r.URL.Path, "/copy"):
+			return jsonResp(http.StatusOK, `{"ok":true,"file_id":"fil_copy","name":"f-copy.txt"}`), nil
+		}
+		t.Fatalf("unexpected %s %s", r.Method, r.URL.Path)
+		return nil, nil
+	})
+	f := newFakeFs(rt)
+	f.dirCache = dircache.New("", rootID, f)
+
+	mt := time.Date(2001, 2, 3, 4, 5, 10, 0, time.UTC)
+	ct := time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)
+	src := &Object{fs: f, remote: "orig.txt", file: api.FileItem{
+		ID: "fil_src", Name: "orig.txt", SizeBytes: 100, MimeType: "text/plain",
+		CreatedAt: float64(ct.Unix()), UpdatedAt: float64(mt.Unix()), ContentHash: "srchash",
+	}}
+
+	dst, err := f.Copy(context.Background(), src, "f-copy.txt")
+	require.NoError(t, err)
+
+	assert.True(t, mt.Equal(dst.ModTime(context.Background())), "copy keeps the source modtime")
+	assert.Equal(t, int64(100), dst.Size())
+	got, err := dst.Hash(context.Background(), hash.SHA256)
+	require.NoError(t, err)
+	assert.Equal(t, "srchash", got, "copy keeps the source hash")
+}

backend/dosya/dosya_internal_test.go

@@ -0,0 +1,156 @@
+package dosya
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/dosya/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const concurrentLimitBody = `{"ok":false,"error":"You have 5 uploads in progress. Wait for them to finish before starting more.","error_code":"concurrent_upload_limit","max_concurrent_uploads":5}`
+
+func TestErrorHandlerDecodesAPIError(t *testing.T) {
+	resp := &http.Response{
+		StatusCode: http.StatusBadRequest,
+		Status:     "400 Bad Request",
+		Body:       io.NopCloser(strings.NewReader(concurrentLimitBody)),
+	}
+	err := errorHandler(resp)
+	var apiErr *api.Error
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, http.StatusBadRequest, apiErr.StatusCode)
+	assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
+	assert.Equal(t, 5, apiErr.MaxConcurrentUploads)
+	assert.Contains(t, err.Error(), "uploads in progress")
+	assert.Contains(t, err.Error(), "concurrent_upload_limit")
+}
+
+func TestErrorHandlerNonJSONBody(t *testing.T) {
+	// e.g. an edge (Cloudflare) response that never reached the API
+	resp := &http.Response{
+		StatusCode: http.StatusTooManyRequests,
+		Status:     "429 Too Many Requests",
+		Body:       io.NopCloser(strings.NewReader("error code: 1015\n")),
+	}
+	err := errorHandler(resp)
+	var apiErr *api.Error
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, http.StatusTooManyRequests, apiErr.StatusCode)
+	assert.Equal(t, "", apiErr.ErrorCode)
+	assert.Contains(t, err.Error(), "429")
+	assert.Contains(t, err.Error(), "error code: 1015")
+}
+
+func TestShouldRetryAPIErrors(t *testing.T) {
+	ctx := context.Background()
+
+	// The concurrent upload limit is handled by initUpload's slot wait,
+	// not by the pacer's low level retries.
+	limitErr := &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "concurrent_upload_limit"}
+	retry, err := shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, limitErr)
+	assert.False(t, retry)
+	assert.Equal(t, limitErr, err)
+
+	retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusTooManyRequests}, &api.Error{StatusCode: http.StatusTooManyRequests})
+	assert.True(t, retry)
+
+	retry, _ = shouldRetry(ctx, &http.Response{StatusCode: http.StatusBadRequest}, &api.Error{StatusCode: http.StatusBadRequest, ErrorCode: "something_else"})
+	assert.False(t, retry)
+}
+
+// swapWaitTimings shortens the slot wait timings for a test and returns a
+// function which restores them
+func swapWaitTimings(delay, max time.Duration) func() {
+	oldDelay, oldMax := concurrentUploadRetryDelay, concurrentUploadWaitMax
+	concurrentUploadRetryDelay, concurrentUploadWaitMax = delay, max
+	return func() { concurrentUploadRetryDelay, concurrentUploadWaitMax = oldDelay, oldMax }
+}
+
+// newTestFs makes an Fs which talks to srv
+func newTestFs(srv *httptest.Server) *Fs {
+	ctx := context.Background()
+	f := &Fs{
+		name: "test",
+		opt:  Options{WorkspaceID: "ws_test"},
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(
+			pacer.MinSleep(time.Millisecond),
+			pacer.MaxSleep(time.Millisecond),
+			pacer.DecayConstant(decayConstant),
+			pacer.AttackConstant(attackConstant),
+		)),
+	}
+	f.rest = rest.NewClient(srv.Client()).SetRoot(srv.URL).SetErrorHandler(errorHandler)
+	return f
+}
+
+// limitServer answers the first n init calls with the concurrent upload
+// limit error and the rest with success
+func limitServer(t *testing.T, n int32, calls *atomic.Int32) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/api/upload/init", r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		if calls.Add(1) <= n {
+			w.WriteHeader(http.StatusBadRequest)
+			_, _ = w.Write([]byte(concurrentLimitBody))
+			return
+		}
+		w.WriteHeader(http.StatusCreated)
+		_, _ = w.Write([]byte(`{"ok":true}`))
+	}))
+}
+
+func TestInitUploadWaitsForFreeSlot(t *testing.T) {
+	defer swapWaitTimings(5*time.Millisecond, 10*time.Second)()
+	var calls atomic.Int32
+	srv := limitServer(t, 2, &calls)
+	defer srv.Close()
+	f := newTestFs(srv)
+
+	resp, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
+	require.NoError(t, err)
+	assert.True(t, resp.OK)
+	assert.Equal(t, int32(3), calls.Load())
+}
+
+func TestInitUploadGivesUpWhenSlotsStayFull(t *testing.T) {
+	defer swapWaitTimings(5*time.Millisecond, 40*time.Millisecond)()
+	var calls atomic.Int32
+	srv := limitServer(t, 1<<30, &calls)
+	defer srv.Close()
+	f := newTestFs(srv)
+
+	_, err := f.initUpload(context.Background(), "file.txt", 10, "text/plain", "", nil)
+	require.Error(t, err)
+	var apiErr *api.Error
+	require.ErrorAs(t, err, &apiErr)
+	assert.Equal(t, "concurrent_upload_limit", apiErr.ErrorCode)
+	assert.Contains(t, err.Error(), "couldn't init upload")
+	assert.GreaterOrEqual(t, calls.Load(), int32(2), "should have retried at least once")
+}
+
+func TestInitUploadStopsWaitingWhenContextCancelled(t *testing.T) {
+	defer swapWaitTimings(time.Minute, time.Hour)()
+	var calls atomic.Int32
+	srv := limitServer(t, 1<<30, &calls)
+	defer srv.Close()
+	f := newTestFs(srv)
+
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	_, err := f.initUpload(ctx, "file.txt", 10, "text/plain", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, context.DeadlineExceeded)
+	assert.Less(t, time.Since(start), 5*time.Second)
+}

backend/dosya/dosya_list_test.go

@@ -0,0 +1,68 @@
+package dosya
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeListServer serves GET /api/files the way the API does: files paged at
+// per_page, every folder repeated on each page
+func fakeListServer(t *testing.T, totalFiles int, pages *[]int) rtFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		require.Equal(t, "/api/files", r.URL.Path)
+		q := r.URL.Query()
+		assert.Equal(t, "name", q.Get("sort"))
+		assert.Equal(t, "asc", q.Get("dir"))
+		page, _ := strconv.Atoi(q.Get("page"))
+		perPage, _ := strconv.Atoi(q.Get("per_page"))
+		require.Positive(t, page)
+		require.Positive(t, perPage)
+		*pages = append(*pages, page)
+
+		var files []string
+		for i := (page - 1) * perPage; i < page*perPage && i < totalFiles; i++ {
+			files = append(files, fmt.Sprintf(`{"id":"fil_%d","name":"f%05d.txt","size":1}`, i, i))
+		}
+		totalPages := max(1, (totalFiles+perPage-1)/perPage)
+		body := fmt.Sprintf(`{"ok":true,"files":[%s],"folders":[{"id":"fld_1","name":"sub"}],`+
+			`"pagination":{"page":%d,"per_page":%d,"total_files":%d,"total_pages":%d}}`,
+			strings.Join(files, ","), page, perPage, totalFiles, totalPages)
+		return jsonResp(http.StatusOK, body), nil
+	}
+}
+
+func TestListFilesAndFoldersFollowsPages(t *testing.T) {
+	var pages []int
+	f := newFakeFs(fakeListServer(t, 2*listPageSize+7, &pages))
+
+	listing, err := f.listFilesAndFolders(context.Background(), "fld_parent")
+	require.NoError(t, err)
+
+	assert.Equal(t, []int{1, 2, 3}, pages)
+	require.Len(t, listing.Files, 2*listPageSize+7)
+	assert.Equal(t, "f01006.txt", listing.Files[len(listing.Files)-1].Name)
+	// folders repeat on every page and must not be duplicated
+	assert.Len(t, listing.Folders, 1)
+}
+
+func TestListFilesAndFoldersSinglePage(t *testing.T) {
+	for _, total := range []int{0, 1, listPageSize} {
+		t.Run(strconv.Itoa(total), func(t *testing.T) {
+			var pages []int
+			f := newFakeFs(fakeListServer(t, total, &pages))
+
+			listing, err := f.listFilesAndFolders(context.Background(), rootID)
+			require.NoError(t, err)
+
+			assert.Equal(t, []int{1}, pages)
+			assert.Len(t, listing.Files, total)
+		})
+	}
+}

backend/dosya/dosya_parallel_test.go

@@ -0,0 +1,153 @@
+package dosya
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// parallelUploadServer answers the multipart endpoints without holding a
+// lock across a request, so parts really do overlap, and records what the
+// ordering rules below are about: how many parts were in flight at once, and
+// whether any part started before part 1 had finished.
+//
+// Part 1 matters because the API creates the R2 multipart upload (and, for a
+// new file, its id and key) on the first part it sees. Parts that race ahead
+// of part 1 would each create their own upload, and only one of those would
+// be completed.
+type parallelUploadServer struct {
+	t          *testing.T
+	resumable  string
+	partDelay  time.Duration
+	failPart   int // answer this part with a 400, 0 for none
+	mu         sync.Mutex
+	inFlight   int
+	maxFlight  int
+	part1Done  bool
+	earlyParts []int // parts that started before part 1 finished
+	parts      map[int]string
+	completes  int
+	aborts     int
+}
+
+func (s *parallelUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
+	var body []byte
+	if r.Body != nil {
+		var err error
+		body, err = io.ReadAll(r.Body)
+		require.NoError(s.t, err)
+	}
+	path := r.URL.Path
+	switch {
+	case r.Method == "POST" && path == "/api/upload/init":
+		return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
+	case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
+		var n int
+		_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
+		require.NoError(s.t, err)
+
+		s.mu.Lock()
+		if n != 1 && !s.part1Done {
+			s.earlyParts = append(s.earlyParts, n)
+		}
+		s.inFlight++
+		s.maxFlight = max(s.maxFlight, s.inFlight)
+		s.mu.Unlock()
+
+		time.Sleep(s.partDelay)
+
+		s.mu.Lock()
+		s.inFlight--
+		if n == 1 {
+			s.part1Done = true
+		}
+		s.parts[n] = string(body)
+		fail := n == s.failPart
+		s.mu.Unlock()
+
+		if fail {
+			return jsonResp(http.StatusBadRequest, `{"ok":false,"error":"part refused"}`), nil
+		}
+		return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"part_number":%d,"etag":"e%d"}`, n, n)), nil
+	case r.Method == "POST" && path == "/api/upload/upl_1/complete":
+		s.mu.Lock()
+		s.completes++
+		s.mu.Unlock()
+		return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":20}}`), nil
+	case r.Method == "DELETE" && path == "/api/upload/upl_1":
+		s.mu.Lock()
+		s.aborts++
+		s.mu.Unlock()
+		return jsonResp(http.StatusOK, `{"ok":true}`), nil
+	}
+	s.t.Fatalf("unexpected request %s %s", r.Method, path)
+	return nil, nil
+}
+
+func newParallelServer(t *testing.T, resumable string) *parallelUploadServer {
+	return &parallelUploadServer{t: t, resumable: resumable, partDelay: 50 * time.Millisecond, parts: map[int]string{}}
+}
+
+const tenParts = `{"part_size":2,"total_parts":10}`
+
+func TestUploadMultipartSendsPartsConcurrently(t *testing.T) {
+	srv := newParallelServer(t, tenParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 4
+	content := "aabbccddeeffgghhiijj"
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+
+	assert.Equal(t, 4, srv.maxFlight, "parts after the first should use the whole concurrency")
+	assert.Empty(t, srv.earlyParts, "no part may start before part 1 has finished")
+	assert.Equal(t, 1, srv.completes)
+	for i := 1; i <= 10; i++ {
+		assert.Equal(t, content[2*(i-1):2*i], srv.parts[i], "part %d carries its own bytes", i)
+	}
+}
+
+func TestUploadMultipartConcurrencyOneIsSequential(t *testing.T) {
+	srv := newParallelServer(t, tenParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 1
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, 1, srv.maxFlight)
+	assert.Equal(t, 1, srv.completes)
+}
+
+func TestUploadMultipartFailedPartIsNotCompleted(t *testing.T) {
+	srv := newParallelServer(t, tenParts)
+	srv.failPart = 6
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 4
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffgghhiijj"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "part 6")
+	assert.Equal(t, 0, srv.completes, "an upload with a failed part must not be completed")
+}
+
+func TestUploadMultipartParallelRejectsShortSource(t *testing.T) {
+	// 20 bytes declared, 15 supplied: the short read is found by the
+	// reader, whichever parts are already in flight, and nothing completes.
+	srv := newParallelServer(t, tenParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	f.opt.UploadConcurrency = 4
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("aabbccddeeffggh"), "file.txt", 20, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	assert.Equal(t, 0, srv.completes)
+	assert.Empty(t, srv.parts[8], "a part the source never filled must not be sent")
+}

backend/dosya/dosya_test.go

@@ -0,0 +1,15 @@
+// Test Dosya filesystem interface
+package dosya
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestDosya:",
+	})
+}

backend/dosya/dosya_upload_test.go

@@ -0,0 +1,247 @@
+package dosya
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// rtFunc adapts a function into an http.RoundTripper so the upload path
+// can be driven without a network. Unlike a real server it always reads
+// the whole request body and answers, which is exactly the misbehaviour
+// (accepting a short body as a complete upload) these tests guard against.
+type rtFunc func(*http.Request) (*http.Response, error)
+
+func (f rtFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
+
+func jsonResp(status int, body string) *http.Response {
+	return &http.Response{
+		StatusCode:    status,
+		Status:        fmt.Sprintf("%d %s", status, http.StatusText(status)),
+		Header:        http.Header{"Content-Type": {"application/json"}},
+		Body:          io.NopCloser(strings.NewReader(body)),
+		ContentLength: int64(len(body)),
+	}
+}
+
+func newFakeFs(rt http.RoundTripper) *Fs {
+	ctx := context.Background()
+	f := &Fs{
+		name: "test",
+		opt:  Options{WorkspaceID: "ws_test"},
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(
+			pacer.MinSleep(time.Millisecond),
+			pacer.MaxSleep(time.Millisecond),
+			pacer.DecayConstant(decayConstant),
+			pacer.AttackConstant(attackConstant),
+		)),
+	}
+	f.rest = rest.NewClient(&http.Client{Transport: rt}).SetRoot("https://fake.invalid").SetErrorHandler(errorHandler)
+	return f
+}
+
+// fakeUploadServer records what an upload sends and lets a test inject
+// failures per request
+type fakeUploadServer struct {
+	t          *testing.T
+	mu         sync.Mutex
+	resumable  string           // JSON for the init "resumable" field, "" for a single PUT
+	puts       [][]byte         // bodies of PUT /api/upload/<id>
+	parts      map[int][][]byte // bodies of PUT /api/upload/<id>/part/<n>, per part, per attempt
+	completes  int
+	aborts     int             // DELETE /api/upload/<id>
+	failFirst  map[string]bool // path -> answer the first request with a 502
+	seenBefore map[string]bool
+}
+
+func newFakeUploadServer(t *testing.T, resumable string) *fakeUploadServer {
+	return &fakeUploadServer{
+		t:          t,
+		resumable:  resumable,
+		parts:      map[int][][]byte{},
+		failFirst:  map[string]bool{},
+		seenBefore: map[string]bool{},
+	}
+}
+
+func (s *fakeUploadServer) roundTrip(r *http.Request) (*http.Response, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var body []byte
+	if r.Body != nil {
+		var err error
+		body, err = io.ReadAll(r.Body)
+		require.NoError(s.t, err)
+	}
+	path := r.URL.Path
+	// Bodies are recorded for every attempt, including the ones answered
+	// with a 502, so tests can see exactly what each attempt carried.
+	fail := s.failFirst[path] && !s.seenBefore[path]
+	s.seenBefore[path] = true
+	bad := jsonResp(http.StatusBadGateway, `{"ok":false,"error":"bad gateway"}`)
+	switch {
+	case r.Method == "POST" && path == "/api/upload/init":
+		if s.resumable != "" {
+			return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1","resumable":`+s.resumable+`}`), nil
+		}
+		return jsonResp(http.StatusCreated, `{"ok":true,"session_id":"upl_1"}`), nil
+	case r.Method == "PUT" && path == "/api/upload/upl_1":
+		s.puts = append(s.puts, body)
+		if fail {
+			return bad, nil
+		}
+		return jsonResp(http.StatusCreated, fmt.Sprintf(`{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":%d}}`, len(body))), nil
+	case r.Method == "PUT" && strings.HasPrefix(path, "/api/upload/upl_1/part/"):
+		var n int
+		_, err := fmt.Sscanf(strings.TrimPrefix(path, "/api/upload/upl_1/part/"), "%d", &n)
+		require.NoError(s.t, err)
+		s.parts[n] = append(s.parts[n], body)
+		if fail {
+			return bad, nil
+		}
+		return jsonResp(http.StatusOK, fmt.Sprintf(`{"ok":true,"part_number":%d,"bytes_uploaded":%d}`, n, len(body))), nil
+	case r.Method == "POST" && path == "/api/upload/upl_1/complete":
+		s.completes++
+		return jsonResp(http.StatusOK, `{"ok":true,"file":{"id":"fil_1","name":"file.txt","size_bytes":10}}`), nil
+	case r.Method == "DELETE" && path == "/api/upload/upl_1":
+		s.aborts++
+		return jsonResp(http.StatusOK, `{"ok":true}`), nil
+	}
+	s.t.Fatalf("unexpected request %s %s", r.Method, path)
+	return nil, nil
+}
+
+func TestUploadSmallSendsWholeBody(t *testing.T) {
+	srv := newFakeUploadServer(t, "")
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	content := strings.Repeat("a", 100)
+
+	resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "fil_1", resp.File.ID)
+	require.Len(t, srv.puts, 1)
+	assert.Equal(t, content, string(srv.puts[0]))
+}
+
+func TestUploadSmallRejectsShortSource(t *testing.T) {
+	// The source declares 100 bytes but only supplies 60. A server which
+	// accepts that as a complete upload must not turn into a success.
+	srv := newFakeUploadServer(t, "")
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 60)), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	assert.Contains(t, err.Error(), "expected 100 bytes in input, but got 60")
+}
+
+func TestUploadSmallDoesNotResendDrainedBody(t *testing.T) {
+	// A retried PUT would resend an already-consumed reader, i.e. an
+	// empty body, which the server would store as an empty file. The
+	// single PUT therefore must not be retried at the low level.
+	srv := newFakeUploadServer(t, "")
+	srv.failFirst["/api/upload/upl_1"] = true
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	content := strings.Repeat("a", 100)
+	_, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 100, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	require.Len(t, srv.puts, 1, "the PUT must not be retried with a drained body")
+	assert.Equal(t, content, string(srv.puts[0]))
+}
+
+const threeParts = `{"part_size":4,"total_parts":3}`
+
+func TestUploadMultipartAssemblesAllBytes(t *testing.T) {
+	srv := newFakeUploadServer(t, threeParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+	content := "0123456789"
+
+	resp, err := f.uploadFile(context.Background(), strings.NewReader(content), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	assert.Equal(t, "fil_1", resp.File.ID)
+	assert.Equal(t, 1, srv.completes)
+	require.Len(t, srv.parts, 3)
+	assert.Equal(t, "0123", string(srv.parts[1][0]))
+	assert.Equal(t, "4567", string(srv.parts[2][0]))
+	assert.Equal(t, "89", string(srv.parts[3][0]))
+}
+
+func TestUploadMultipartRejectsShortSource(t *testing.T) {
+	// 10 bytes declared, 8 supplied: the last part must never be sent
+	// and the upload must not be completed.
+	srv := newFakeUploadServer(t, threeParts)
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("01234567"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, io.ErrUnexpectedEOF)
+	assert.Contains(t, err.Error(), "expected 10 bytes in input, but got 8")
+	assert.Empty(t, srv.parts[3], "short final part must not be uploaded")
+	assert.Equal(t, 0, srv.completes, "upload must not be completed")
+}
+
+func TestUploadMultipartRetriesPartWithFullBody(t *testing.T) {
+	// A part which fails transiently must be resent with all its bytes,
+	// not with whatever is left of a consumed reader.
+	srv := newFakeUploadServer(t, threeParts)
+	srv.failFirst["/api/upload/upl_1/part/2"] = true
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader("0123456789"), "file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+	require.NoError(t, err)
+	require.Len(t, srv.parts[2], 2, "part 2 should have been sent twice")
+	assert.Equal(t, "4567", string(srv.parts[2][0]))
+	assert.Equal(t, "4567", string(srv.parts[2][1]))
+	assert.Equal(t, "89", string(srv.parts[3][0]))
+	assert.Equal(t, 1, srv.completes)
+}
+
+// An upload that fails after init has to hand its session back: the session
+// holds one of the workspace's upload slots for ten minutes otherwise, and
+// rclone's high-level retry opens a new one rather than resuming it.
+func TestUploadAbortsTheSessionOnFailure(t *testing.T) {
+	for _, tc := range []struct {
+		name      string
+		resumable string
+		content   string
+		size      int64
+	}{
+		{"small file short source", "", "0123", 10},
+		{"multipart short source", `{"part_size":5,"total_parts":4}`, "aabbccddeeffggh", 20},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			srv := newFakeUploadServer(t, tc.resumable)
+			f := newFakeFs(rtFunc(srv.roundTrip))
+
+			_, err := f.uploadFile(context.Background(), strings.NewReader(tc.content),
+				"file.txt", tc.size, time.Time{}, "text/plain; charset=utf-8", "", nil)
+
+			require.Error(t, err)
+			assert.Equal(t, 1, srv.aborts)
+		})
+	}
+}
+
+// A successful upload must not abort anything
+func TestUploadDoesNotAbortOnSuccess(t *testing.T) {
+	srv := newFakeUploadServer(t, "")
+	f := newFakeFs(rtFunc(srv.roundTrip))
+
+	_, err := f.uploadFile(context.Background(), strings.NewReader(strings.Repeat("a", 10)),
+		"file.txt", 10, time.Time{}, "text/plain; charset=utf-8", "", nil)
+
+	require.NoError(t, err)
+	assert.Equal(t, 0, srv.aborts)
+}

backend/dosya/dosya_workspace_hint_test.go

@@ -0,0 +1,73 @@
+package dosya
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Every by-id call names the workspace, so the API can find an id it minted
+// moments ago before its file_home index row lands.
+func TestByIDCallsSendWorkspaceHint(t *testing.T) {
+	seen := map[string]string{}
+	f := newFakeFs(rtFunc(func(r *http.Request) (*http.Response, error) {
+		seen[r.Method+" "+r.URL.Path] = r.Header.Get("X-Dosya-Workspace")
+		switch r.URL.Path {
+		case "/api/folders/fld_1":
+			return jsonResp(http.StatusOK, `{"ok":true,"permanent":true,"complete":true}`), nil
+		case "/api/files/fil_1":
+			return jsonResp(http.StatusOK, `{"ok":true,"permanent":true}`), nil
+		case "/api/folders/fld_1/move":
+			return jsonResp(http.StatusOK, `{"ok":true,"name":"n"}`), nil
+		default:
+			return jsonResp(http.StatusOK, `{"ok":true}`), nil
+		}
+	}))
+	ctx := context.Background()
+
+	require.NoError(t, f.removeFolder(ctx, "fld_1", false))
+	require.NoError(t, f.renameFolder(ctx, "fld_1", "n"))
+	_, err := f.moveFolder(ctx, "fld_1", "fld_2", "n")
+	require.NoError(t, err)
+	require.NoError(t, f.deleteFile(ctx, "fil_1"))
+	require.NoError(t, f.renameFile(ctx, "fil_1", "n"))
+	require.NoError(t, f.moveFile(ctx, "fil_1", "fld_2", "n"))
+	_, err = f.copyFile(ctx, "fil_1", "fld_2", "n")
+	require.NoError(t, err)
+	_, err = f.createShareLink(ctx, "fil_1", 0)
+	require.NoError(t, err)
+
+	for call, header := range seen {
+		assert.Equal(t, "ws_test", header, call)
+	}
+	assert.Len(t, seen, 8)
+}
+
+// The download's API request carries the hint; the presigned R2 fetch must not.
+func TestDownloadSendsWorkspaceHintOnlyToTheAPI(t *testing.T) {
+	var apiHint, blobHint string
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api/files/fil_1/download" {
+			apiHint = r.Header.Get("X-Dosya-Workspace")
+			http.Redirect(w, r, srv.URL+"/blob", http.StatusFound)
+			return
+		}
+		blobHint = r.Header.Get("X-Dosya-Workspace")
+		_, _ = w.Write([]byte("bytes"))
+	}))
+	defer srv.Close()
+	f := newTestFs(srv)
+	f.opt.APIURL = srv.URL
+
+	rc, err := f.downloadFile(context.Background(), "fil_1", nil)
+	require.NoError(t, err)
+	_ = rc.Close()
+
+	assert.Equal(t, "ws_test", apiHint)
+	assert.Equal(t, "", blobHint)
+}

backend/dosya/dosya_workspace_test.go

@@ -0,0 +1,52 @@
+package dosya
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeWorkspaceServer answers GET /api/workspaces with the given body
+func fakeWorkspaceServer(t *testing.T, body string, calls *int) rtFunc {
+	return func(r *http.Request) (*http.Response, error) {
+		require.Equal(t, "/api/workspaces", r.URL.Path)
+		*calls++
+		return jsonResp(http.StatusOK, body), nil
+	}
+}
+
+func TestFindWorkspaceAdoptsTheOnlyOne(t *testing.T) {
+	calls := 0
+	f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[{"id":"ws_only","name":"Mine"}]}`, &calls))
+	f.opt.WorkspaceID = ""
+
+	require.NoError(t, f.findWorkspace(context.Background()))
+	assert.Equal(t, "ws_only", f.opt.WorkspaceID)
+	assert.Equal(t, 1, calls)
+}
+
+func TestFindWorkspaceListsTheChoices(t *testing.T) {
+	calls := 0
+	f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[`+
+		`{"id":"ws_one","name":"One"},{"id":"ws_two","name":"Two"}]}`, &calls))
+	f.opt.WorkspaceID = ""
+
+	err := f.findWorkspace(context.Background())
+	require.Error(t, err)
+	// the ids have to be in the message: they are the only place the user
+	// can read them, which is the whole point of listing them
+	assert.Contains(t, err.Error(), "ws_one (One)")
+	assert.Contains(t, err.Error(), "ws_two (Two)")
+	assert.Empty(t, f.opt.WorkspaceID)
+}
+
+func TestFindWorkspaceWithNone(t *testing.T) {
+	calls := 0
+	f := newFakeFs(fakeWorkspaceServer(t, `{"ok":true,"workspaces":[]}`, &calls))
+	f.opt.WorkspaceID = ""
+
+	require.ErrorContains(t, f.findWorkspace(context.Background()), "no workspaces found")
+}

backend/drime/drime.go

@@ -321,10 +321,13 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
 		}
 		return false
 	})
+	if err != nil {
+		return nil, err
+	}
 	if !found {
 		return nil, fs.ErrorObjectNotFound
 	}
-	return info, err
+	return info, nil
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -546,6 +549,11 @@ OUTER:
 		if err != nil {
 			return found, fmt.Errorf("couldn't list files: %w", err)
 		}
+		// Drime won't page past a fixed number of entries (20,000 for
+		// rclone) and serves the last page it allows again instead
+		if result.CurrentPage != page {
+			return found, errors.New("couldn't list files: directory has more entries than drime will list")
+		}
 		for _, item := range result.Data {
 			if item.Type == api.ItemTypeFolder {
 				if filesOnly {
@@ -692,13 +700,27 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // deleteObject removes an object by ID
 func (f *Fs) deleteObject(ctx context.Context, id string) error {
+	if f.opt.HardDelete {
+		// Deleting forever doesn't invalidate the server's cache of
+		// the parent folder listing, so the entry keeps being listed
+		// for about a second. Moving to the trash first does.
+		err := f.deleteEntry(ctx, id, false)
+		if err != nil {
+			return err
+		}
+	}
+	return f.deleteEntry(ctx, id, f.opt.HardDelete)
+}
+
+// deleteEntry moves the entry with ID to the trash or deletes it forever
+func (f *Fs) deleteEntry(ctx context.Context, id string, deleteForever bool) error {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/file-entries/delete",
 	}
 	request := api.DeleteRequest{
 		EntryIDs:      []string{id},
-		DeleteForever: f.opt.HardDelete,
+		DeleteForever: deleteForever,
 	}
 	var result api.DeleteResponse
 	err := f.pacer.Call(func() (bool, error) {
@@ -1040,17 +1062,28 @@ func (f *Fs) copy(ctx context.Context, id, newDirID string) (item *api.Item, err
 }
 
 // copy and rename a file or folder to directoryID with leaf
-func (f *Fs) copyTo(ctx context.Context, srcID, srcLeaf, dstLeaf, dstDirectoryID string) (info *api.Item, err error) {
-	// Can have duplicates so don't have to be careful here
-
+//
+// If existing is set it is removed once the copy has succeeded.
+func (f *Fs) copyTo(ctx context.Context, srcID, dstLeaf, dstDirectoryID string, existing fs.Object) (info *api.Item, err error) {
 	// Copy to dstDirectoryID first
 	info, err = f.copy(ctx, srcID, dstDirectoryID)
 	if err != nil {
 		return nil, err
 	}
 
+	// The server names the copy "name (1)" if the destination
+	// directory already has an entry called "name" and refuses to
+	// rename it to "name" until that entry is removed.
+	if existing != nil {
+		fs.Debugf(existing, "Server side copy: removing existing object after successful copy")
+		err = existing.Remove(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	// Rename if required
-	if srcLeaf != dstLeaf {
+	if info.Name != f.opt.Enc.FromStandardName(dstLeaf) {
 		info, err = f.rename(ctx, info.ID.String(), dstLeaf)
 		if err != nil {
 			return nil, err
@@ -1074,8 +1107,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	srcLeaf := path.Base(srcObj.remote)
-
 	srcPath := srcObj.fs.rootSlash() + srcObj.remote
 	dstPath := f.rootSlash() + remote
 	if srcPath == dstPath {
@@ -1084,15 +1115,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 
 	// Find existing object
 	existingObj, err := f.NewObject(ctx, remote)
-	if err == nil {
-		defer func() {
-			// Don't remove existing object if returning an error
-			if err != nil {
-				return
-			}
-			fs.Debugf(existingObj, "Server side copy: removing existing object after successful copy")
-			err = existingObj.Remove(ctx)
-		}()
+	if err != nil {
+		existingObj = nil
 	}
 
 	// Create temporary object
@@ -1102,7 +1126,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	// Copy the object
-	info, err := f.copyTo(ctx, srcObj.id, srcLeaf, dstLeaf, dstDirectoryID)
+	info, err := f.copyTo(ctx, srcObj.id, dstLeaf, dstDirectoryID, existingObj)
 	if err != nil {
 		return nil, err
 	}

backend/drime/drime_internal_test.go

@@ -0,0 +1,158 @@
+package drime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/drime/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testPerPage = 5  // entries per page the test server returns
+	testLimit   = 20 // entries the test server will page through
+)
+
+// listingServer serves /drive/file-entries from entries the way Drime
+// does.
+//
+// It returns testPerPage entries per page and won't page past testLimit
+// entries. Asking for a later page returns the last page it allows
+// again, with last_page one beyond it.
+//
+// It counts the requests made in requests.
+func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		assert.Equal(t, "/drive/file-entries", r.URL.Path)
+		params := r.URL.Query()
+		page, err := strconv.Atoi(params.Get("page"))
+		require.NoError(t, err)
+		maxPage := testLimit / testPerPage
+		currentPage := min(page, maxPage)
+		start := min((currentPage-1)*testPerPage, len(entries))
+		end := min(start+testPerPage, len(entries))
+		lastPage := currentPage
+		if page > maxPage || end < len(entries) {
+			lastPage = currentPage + 1
+		}
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"current_page": currentPage,
+			"last_page":    lastPage,
+			"per_page":     testPerPage,
+			"total":        (currentPage + 1) * testPerPage,
+			"data":         entries[start:end],
+		}))
+	}))
+}
+
+// makeEntries makes n file entries
+func makeEntries(n int) []api.Item {
+	entries := make([]api.Item, n)
+	for i := range entries {
+		entries[i] = api.Item{
+			ID:   json.Number(strconv.Itoa(1000 + i)),
+			Name: fmt.Sprintf("file%05d", i),
+			Type: "text",
+		}
+	}
+	return entries
+}
+
+// testFs makes an Fs which uses server as its API
+func testFs(t *testing.T, server *httptest.Server) *Fs {
+	t.Helper()
+	f := &Fs{
+		opt: Options{ListChunk: 1000},
+		srv: rest.NewClient(server.Client()).SetRoot(server.URL),
+	}
+	f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+	f.srv.SetErrorHandler(errorHandler)
+	return f
+}
+
+// listNames lists a directory with listAll and returns the names found
+func listNames(ctx context.Context, f *Fs) (names []string, err error) {
+	_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
+		names = append(names, item.Name)
+		return false
+	})
+	return names, err
+}
+
+// TestListAll checks that listAll returns every entry exactly once
+func TestListAll(t *testing.T) {
+	for _, test := range []struct {
+		name    string
+		entries int
+	}{
+		{name: "Empty", entries: 0},
+		{name: "OnePage", entries: 3},
+		{name: "ExactPages", entries: 10},
+		{name: "PartialLastPage", entries: 13},
+		{name: "AtLimit", entries: testLimit},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			var requests atomic.Int64
+			entries := makeEntries(test.entries)
+			server := listingServer(t, entries, &requests)
+			defer server.Close()
+			f := testFs(t, server)
+
+			names, err := listNames(ctx, f)
+			require.NoError(t, err)
+			var want []string
+			for _, entry := range entries {
+				want = append(want, entry.Name)
+			}
+			assert.ElementsMatch(t, want, names)
+		})
+	}
+}
+
+// TestListAllTooManyEntries checks that listAll returns an error for a
+// directory with more entries than the server will list, rather than
+// looping forever.
+func TestListAllTooManyEntries(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	var requests atomic.Int64
+	server := listingServer(t, makeEntries(2*testLimit), &requests)
+	defer server.Close()
+	f := testFs(t, server)
+
+	_, err := listNames(ctx, f)
+	require.Error(t, err)
+	assert.NoError(t, ctx.Err(), "listing didn't finish")
+	assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
+}
+
+// TestGetItemListError checks that getItem returns listing errors
+// rather than fs.ErrorObjectNotFound.
+func TestGetItemListError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
+	}))
+	defer server.Close()
+	f := testFs(t, server)
+
+	_, err := f.getItem(context.Background(), "1000", "1", "file00000")
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
+	assert.Contains(t, err.Error(), "access denied")
+}

backend/drive/drive.go

@@ -568,7 +568,7 @@ recommended usage is using the flag form --drive-size-as-quota when
 doing rclone ls/lsl/lsf/lsjson/etc only.
 
 If you do use this flag for syncing (not recommended) then you will
-need to use --ignore size also.`,
+need to use --ignore-size also.`,
 			Advanced: true,
 			Hide:     fs.OptionHideConfigurator,
 		}, {
@@ -2298,15 +2298,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
 		for len(overflow) > 0 {
-			mu.Lock()
-			// only fill half of the channel to prevent entries being put into overflow again
-			l := min(len(overflow), listRInputBuffer/2)
-			wg.Add(l)
-			for _, d := range overflow[:l] {
-				in <- d
+			if !refillListRInput(&mu, &in, &overflow, &wg) {
+				return
 			}
-			overflow = overflow[l:]
-			mu.Unlock()
 
 			// wait again for the completion of all directories
 			wg.Wait()
@@ -2354,6 +2348,25 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	return nil
 }
 
+// refillListRInput moves queued directory jobs back onto in.
+// It returns false when a listing error has already closed in.
+// Sending on that nil channel would block forever while holding mu.
+func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
+	mu.Lock()
+	defer mu.Unlock()
+	if *in == nil {
+		return false
+	}
+	// only fill half of the channel to prevent entries being put into overflow again
+	l := min(len(*overflow), listRInputBuffer/2)
+	wg.Add(l)
+	for _, d := range (*overflow)[:l] {
+		*in <- d
+	}
+	*overflow = (*overflow)[l:]
+	return true
+}
+
 const shortcutSeparator = '\t'
 
 // joinID adds an actual drive ID to the shortcut ID it came from

backend/drive/drive_internal_test.go

@@ -916,3 +916,51 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
+	mu := stdsync.Mutex{}
+	wg := stdsync.WaitGroup{}
+	in := make(chan listREntry, listRInputBuffer)
+	overflow := []listREntry{{id: "folder", path: "folder"}}
+
+	mu.Lock()
+	close(in)
+	in = nil
+	mu.Unlock()
+
+	done := make(chan bool, 1)
+	go func() {
+		done <- refillListRInput(&mu, &in, &overflow, &wg)
+	}()
+	select {
+	case ok := <-done:
+		require.False(t, ok, "refill continued after the input channel was closed")
+	case <-time.After(time.Second):
+		t.Fatal("refill blocked after the input channel was closed")
+	}
+	require.Len(t, overflow, 1)
+
+	finished := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(time.Second):
+		t.Fatal("refill incremented the wait group after the input channel was closed")
+	}
+}
+
+func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
+	mu := stdsync.Mutex{}
+	wg := stdsync.WaitGroup{}
+	in := make(chan listREntry, listRInputBuffer)
+	overflow := []listREntry{{id: "folder", path: "folder"}}
+
+	require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
+	require.Empty(t, overflow)
+	got := <-in
+	require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
+	wg.Done()
+}

backend/filelu/filelu_client.go

@@ -16,13 +16,16 @@ import (
 
 // multipartInit starts a new multipart upload and returns server details.
 func (f *Fs) multipartInit(ctx context.Context, folderPath, filename string) (*api.MultipartInitResponse, error) {
+	encodedFolderPath := f.fromStandardPath(folderPath)
+	encodedFilename := f.fromStandardPath(filename)
+
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   "/multipart/init",
 		Parameters: url.Values{
 			"key":         {f.opt.Key},
-			"filename":    {filename},
-			"folder_path": {folderPath},
+			"filename":    {encodedFilename},
+			"folder_path": {encodedFolderPath},
 		},
 	}
 

backend/ftp/ftp.go

@@ -584,8 +584,9 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
 //
 // It nils the pointed to connection out so it can't be reused
 //
-// if err is not nil then it checks the connection is alive using a
-// NOOP request
+// if err is not nil then it discards the connection unless the error
+// carries an FTP reply code, in which case it checks the connection
+// is alive using a NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	if f.opt.Concurrency > 0 {
 		defer f.tokens.Put()
@@ -599,14 +600,17 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	*pc = nil
 	if err != nil {
-		// If not a regular FTP error code then check the connection
-		if tpErr := textprotoError(err); tpErr != nil {
-			nopErr := c.NoOp()
-			if nopErr != nil {
-				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
-				_ = c.Quit()
-				return
-			}
+		// An error without an FTP reply code means the control
+		// connection can no longer be trusted, so discard it.
+		if textprotoError(err) == nil {
+			fs.Debugf(f, "Discarding connection after non-FTP error: %v", err)
+			_ = c.Quit()
+			return
+		}
+		if nopErr := c.NoOp(); nopErr != nil {
+			fs.Debugf(f, "Connection failed, closing: %v", nopErr)
+			_ = c.Quit()
+			return
 		}
 	}
 	f.poolMu.Lock()
@@ -1265,15 +1269,18 @@ func (o *Object) Storable() bool {
 
 // ftpReadCloser implements io.ReadCloser for FTP objects.
 type ftpReadCloser struct {
-	rc  io.ReadCloser
-	c   *ftp.ServerConn
-	f   *Fs
-	err error // errors found during read
+	rc     io.ReadCloser
+	c      *ftp.ServerConn
+	f      *Fs
+	err    error // errors found during read
+	closed bool  // set by Close so a second call is a no-op
+	read   int64 // bytes returned by Read
 }
 
 // Read bytes into p
 func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 	n, err = f.rc.Read(p)
+	f.read += int64(n)
 	if err != nil && err != io.EOF {
 		f.err = err // store any errors for Close to examine
 	}
@@ -1282,6 +1289,10 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
+	if f.closed {
+		return nil
+	}
+	f.closed = true
 	var err error
 	errchan := make(chan error, 1)
 	go func() {
@@ -1305,6 +1316,7 @@ func (f *ftpReadCloser) Close() error {
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
+		f.c = nil // never pool a connection after Quit
 		f.f.putFtpConnection(nil, nil)
 	} else {
 		f.f.putFtpConnection(&f.c, nil)
@@ -1316,6 +1328,12 @@ func (f *ftpReadCloser) Close() error {
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend, ftp.StatusRequestedFileActionOK:
 			err = nil
+		case ftp.StatusCanNotOpenDataConnection:
+			// Egnyte answers 425 when a RETR is closed before the first
+			// read; a transfer cut after data was read stays an error
+			if f.read == 0 {
+				err = nil
+			}
 		}
 	}
 	return err

backend/ftp/ftp_internal_test.go

@@ -1,12 +1,19 @@
 package ftp
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
+	"io"
+	"net"
+	"net/textproto"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
@@ -144,3 +151,159 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// fakeFTPServer is a minimal FTP control-connection server which answers
+// every command with 200 OK.
+type fakeFTPServer struct {
+	ln   net.Listener
+	mu   sync.Mutex
+	conn net.Conn
+}
+
+func startFakeFTPServer(t *testing.T) *fakeFTPServer {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	s := &fakeFTPServer{ln: ln}
+	go s.serve()
+	t.Cleanup(func() { s.close() })
+	return s
+}
+
+func (s *fakeFTPServer) serve() {
+	conn, err := s.ln.Accept()
+	if err != nil {
+		return
+	}
+	s.mu.Lock()
+	s.conn = conn
+	s.mu.Unlock()
+	defer func() { _ = conn.Close() }()
+	_, _ = fmt.Fprint(conn, "220 fake FTP server\r\n")
+	scanner := bufio.NewScanner(conn)
+	for scanner.Scan() {
+		_, _ = fmt.Fprint(conn, "200 OK\r\n")
+	}
+}
+
+// killConnection closes the server side of the control connection,
+// simulating a server that dropped an idle connection.
+func (s *fakeFTPServer) killConnection() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.conn != nil {
+		_ = s.conn.Close()
+	}
+}
+
+func (s *fakeFTPServer) close() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.conn != nil {
+		_ = s.conn.Close()
+	}
+	_ = s.ln.Close()
+}
+
+// TestPutFtpConnectionDiscardsDeadConnection checks that a connection
+// returned with a raw network error is closed and never re-pooled, while
+// a connection returned with a plain FTP reply-code error is probed with
+// NOOP and pooled while the probe succeeds.
+func TestPutFtpConnectionDiscardsDeadConnection(t *testing.T) {
+	t.Run("network error discards", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, errors.New("unexpected EOF"))
+		assert.Nil(t, c)
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("FTP reply error with live connection pools", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
+		assert.Nil(t, c)
+		require.Len(t, f.pool, 1)
+		assert.NoError(t, f.pool[0].NoOp())
+		_ = f.pool[0].Quit()
+	})
+
+	t.Run("FTP reply error with dead connection discards", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		server.killConnection()
+		// Let the FIN propagate so the NOOP probe observes the dead connection.
+		time.Sleep(100 * time.Millisecond)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
+		assert.Nil(t, c)
+		assert.Empty(t, f.pool)
+	})
+}
+
+// fakeDataReader stands in for the RETR data stream of ftpReadCloser.
+type fakeDataReader struct {
+	data     []byte
+	closeErr error
+}
+
+func (r *fakeDataReader) Read(p []byte) (int, error) {
+	if len(r.data) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, r.data)
+	r.data = r.data[n:]
+	return n, nil
+}
+
+func (r *fakeDataReader) Close() error { return r.closeErr }
+
+// TestFtpReadCloserClose checks that an early-closed RETR answered with 425
+// is masked only when nothing was read, that a connection which got QUIT is
+// never pooled, and that a second Close does not touch the pool.
+func TestFtpReadCloserClose(t *testing.T) {
+	err425 := &textproto.Error{Code: ftp.StatusCanNotOpenDataConnection, Msg: "Can't open data connection."}
+	newReader := func(t *testing.T, data []byte, closeErr error) (*ftpReadCloser, *Fs) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		return &ftpReadCloser{rc: &fakeDataReader{data: data, closeErr: closeErr}, c: c, f: f}, f
+	}
+
+	t.Run("425 before any read is masked and the connection is not pooled", func(t *testing.T) {
+		rc, f := newReader(t, nil, err425)
+		assert.NoError(t, rc.Close())
+		assert.Empty(t, f.pool)
+		assert.NoError(t, rc.Close())
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("425 after data was read stays an error", func(t *testing.T) {
+		rc, f := newReader(t, []byte("partial"), err425)
+		buf := make([]byte, 4)
+		n, err := rc.Read(buf)
+		require.NoError(t, err)
+		require.Equal(t, 4, n)
+		err = rc.Close()
+		require.Error(t, err)
+		assert.Equal(t, ftp.StatusCanNotOpenDataConnection, textprotoError(err).Code)
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("clean close pools the connection once", func(t *testing.T) {
+		rc, f := newReader(t, []byte("data"), nil)
+		_, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		assert.NoError(t, rc.Close())
+		require.Len(t, f.pool, 1)
+		assert.NoError(t, rc.Close())
+		assert.Len(t, f.pool, 1)
+		_ = f.pool[0].Quit()
+	})
+}

backend/hasher/hasher_internal_test.go

@@ -122,6 +122,49 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {
 	_ = operations.Purge(ctx, f, dirName)
 }
 
+func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {
+	ctx := context.Background()
+
+	const dirName = "setmodtime_hash_1"
+	const fileName = dirName + "/file_setmodtime_1"
+	const longTime = fs.ModTimeNotSupported
+	hashType := f.keepHashes.GetOne()
+
+	// upload a file to hasher via Put
+	src := putFile(ctx, t, f, fileName, "setmodtime content")
+	dst, err := f.NewObject(ctx, fileName)
+	require.NoError(t, err)
+	require.NotNil(t, dst)
+	_ = src
+
+	// verify hash was stored after Put
+	if f.opt.MaxAge <= 0 {
+		t.Skip("hash cache is disabled")
+	}
+	hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+	require.NoError(t, err)
+	require.NotEmpty(t, hash1)
+
+	// change only the modification time, as operations does when it
+	// syncs the modtime of an otherwise unchanged destination file
+	newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")
+	require.NoError(t, dst.SetModTime(ctx, newTime))
+
+	// the cached hash must survive a pure modtime change
+	hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+	assert.NoError(t, err)
+	assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")
+
+	// and it must be found via the fingerprint-checked lookup too
+	dst2, err := f.NewObject(ctx, fileName)
+	require.NoError(t, err)
+	hashVal, err := dst2.Hash(ctx, hashType)
+	assert.NoError(t, err)
+	assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")
+
+	_ = operations.Purge(ctx, f, dirName)
+}
+
 // InternalTest dispatches all internal tests
 func (f *Fs) InternalTest(t *testing.T) {
 	if !kv.Supported() {
@@ -129,6 +172,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	}
 	t.Run("UploadFromCrypt", f.testUploadFromCrypt)
 	t.Run("UpdateStoresHash", f.testUpdateStoresHash)
+	t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/hasher/kv.go

@@ -206,6 +206,34 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
 	return err
 }
 
+// kvUpdateFingerprint: rewrite the fingerprint of an existing record
+type kvUpdateFingerprint struct {
+	key   string
+	oldFp string
+	newFp string
+	age   time.Duration
+}
+
+func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {
+	data := b.Get([]byte(op.key))
+	if len(data) == 0 {
+		return nil
+	}
+	var r hashRecord
+	if err := r.decode(op.key, data); err != nil {
+		return nil
+	}
+	if r.Fp != op.oldFp || time.Since(r.Created) > op.age {
+		return nil
+	}
+	r.Fp = op.newFp
+	data, err := r.encode(op.key)
+	if err != nil {
+		return err
+	}
+	return b.Put([]byte(op.key), data)
+}
+
 // kvDump: dump the database.
 // Note: long dump can cause concurrent operations to fail.
 type kvDump struct {

backend/hasher/object.go

@@ -56,6 +56,23 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {
 	return o.f.putRawHashes(ctx, key, fp, hashes)
 }
 
+// updateFingerprint rewrites the fingerprint of the cached hash
+// record for remote from oldFp to newFp, keeping the hashes. It is
+// used when only the modification time of an object changes, so the
+// cached hashes remain valid under the new fingerprint.
+func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {
+	if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {
+		return nil
+	}
+	key := path.Join(f.Fs.Root(), remote)
+	return f.db.Do(true, &kvUpdateFingerprint{
+		key:   key,
+		oldFp: oldFp,
+		newFp: newFp,
+		age:   time.Duration(f.opt.MaxAge),
+	})
+}
+
 // set hashes for a path without any validation
 func (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {
 	return f.db.Do(true, &kvPut{
@@ -196,14 +213,27 @@ func (o *Object) Remove(ctx context.Context) error {
 }
 
 // SetModTime sets the modification time of the file.
-// Also prunes the cache entry when modtime changes so that
-// touching a file will trigger checksum recalculation even
-// on backends that don't provide modTime with fingerprint.
+//
+// When the fingerprint includes the modtime, the contents have not
+// changed, so the cached hashes are carried over to the fingerprint
+// for the new modtime. Otherwise the cache entry is pruned when the
+// modtime changes so that touching a file will trigger checksum
+// recalculation even on backends that don't provide modTime with
+// fingerprint.
 func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
-	if mtime != o.Object.ModTime(ctx) {
-		_ = o.f.pruneHash(o.Remote())
+	if mtime == o.Object.ModTime(ctx) {
+		return o.Object.SetModTime(ctx, mtime)
 	}
-	return o.Object.SetModTime(ctx, mtime)
+	if !o.f.fpTime {
+		_ = o.f.pruneHash(o.Remote())
+		return o.Object.SetModTime(ctx, mtime)
+	}
+	oldFp := o.fingerprint(ctx)
+	if err := o.Object.SetModTime(ctx, mtime); err != nil {
+		return err
+	}
+	_ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))
+	return nil
 }
 
 // Open opens the file for read.

backend/hidrive/helpers.go

@@ -642,7 +642,7 @@ func (f *Fs) updateFileChunked(ctx context.Context, path string, content io.Read
 			chunkReader *pool.RW
 			bytesRead   int64
 		)
-		chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize))
+		chunkReader, bytesRead, readErr = readerForChunk(content, int64(chunkSize), true)
 		if bytesRead < int64(chunkSize) {
 			startMoreTransfers = false
 		}
@@ -869,14 +869,23 @@ func unwrapAccounting(reader io.Reader) (unwrapped io.Reader, acc *accounting.Ac
 // along with the number of bytes that have been read from reader.
 // Reaching the end of reader early is not an error.
 //
-// Any accounting on reader is applied when the buffer is read instead,
-// so the upload is what gets accounted.
+// If accountUpload is set, any accounting on reader is applied
+// when the buffer is read instead, so the upload is what gets accounted.
+//
+// This must not be set for a buffer which creates or replaces a file
+// in a single request. Accounting can fail part way through the upload,
+// for example when --max-transfer is reached, and HiDrive keeps
+// what it has received of an aborted request as the file.
+//
 // The caller must Close the buffer when done with it.
-func readerForChunk(reader io.Reader, length int64) (chunkReader *pool.RW, bytesRead int64, err error) {
-	reader, acc := unwrapAccounting(reader)
+func readerForChunk(reader io.Reader, length int64, accountUpload bool) (chunkReader *pool.RW, bytesRead int64, err error) {
 	chunkReader = multipart.NewRW()
-	if acc != nil {
-		chunkReader.SetAccounting(acc.AccountRead)
+	if accountUpload {
+		var acc *accounting.Account
+		reader, acc = unwrapAccounting(reader)
+		if acc != nil {
+			chunkReader.SetAccounting(acc.AccountRead)
+		}
 	}
 
 	bytesRead, err = io.CopyN(chunkReader, reader, length)

backend/hidrive/hidrive.go

@@ -522,7 +522,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	if size := src.Size(); size >= 0 && size < prefixSize {
 		prefixSize = size
 	}
-	cutoffReader, bytesRead, err := readerForChunk(in, prefixSize)
+	cutoffReader, bytesRead, err := readerForChunk(in, prefixSize, false)
 	if err != nil {
 		return nil, err
 	}
@@ -980,7 +980,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		info, metaErr = o.fs.fetchMetadataForPath(ctx, resolvedPath, api.HiDriveObjectWithMetadataFields)
 	} else {
 		var content *pool.RW
-		content, _, err = readerForChunk(in, src.Size())
+		content, _, err = readerForChunk(in, src.Size(), false)
 		if err == nil {
 			info, err = o.fs.overwriteFile(ctx, resolvedPath, content, modTime)
 			_ = content.Close()

backend/http/http.go

@@ -63,7 +63,8 @@ The headers are only sent to the host in the configured URL. If the
 server redirects to another host (including a subdomain or a different
 port) the headers are not sent to it, or to any further hop in that
 redirect chain. When headers are set, a redirect from https to http is
-refused as it would send them in cleartext.`,
+refused as it would send them in cleartext, even with
+--allow-https-downgrade.`,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
 		}, {
@@ -100,6 +101,30 @@ that directory listings are much quicker, but rclone won't have the times or
 sizes of any files, and some files that don't exist may be in the listing.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_head_size",
+			Help: `Set the reported file size for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unknown file size.
+Set this option to a non-negative value to report that size for every file,
+including 0. Leave it unset (the default, shown as "off") to keep reporting
+an unknown size.
+
+The configured size is authoritative. If it does not match the actual file
+size, transfers may fail their size check and retry unless --ignore-size is
+used. It can also make multi-thread transfers request incorrect ranges and
+make mounted files unreadable past the reported size.`,
+			Default:  fs.SizeSuffix(-1),
+			Advanced: true,
+		}, {
+			Name: "no_head_time",
+			Help: `Set a default modification time for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unset modification
+time. Set this option to report the specified time for every file instead.
+Changing the configured time later makes every file appear modified to a sync.`,
+			Default:  fs.Time{},
+			Advanced: true,
 		}, {
 			Name:    "no_escape",
 			Help:    "Do not escape URL metacharacters in path names.",
@@ -145,11 +170,13 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 
 // Options defines the configuration for this backend
 type Options struct {
-	Endpoint string          `config:"url"`
-	NoSlash  bool            `config:"no_slash"`
-	NoHead   bool            `config:"no_head"`
-	Headers  fs.CommaSepList `config:"headers"`
-	NoEscape bool            `config:"no_escape"`
+	Endpoint   string          `config:"url"`
+	NoSlash    bool            `config:"no_slash"`
+	NoHead     bool            `config:"no_head"`
+	NoHeadSize fs.SizeSuffix   `config:"no_head_size"`
+	NoHeadTime fs.Time         `config:"no_head_time"`
+	Headers    fs.CommaSepList `config:"headers"`
+	NoEscape   bool            `config:"no_escape"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -283,14 +310,15 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	if err != nil {
 		return false, err
 	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
+	u, err := rest.URLJoinRoot(base, f.root)
 	if err != nil {
 		return false, err
 	}
 
 	client := fshttp.NewClient(ctx)
-	// Without configured headers keep the default policy which
-	// public mirrors that redirect from https to http rely on.
+	// Without configured headers keep the default policy so
+	// --allow-https-downgrade can follow public mirrors that
+	// redirect from https to http.
 	if len(opt.Headers) > 0 {
 		client.CheckRedirect = checkRedirect(opt)
 	}
@@ -322,7 +350,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 // the host specified in the config file.
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
-	opt := new(Options)
+	opt := &Options{
+		NoHeadSize: fs.SizeSuffix(-1),
+	}
 	err := configstruct.Set(m, opt)
 	if err != nil {
 		return nil, err
@@ -526,40 +556,20 @@ func addHeaders(req *http.Request, opt *Options) {
 // rest.ErrHTTPSDowngrade and strips the configured headers when the
 // redirect chain has left the originally requested host at any point.
 func checkRedirect(opt *Options) func(req *http.Request, via []*http.Request) error {
+	headers := make([]string, 0, len(opt.Headers)/2)
+	for i := 0; i < len(opt.Headers); i += 2 {
+		headers = append(headers, opt.Headers[i])
+	}
+	stripHeaders := rest.StripHeadersOnCrossHostRedirectFn(headers...)
 	return func(req *http.Request, via []*http.Request) error {
-		if err := rest.RefuseHTTPSDowngradeRedirectFn(req, via); err != nil {
-			if errors.Is(err, rest.ErrHTTPSDowngrade) {
-				err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
-			}
-			return err
+		err := stripHeaders(req, via)
+		if errors.Is(err, rest.ErrHTTPSDowngrade) {
+			err = fmt.Errorf("%w (the configured headers would be sent to the plaintext target)", err)
 		}
-		if redirectLeavesHost(req, via) {
-			for i := 0; i < len(opt.Headers); i += 2 {
-				req.Header.Del(opt.Headers[i])
-			}
-		}
-		return nil
+		return err
 	}
 }
 
-// redirectLeavesHost reports whether any hop in the redirect chain
-// via plus the pending request req is to a different host from the
-// original request via[0].
-//
-// net/http copies the headers afresh from the original request for
-// every hop, so once the chain has visited another host the headers
-// must be stripped from every subsequent hop, even one back to the
-// original host, as the other host chose the URL.
-func redirectLeavesHost(req *http.Request, via []*http.Request) bool {
-	origin := via[0].URL
-	for _, hop := range via {
-		if !rest.SameHost(hop.URL, origin) {
-			return true
-		}
-	}
-	return !rest.SameHost(req.URL, origin)
-}
-
 // Adds the configured headers to the request if any
 func (f *Fs) addHeaders(req *http.Request) {
 	addHeaders(req, &f.opt)
@@ -737,8 +747,11 @@ func (o *Object) url() string {
 // head sends a HEAD request to update info fields in the Object
 func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
-		o.size = -1
+		o.size = int64(o.fs.opt.NoHeadSize)
 		o.modTime = timeUnset
+		if o.fs.opt.NoHeadTime.IsSet() {
+			o.modTime = time.Time(o.fs.opt.NoHeadTime)
+		}
 		o.contentType = fs.MimeType(ctx, o)
 		return nil
 	}

backend/http/http_internal_test.go

@@ -314,6 +314,54 @@ func TestOpen(t *testing.T) {
 	}
 }
 
+func TestNoHeadMetadataDefaults(t *testing.T) {
+	tests := []struct {
+		name        string
+		size        string
+		modTime     string
+		wantSize    int64
+		wantModTime time.Time
+	}{
+		{
+			name:        "defaults",
+			wantSize:    -1,
+			wantModTime: timeUnset,
+		}, {
+			name:        "zero size",
+			size:        "0B",
+			wantSize:    0,
+			wantModTime: timeUnset,
+		}, {
+			name:        "configured metadata",
+			size:        "100B",
+			modTime:     "2024-08-05T07:36:07Z",
+			wantSize:    100,
+			wantModTime: time.Date(2024, 8, 5, 7, 36, 7, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			m := prepareServer(t)
+			m.Set("no_head", "true")
+			if test.size != "" {
+				m.Set("no_head_size", test.size)
+			}
+			if test.modTime != "" {
+				m.Set("no_head_time", test.modTime)
+			}
+
+			f, err := NewFs(context.Background(), remoteName, "", m)
+			require.NoError(t, err)
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			assert.Equal(t, test.wantSize, o.Size())
+			assert.Equal(t, test.wantModTime, o.ModTime(context.Background()))
+		})
+	}
+}
+
 func TestMimeType(t *testing.T) {
 	f := prepare(t)
 
@@ -640,7 +688,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 
 	// open makes a remote pointing at ts with or without headers
 	// and opens file.txt on it
-	open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
+	open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {
 		configfile.Install()
 		m := configmap.Simple{
 			"type":    "http",
@@ -652,6 +700,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 		}
 		ctx, ci := fs.AddConfig(context.Background())
 		ci.InsecureSkipVerify = true
+		ci.AllowHTTPSDowngrade = allowDowngrade
 		f, err := NewFs(ctx, remoteName, "", m)
 		require.NoError(t, err)
 
@@ -663,13 +712,25 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 
 	t.Run("WithHeaders", func(t *testing.T) {
 		// Headers could leak so the downgrade must be refused
-		_, err := open(t, true)
+		_, err := open(t, true, false)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {
+		// Headers could leak so the downgrade is refused even when allowed
+		_, err := open(t, true, true)
 		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
 	})
 
 	t.Run("WithoutHeaders", func(t *testing.T) {
-		// Nothing to leak so the downgrade is followed
-		fd, err := open(t, false)
+		// The transport refuses downgrades by default
+		_, err := open(t, false, false)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {
+		// Nothing to leak so the downgrade is followed when allowed
+		fd, err := open(t, false, true)
 		require.NoError(t, err)
 		data, err := io.ReadAll(fd)
 		require.NoError(t, err)
@@ -760,3 +821,34 @@ func TestRedirectStripsHeadersOnHostChange(t *testing.T) {
 		assert.Empty(t, got.Values(headers[i]), "header %q leaked to redirect target", headers[i])
 	}
 }
+
+func TestRootCannotChangeHost(t *testing.T) {
+	// A root starting with "//" is a network-path reference under the
+	// URL standard. It must not be able to move the remote (and the
+	// configured headers) to a host other than the configured one.
+	var requests int
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		_, _ = w.Write([]byte("hello"))
+	}))
+	defer other.Close()
+
+	m := prepareServer(t)
+	otherHost := strings.TrimPrefix(other.URL, "http://")
+	for _, root := range []string{
+		"//" + otherHost + "/",
+		"//" + otherHost + "/one%.txt",
+		"//" + otherHost,
+		"//user:pass@" + otherHost + "/",
+	} {
+		f, err := NewFs(context.Background(), remoteName, root, m)
+		assert.ErrorContains(t, err, "must not change the scheme, host or user", root)
+		assert.Nil(t, f, root)
+	}
+	assert.Equal(t, 0, requests, "request sent to the other host")
+
+	// A same-origin absolute path must still work
+	f, err := NewFs(context.Background(), remoteName, "/", m)
+	require.NoError(t, err)
+	assert.Equal(t, m["url"]+"/", f.String())
+}

backend/huaweidrive/huaweidrive.go

@@ -50,6 +50,10 @@ const (
 	defaultChunkSize            = 8 * fs.Mebi
 )
 
+// jsonContentType is sent with JSON request bodies as the API doesn't
+// decode non-ASCII file names in them as UTF-8 unless told the charset.
+const jsonContentType = "application/json; charset=UTF-8"
+
 // OAuth2 configuration
 var oauthConfig = &oauthutil.Config{
 	Scopes: []string{
@@ -547,6 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File
@@ -1015,6 +1020,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	copyReq := api.CopyFileRequest{
@@ -1199,9 +1205,10 @@ func (f *Fs) updateFile(ctx context.Context, fileID string, parameters url.Value
 	}
 	parameters.Set("fields", "*")
 	opts := rest.Opts{
-		Method:     "PATCH",
-		Path:       "/files/" + fileID,
-		Parameters: parameters,
+		Method:       "PATCH",
+		Path:         "/files/" + fileID,
+		Parameters:   parameters,
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, req, &info)
@@ -1671,6 +1678,7 @@ func (o *Object) uploadEmpty(ctx context.Context, leaf, directoryID string, src
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File
@@ -1846,6 +1854,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 			"fields":     []string{"*"},
 		},
 		ExtraHeaders: map[string]string{
+			"Content-Type":            jsonContentType,
 			"X-Upload-Content-Length": strconv.FormatInt(size, 10),
 		},
 		RootURL: o.fs.uploadURL,
@@ -2145,6 +2154,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File

backend/iclouddrive/iclouddrive.go

@@ -92,7 +92,7 @@ func (f *Fs) findItem(ctx context.Context, dir string) (item *api.DriveItem, fou
 		item, resp, err = f.service.GetItemByPath(ctx, path.Join(f.root, dir))
 		return shouldRetry(ctx, resp, err)
 	}); err != nil {
-		if item == nil && resp.StatusCode == 404 {
+		if item == nil && resp != nil && resp.StatusCode == 404 {
 			return nil, false, nil
 		}
 		return nil, false, err
@@ -694,6 +694,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
+		CaseInsensitive:         true, // names are matched with EqualFold, see findLeafItem
 		CanHaveEmptyDirectories: true,
 		PartialUploads:          false,
 	}).Fill(ctx, f)

backend/imagekit/client/upload.go

@@ -77,6 +77,8 @@ func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadPara
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading file if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	opts := rest.Opts{
 		Method:      "POST",

backend/imagekit/imagekit.go

@@ -477,7 +477,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 			FolderPath: f.EncodePath(path.Join(f.root, dir)),
 		})
 
-		if res.StatusCode == http.StatusNotFound {
+		if res != nil && res.StatusCode == http.StatusNotFound {
 			return false, fs.ErrorDirNotFound
 		}
 
@@ -502,7 +502,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
 			FolderPath: f.EncodePath(remote),
 		})
 
-		if res.StatusCode == http.StatusNotFound {
+		if res != nil && res.StatusCode == http.StatusNotFound {
 			return false, fs.ErrorDirNotFound
 		}
 

backend/internetarchive/internetarchive.go

@@ -634,6 +634,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		"x-archive-filemeta-rclone-update-track": updateTracker,
 	}
 
+	mdata, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
+	if err != nil {
+		return nil, err
+	}
+	if mdata != nil {
+		// IA ignores file metadata headers with the COPY directive.
+		headers["x-amz-metadata-directive"] = "REPLACE"
+	}
+	for key, value := range mdata {
+		key = strings.ToLower(key)
+		if key == "mtime" {
+			key = "rclone-mtime"
+		} else if strings.HasPrefix(key, "rclone-") {
+			// Keep the new copy tracker and mapped mtime, not the source internals.
+			continue
+		} else if _, readOnly := roMetadataKey[key]; readOnly {
+			continue
+		}
+		headers["x-archive-filemeta-"+key] = value
+	}
+
 	// make a PUT request at (IAS3)/:item/:path without body
 	var resp *http.Response
 	opts := rest.Opts{

backend/internxt/internxt.go

@@ -474,8 +474,10 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	if len(childFolders) > 0 {
-		return fs.ErrorDirectoryNotEmpty
+	for _, e := range childFolders {
+		if !recent.stale(e.UUID, id, e.PlainName) {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 
 	var childFiles []folders.File
@@ -487,8 +489,10 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	if len(childFiles) > 0 {
-		return fs.ErrorDirectoryNotEmpty
+	for _, e := range childFiles {
+		if !recent.stale(e.UUID, id, joinNameExt(e.PlainName, e.Type)) {
+			return fs.ErrorDirectoryNotEmpty
+		}
 	}
 
 	// Delete the directory
@@ -502,6 +506,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
+	recent.deleted(id)
 
 	f.dirCache.FlushDir(dir)
 	return nil
@@ -520,6 +525,9 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (string, bool, e
 		return "", false, err
 	}
 	for _, e := range entries {
+		if recent.stale(e.UUID, pathID, e.PlainName) {
+			continue
+		}
 		if f.opt.Encoding.ToStandardName(e.PlainName) == leaf {
 			return e.UUID, true, nil
 		}
@@ -592,6 +600,73 @@ func existenceCheck(name, ext string) files.FileExistenceCheck {
 	}
 }
 
+// staleWindow is how long the API may carry on returning an item from
+// where it was before this process moved or deleted it.
+const staleWindow = time.Minute
+
+// location is where this process last put an item. An empty parentUUID
+// means the item was deleted.
+type location struct {
+	parentUUID string
+	name       string // encoded name, including any extension
+	expires    time.Time
+}
+
+// recentChanges holds the files and folders this process has moved or
+// deleted recently, keyed by UUID.
+//
+// The API serves lookups and listings from read replicas which lag behind
+// writes, so an item can be returned from its old location for a while after
+// it was moved or deleted. Acting on such a stale entry can be destructive,
+// eg overwriting it would delete the file from where it was moved to.
+type recentChanges struct {
+	mu        sync.Mutex
+	items     map[string]location
+	lastPrune time.Time
+}
+
+// recent is shared by all the Fs in the process as UUIDs are global.
+var recent = recentChanges{items: map[string]location{}}
+
+// set records the new location of uuid, pruning expired entries.
+func (r *recentChanges) set(uuid string, loc location) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	now := time.Now()
+	if now.Sub(r.lastPrune) > staleWindow {
+		for k, v := range r.items {
+			if now.After(v.expires) {
+				delete(r.items, k)
+			}
+		}
+		r.lastPrune = now
+	}
+	loc.expires = now.Add(staleWindow)
+	r.items[uuid] = loc
+}
+
+// moved records that uuid was moved to name in parentUUID.
+func (r *recentChanges) moved(uuid, parentUUID, name string) {
+	r.set(uuid, location{parentUUID: parentUUID, name: name})
+}
+
+// deleted records that uuid was deleted.
+func (r *recentChanges) deleted(uuid string) {
+	r.set(uuid, location{})
+}
+
+// stale reports whether an API result saying uuid is called name in
+// parentUUID is out of date because of a change made by this process.
+func (r *recentChanges) stale(uuid, parentUUID, name string) bool {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	loc, ok := r.items[uuid]
+	if !ok || time.Now().After(loc.expires) {
+		return false
+	}
+	return loc.parentUUID != parentUUID || loc.name != name
+}
+
 // findFile looks up a single file by name within directoryID. Returns
 // (nil, nil) when the file does not exist; surfaces transport/API errors.
 func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.File, error) {
@@ -623,6 +698,10 @@ func (f *Fs) findFile(ctx context.Context, leaf, directoryID string) (*folders.F
 		if !result.FileExists() || result.UUID == "" {
 			continue
 		}
+		if recent.stale(result.UUID, directoryID, joinNameExt(result.PlainName, result.Type)) {
+			fs.Debugf(f, "Ignoring stale lookup of %q (UUID: %s) which was moved or deleted", leaf, result.UUID)
+			continue
+		}
 		if joinNameExt(result.PlainName, result.Type) == encodedName {
 			uuid = result.UUID
 			break
@@ -682,6 +761,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		return nil, err
 	}
 	for _, e := range foldersList {
+		if recent.stale(e.UUID, dirID, e.PlainName) {
+			continue
+		}
 		remote := path.Join(dir, f.opt.Encoding.ToStandardName(e.PlainName))
 		out = append(out, fs.NewDir(remote, e.ModificationTime))
 	}
@@ -699,6 +781,9 @@ func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
 		if len(e.Type) > 0 {
 			remote += "." + e.Type
 		}
+		if recent.stale(e.UUID, dirID, remote) {
+			continue
+		}
 		remote = path.Join(dir, f.opt.Encoding.ToStandardName(remote))
 		out = append(out, newObjectWithFile(f, remote, &e))
 	}
@@ -771,6 +856,7 @@ func (f *Fs) Remove(ctx context.Context, remote string) error {
 	if err != nil {
 		return err
 	}
+	recent.deleted(dirID)
 	f.dirCache.FlushDir(remote)
 	return nil
 }
@@ -900,11 +986,18 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// Move the file server-side
 	err = f.pacer.Call(func() (bool, error) {
 		err := files.MoveFile(ctx, f.cfg, srcObj.uuid, directoryID, newName, newType)
+		// The move is checked against lagging read replicas so it can
+		// fail with 404 if the destination directory was just created
+		// or 409 if a file at the destination was just deleted.
+		if err != nil && isStaleReadError(err, true) {
+			return true, err
+		}
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return nil, err
 	}
+	recent.moved(srcObj.uuid, directoryID, joinNameExt(newName, newType))
 
 	dstObj := &Object{
 		f:       f,
@@ -940,11 +1033,17 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	encodedLeaf := f.opt.Encoding.FromStandardName(dstLeaf)
 	err = f.pacer.Call(func() (bool, error) {
 		err := folders.MoveFolder(ctx, f.cfg, srcID, dstDirectoryID, encodedLeaf)
+		// Moving a large folder can outlast the gateway timeout (520/502)
+		// yet still complete, in which case the retry reports this.
+		if err != nil && isConflictError(err) && strings.Contains(err.Error(), "already moved to that location") {
+			return false, nil
+		}
 		return f.shouldRetry(ctx, err)
 	})
 	if err != nil {
 		return err
 	}
+	recent.moved(srcID, dstDirectoryID, encodedLeaf)
 
 	srcFs.dirCache.FlushDir(srcRemote)
 	return nil
@@ -1124,6 +1223,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				backupName, backupType, backupUUID, err)
 			// Don't fail the upload just because backup deletion failed
 		} else {
+			recent.deleted(backupUUID)
 			fs.Debugf(o.f, "Successfully deleted backup file")
 		}
 	}
@@ -1229,8 +1329,13 @@ func (o *Object) restoreBackupFile(ctx context.Context, backupUUID, origName, or
 
 // Remove deletes a file
 func (o *Object) Remove(ctx context.Context) error {
-	return o.f.pacer.Call(func() (bool, error) {
+	err := o.f.pacer.Call(func() (bool, error) {
 		err := files.DeleteFile(ctx, o.f.cfg, o.uuid)
 		return o.f.shouldRetry(ctx, err)
 	})
+	if err != nil {
+		return err
+	}
+	recent.deleted(o.uuid)
+	return nil
 }

backend/internxt/internxt_internal_test.go

@@ -165,3 +165,27 @@ func TestFindFile(t *testing.T) {
 		})
 	}
 }
+
+func TestFindFileIgnoresStale(t *testing.T) {
+	ctx := context.Background()
+	server := existenceServer(t, []storedFile{{"uuid-moved", "foo", "txt"}, {"uuid-deleted", "bar", "txt"}, {"uuid-renamed", "baz", "txt"}})
+	defer server.Close()
+	f := testFs(t, server)
+
+	recent.moved("uuid-moved", "other-dir-uuid", "foo.txt.bak")
+	recent.deleted("uuid-deleted")
+	recent.moved("uuid-renamed", "dir-uuid", "baz.txt")
+
+	file, err := f.findFile(ctx, "foo.txt", "dir-uuid")
+	require.NoError(t, err)
+	assert.Nil(t, file, "moved away")
+
+	file, err = f.findFile(ctx, "bar.txt", "dir-uuid")
+	require.NoError(t, err)
+	assert.Nil(t, file, "deleted")
+
+	file, err = f.findFile(ctx, "baz.txt", "dir-uuid")
+	require.NoError(t, err)
+	require.NotNil(t, file, "moved here")
+	assert.Equal(t, "uuid-renamed", file.UUID)
+}

backend/jottacloud/jottacloud.go

@@ -86,8 +86,8 @@ func getServices() []service {
 		{"jottacloud", "Jottacloud", "id.jottacloud.com", "jottacloud", "desktop", []string{"openid", "jotta-default", "offline_access"}}, // Chose client id "desktop" here, will be identified as "Jottacloud for Desktop" in "My logged in devices", but could have used "jottacli" here as well.
 		{"elgiganten_dk", "Elgiganten Cloud (Denmark)", "cloud.elgiganten.dk", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
 		{"elgiganten_se", "Elgiganten Cloud (Sweden)", "cloud.elgiganten.se", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
-		{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "jotta-default", "offline_access"}},
-		{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "jotta-default", "offline_access"}},
+		{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "offline_access"}},
+		{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "offline_access"}},
 		{"gigantti", "Gigantti Cloud (Finland)", "cloud.gigantti.fi", "gigantti", "desktop", []string{"openid", "jotta-default", "offline_access"}},
 		{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
 		{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
@@ -258,6 +258,7 @@ service, for the official service on https://www.jottacloud.com/web/secure.`)
 		}
 		m.Set("configVersion", fmt.Sprint(configVersion))
 		m.Set(configClientID, service.clientID)
+		m.Set(configClientSecret, "")
 		m.Set(configTokenURL, wellKnown.TokenEndpoint)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
 			OAuth2Config: &oauthutil.Config{
@@ -1500,9 +1501,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
 	if err == nil {
-		var createTime time.Time
+		createTime := srcObj.createTime
 		var createTimeMeta bool
-		var modTime time.Time
+		modTime := srcObj.modTime
 		var modTimeMeta bool
 		if meta != nil {
 			createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
@@ -1518,6 +1519,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
 			fs.Debugf(src, "Server-side copied to trashed destination, restoring")
 			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
+			// The restored file keeps the name it was trashed with, which may differ in case
+			if err == nil && info != nil && info.Name != f.opt.Enc.FromStandardName(path.Base(remote)) {
+				fs.Debugf(src, "Renaming restored %q to %q", info.Name, path.Base(remote))
+				info, err = f.copyOrMove(ctx, "mv", f.filePath(remote), remote)
+			}
 		} else if createTimeMeta || modTimeMeta {
 			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
 		}

backend/kdrive/api/types.go

@@ -0,0 +1,235 @@
+// Package api has type definitions for kDrive
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// Time represents unix timestamp date and time information for the
+// kDrive API
+type Time time.Time
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" {
+		return nil
+	}
+
+	timestamp, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	newT := time.Unix(timestamp, 0)
+	*t = Time(newT)
+	return nil
+}
+
+// ResultStatus return error details from kdrive when things go wrong
+//
+// If Status is "success" then everything is OK
+type ResultStatus struct {
+	Status      string `json:"result"`
+	ErrorDetail struct {
+		Result      string `json:"code"`
+		ErrorString string `json:"description"`
+		Context     struct {
+			Values []string `json:"values"`
+		} `json:"context"`
+		Errors []struct {
+			Result      string `json:"code"`
+			ErrorString string `json:"description"`
+		} `json:"errors"`
+	} `json:"error"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e *ResultStatus) Error() string {
+	var details string
+	for i := range e.ErrorDetail.Errors {
+		details += "|" + e.ErrorDetail.Errors[i].Result
+	}
+
+	return fmt.Sprintf("kDrive error: %s (%s: %s %s)", e.Status, e.ErrorDetail.Result, e.ErrorDetail.ErrorString, details)
+}
+
+// IsError returns true if there is an error
+func (e *ResultStatus) IsError() bool {
+	return e.Status != "success"
+}
+
+// Update returns err directly if it was != nil, otherwise it returns
+// an Error or nil if no error was detected
+func (e *ResultStatus) Update(err error) error {
+	if err != nil {
+		return err
+	}
+	if e.IsError() {
+		return e
+	}
+	return nil
+}
+
+// Check ResultStatus satisfies the error interface
+var _ error = (*ResultStatus)(nil)
+
+// Item describes a folder or a file as returned by Get Folder Items and others
+type Item struct {
+	ID             int    `json:"id"`
+	Name           string `json:"name"`
+	Type           string `json:"type"`
+	FullPath       string `json:"path"`
+	Status         string `json:"status"`
+	Hash           string `json:"hash"`
+	Size           int64  `json:"size"`
+	Visibility     string `json:"visibility"`
+	DriveID        int    `json:"drive_id"`
+	Depth          int    `json:"depth"`
+	CreatedBy      int    `json:"created_by"`
+	CreatedAt      Time   `json:"created_at"`
+	AddedAt        int    `json:"added_at"`
+	LastModifiedAt Time   `json:"last_modified_at"`
+	LastModifiedBy int    `json:"last_modified_by"`
+	RevisedAt      int    `json:"revised_at"`
+	UpdatedAt      int    `json:"updated_at"`
+	MimeType       string `json:"mime_type"`
+	ParentID       int    `json:"parent_id"`
+	Color          string `json:"color"`
+}
+
+// ItemResult is returned when an item is queried
+type ItemResult struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// SearchResult is returned when a list of items is requested
+type SearchResult struct {
+	ResultStatus
+	Data       []Item `json:"data"`
+	Cursor     string `json:"cursor"`
+	HasMore    bool   `json:"has_more"`
+	ResponseAt int    `json:"response_at"`
+}
+
+// ModTime returns the modification time of the item
+func (i *Item) ModTime() (t time.Time) {
+	t = time.Time(i.LastModifiedAt)
+	if t.IsZero() {
+		t = time.Time(i.CreatedAt)
+	}
+	return t
+}
+
+// CancelResource is a kdrive resource that can be cancelled after some action
+type CancelResource struct {
+	CancelID   string `json:"cancel_id"`
+	ValidUntil int    `json:"valid_until"`
+}
+
+// CancellableResponse is returned from kdrive when an action can be cancelled afterwards
+type CancellableResponse struct {
+	ResultStatus
+	Data CancelResource `json:"data"`
+}
+
+// CreateDirResult is returned from kdrive after a call to MkDir
+type CreateDirResult struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// FileCopyResponse is returned from kdrive after a call to Copy
+type FileCopyResponse struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// UploadFileResponse is returned from kdrive after a call to Upload
+type UploadFileResponse struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// ChecksumFileResult is returned from kdrive after a call to Hash
+type ChecksumFileResult struct {
+	ResultStatus
+	Data struct {
+		Hash string `json:"hash"`
+	} `json:"data"`
+}
+
+// PubLinkResult is returned from kdrive after a call to public link API
+type PubLinkResult struct {
+	ResultStatus
+	Data struct {
+		URL          string `json:"url"`
+		FileID       int    `json:"file_id"`
+		Right        string `json:"right"`
+		ValidUntil   int    `json:"valid_until"`
+		CreatedBy    int    `json:"created_by"`
+		CreatedAt    int    `json:"created_at"`
+		UpdatedAt    int    `json:"updated_at"`
+		Capabilities struct {
+			CanEdit          bool `json:"can_edit"`
+			CanSeeStats      bool `json:"can_see_stats"`
+			CanSeeInfo       bool `json:"can_see_info"`
+			CanDownload      bool `json:"can_download"`
+			CanComment       bool `json:"can_comment"`
+			CanRequestAccess bool `json:"can_request_access"`
+		} `json:"capabilities"`
+		AccessBlocked bool `json:"access_blocked"`
+	} `json:"data"`
+}
+
+// PubLinkUpdateResult is returned from kdrive after a updating a public link
+type PubLinkUpdateResult struct {
+	ResultStatus
+	Data bool `json:"data"`
+}
+
+// QuotaInfo is return from kdrive after a call get drive info
+type QuotaInfo struct {
+	ResultStatus
+	Data struct {
+		Size     int64 `json:"size"`
+		UsedSize int64 `json:"used_size"`
+	} `json:"data"`
+}
+
+// SessionStartResponse is used to start a multichunk upload session
+type SessionStartResponse struct {
+	ResultStatus
+	Data struct {
+		Token     string `json:"token"`
+		UploadURL string `json:"upload_url"` // URL pour uploader les chunks
+	} `json:"data"`
+}
+
+// ChunkUploadResponse is returned by kdrive after a chunk upload
+type ChunkUploadResponse struct {
+	ResultStatus
+	Data struct {
+		Hash string `json:"hash,omitempty"` // Only present when requested with with:'hash'
+	} `json:"data"`
+}
+
+// SessionFinishResponse is used to end a multichunk upload session
+type SessionFinishResponse struct {
+	ResultStatus
+	Data struct {
+		Token   string `json:"token"`
+		File    Item   `json:"file"`
+		Result  bool   `json:"result"`
+		Message string `json:"message"`
+	} `json:"data"`
+}
+
+// SessionCancelResponse is used to cancel a multichunk upload session
+type SessionCancelResponse struct {
+	ResultStatus
+	Data bool `json:"data"`
+}

backend/kdrive/kdrive_internal_test.go

@@ -0,0 +1,551 @@
+// Unit tests for the kdrive backend
+package kdrive
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/kdrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestCheckChunkSize checks chunk_size validation against the API limits
+func TestCheckChunkSize(t *testing.T) {
+	for _, test := range []struct {
+		in      fs.SizeSuffix
+		wantErr string
+	}{
+		{in: 20 * fs.Mebi, wantErr: ""},                    // default
+		{in: 1000 * 1000, wantErr: ""},                     // API minimum
+		{in: 1000 * 1000 * 1000, wantErr: ""},              // API maximum
+		{in: 0, wantErr: "too small"},                      // panics without validation
+		{in: 999 * 1000, wantErr: "too small"},             // below API minimum
+		{in: 1000*1000*1000 + 1, wantErr: "too large"},     // above API maximum
+		{in: 2 * 1000 * 1000 * 1000, wantErr: "too large"}, // way above API maximum
+	} {
+		t.Run(test.in.String(), func(t *testing.T) {
+			err := checkChunkSize(test.in)
+			if test.wantErr == "" {
+				assert.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), test.wantErr)
+			}
+		})
+	}
+}
+
+// TestOpenChunkWriterClampsCallerChunkSize checks that a caller
+// supplied chunk size hint below the API minimum is raised to it
+func TestOpenChunkWriterClampsCallerChunkSize(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = fmt.Fprintln(w, `{"result":"success","data":{"token":"tok","upload_url":"http://unused"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0)}
+
+	info, _, err := f.OpenChunkWriter(context.Background(), "file.txt", src, &fs.ChunkOption{ChunkSize: 512 * 1024})
+	require.NoError(t, err)
+	assert.Equal(t, int64(minChunkSize), info.ChunkSize, "a chunk size hint below the API minimum must be raised to it")
+}
+
+// TestObjectSetMetaDataResetsStaleHash checks that setMetaData always
+// replaces the hash (a chunked upload finish response carries none)
+func TestObjectSetMetaDataResetsStaleHash(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		staleHash     string
+		staleNested   bool
+		newHash       string
+		wantHash      string
+		wantIsNested  bool
+		wantHashKnown bool
+	}{
+		{
+			name:          "empty hash clears stale plain hash",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "",
+			wantHash:      "",
+			wantIsNested:  false,
+			wantHashKnown: false,
+		},
+		{
+			name:          "empty hash clears stale nested hash",
+			staleHash:     "deadbeef",
+			staleNested:   true,
+			newHash:       "",
+			wantHash:      "",
+			wantIsNested:  false,
+			wantHashKnown: false,
+		},
+		{
+			name:          "plain hash is set",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "xxh3:12345678",
+			wantHash:      "12345678",
+			wantIsNested:  false,
+			wantHashKnown: true,
+		},
+		{
+			name:          "nested hash is set",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "N:xxh3:87654321",
+			wantHash:      "87654321",
+			wantIsNested:  true,
+			wantHashKnown: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o := &Object{
+				remote:      "file.txt",
+				hasMetaData: true,
+				xxh3:        test.staleHash,
+				nestedHash:  test.staleNested,
+			}
+			info := &api.Item{
+				ID:   42,
+				Name: "file.txt",
+				Type: "file",
+				Hash: test.newHash,
+			}
+
+			err := o.setMetaData(info)
+			require.NoError(t, err)
+			assert.Equal(t, test.wantHash, o.xxh3, "hash after setMetaData")
+			assert.Equal(t, test.wantIsNested, o.nestedHash, "nested flag after setMetaData")
+			assert.True(t, o.hasMetaData)
+
+			// Hash() must reflect the new state. Only checked when a
+			// hash is known: with no hash Hash() would query the API.
+			// Nested hashes are not comparable with plain XXH3, so
+			// Hash() returns "" for them.
+			if test.wantHashKnown {
+				got, gotErr := o.Hash(context.Background(), hash.XXH3)
+				require.NoError(t, gotErr)
+				if test.wantIsNested {
+					assert.Equal(t, "", got, "nested hash must not be returned as XXH3")
+				} else {
+					assert.Equal(t, test.wantHash, got)
+				}
+			}
+		})
+	}
+}
+
+// newTestFs creates an Fs wired to a test server with a plain client
+// (no OAuth) and a fast pacer
+func newTestFs(t *testing.T, endpoint string) *Fs {
+	t.Helper()
+	return &Fs{
+		name: "TestKdrive",
+		root: "",
+		opt:  Options{DriveID: "1234", Endpoint: endpoint},
+		srv:  rest.NewClient(&http.Client{}).SetRoot(endpoint),
+		pacer: fs.NewPacer(context.Background(),
+			pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond), pacer.DecayConstant(2))),
+	}
+}
+
+// TestWriteChunkDetectsResultError checks that a 200 with result:error
+// is reported as an error
+func TestWriteChunkDetectsResultError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"chunk_error","description":"chunk upload failed"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	u := &uploadSession{
+		f:           f,
+		token:       "tok",
+		uploadURL:   ts.URL,
+		fileSize:    4,
+		chunkSize:   4,
+		chunkHashes: make(map[int64]string),
+	}
+
+	_, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
+	require.Error(t, err, "result:error with HTTP 200 must not be treated as success")
+	assert.Contains(t, err.Error(), "chunk upload failed")
+}
+
+// TestWriteChunkRetriesAndRewinds checks that transient failures are
+// retried with the chunk re-sent from the start of the reader
+func TestWriteChunkRetriesAndRewinds(t *testing.T) {
+	var mu sync.Mutex
+	var attempts int
+	var bodies []string
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		attempts++
+		bodies = append(bodies, string(body))
+		current := attempts
+		mu.Unlock()
+
+		w.Header().Set("Content-Type", "application/json")
+		if current == 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
+			return
+		}
+		_, _ = fmt.Fprintln(w, `{"result":"success","data":{"chunk_number":1,"status":"OK"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	u := &uploadSession{
+		f:           f,
+		token:       "tok",
+		uploadURL:   ts.URL,
+		fileSize:    4,
+		chunkSize:   4,
+		chunkHashes: make(map[int64]string),
+	}
+
+	written, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), written)
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Len(t, bodies, 2, "chunk upload must be retried once after a 5xx")
+	assert.Equal(t, "test", bodies[0], "first attempt sends the chunk")
+	assert.Equal(t, "test", bodies[1], "retry must re-send the chunk from the start of the reader")
+}
+
+// kdriveMockItem returns a JSON body for a single item result
+func kdriveMockItem(id int, name string) string {
+	return fmt.Sprintf(`{"result":"success","data":{"id":%d,"name":%q,"type":"file","path":%q,"last_modified_at":1700000000}}`, id, name, "/"+name)
+}
+
+// mockObjectInfo is a minimal fs.ObjectInfo for testing
+type mockObjectInfo struct {
+	remote  string
+	size    int64
+	modTime time.Time
+	hash    string
+}
+
+func (m *mockObjectInfo) Fs() fs.Info                         { return nil }
+func (m *mockObjectInfo) String() string                      { return m.remote }
+func (m *mockObjectInfo) Remote() string                      { return m.remote }
+func (m *mockObjectInfo) Size() int64                         { return m.size }
+func (m *mockObjectInfo) ModTime(_ context.Context) time.Time { return m.modTime }
+func (m *mockObjectInfo) Storable() bool                      { return true }
+func (m *mockObjectInfo) MimeType(_ context.Context) string   { return "application/octet-stream" }
+
+func (m *mockObjectInfo) Hash(_ context.Context, t hash.Type) (string, error) {
+	if m.hash == "" || t != hash.XXH3 {
+		return "", hash.ErrUnsupported
+	}
+	return m.hash, nil
+}
+
+// TestUpdateDirectNoRetryOnServerError checks that a failed single
+// part upload is not retried here: the input must not be buffered, the
+// higher layers retry the whole transfer
+func TestUpdateDirectNoRetryOnServerError(t *testing.T) {
+	var attempts int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&attempts, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	o := &Object{fs: f, remote: "file.txt"}
+	src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: "12345678"}
+
+	err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
+	require.Error(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&attempts), "single part uploads must not be retried here")
+}
+
+// TestUpdateDirect checks the upload parameters and metadata of a
+// successful single part upload, and that total_chunk_hash is only
+// sent when the source hash is known
+func TestUpdateDirect(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		srcHash       string
+		wantTotalHash string
+	}{
+		{name: "hash known", srcHash: "12345678", wantTotalHash: "xxh3:12345678"},
+		{name: "hash unknown", srcHash: "", wantTotalHash: ""},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var gotTotalHash, gotBody string
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				gotTotalHash = r.URL.Query().Get("total_chunk_hash")
+				body, _ := io.ReadAll(r.Body)
+				gotBody = string(body)
+				_, _ = fmt.Fprintln(w, `{"result":"success","data":{"id":55,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:99998888","last_modified_at":1700000000}}`)
+			}))
+			defer ts.Close()
+
+			f := newTestFs(t, ts.URL)
+			o := &Object{fs: f, remote: "file.txt"}
+			src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: test.srcHash}
+
+			err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
+			require.NoError(t, err)
+			assert.Equal(t, "test", gotBody, "the content must be streamed as-is")
+			assert.Equal(t, test.wantTotalHash, gotTotalHash, "total_chunk_hash")
+			assert.Equal(t, "55", o.id)
+			gotHash, hashErr := o.Hash(context.Background(), hash.XXH3)
+			require.NoError(t, hashErr)
+			assert.Equal(t, "99998888", gotHash)
+		})
+	}
+}
+
+// TestNewFsRequiresDriveID checks that NewFs fails fast when drive_id
+// is missing
+func TestNewFsRequiresDriveID(t *testing.T) {
+	m := configmap.Simple{"chunk_size": "20Mi"}
+	_, err := NewFs(context.Background(), "kdrive", "", m)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "drive_id")
+}
+
+// TestUploadSessionStringMasksToken checks that the upload session
+// String does not leak the session token in debug logs
+func TestUploadSessionStringMasksToken(t *testing.T) {
+	u := &uploadSession{token: "very-secret-uuid"}
+	assert.NotContains(t, u.String(), "very-secret-uuid")
+}
+
+// TestListNonRecursiveSingleRoundTrip checks that a non recursive
+// listing needs a single API request: without depth=unlimited the API
+// returns direct children only, so neither the parent item lookup nor
+// a child filter is needed
+func TestListNonRecursiveSingleRoundTrip(t *testing.T) {
+	var requests int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&requests, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/files"):
+			// Direct children only, as returned without depth=unlimited
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":[`+
+				`{"id":10,"name":"sub","type":"dir","path":"/sub"},`+
+				`{"id":11,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:12345678","last_modified_at":1700000000}],"has_more":false}`)
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+
+	entries, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&requests), "non recursive listing must make a single API request")
+
+	names := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		names = append(names, entry.Remote())
+	}
+	assert.ElementsMatch(t, []string{"sub", "file.txt"}, names)
+}
+
+// TestMoveConflictReturnsError checks that moving onto an occupied
+// destination reports the API conflict instead of resolving it: rclone
+// removes the destination itself before calling Move
+func TestMoveConflictReturnsError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"conflict_error","description":"destination already exists"}}`)
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	_, err := f.Move(context.Background(), srcObj, "file2")
+	require.Error(t, err, "a move onto an occupied destination must fail")
+	assert.Contains(t, err.Error(), "conflict", "the API conflict must be reported")
+}
+
+// TestMoveDetectsForeignObjectAtDestination checks that Move does not
+// return a file re-created by another writer at the destination
+func TestMoveDetectsForeignObjectAtDestination(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			// The move API reports success...
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
+			// ...but the file at the destination is not the moved
+			// file: another writer re-created it
+			_, _ = fmt.Fprintln(w, kdriveMockItem(999, "file2"))
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	_, err := f.Move(context.Background(), srcObj, "file2")
+	require.Error(t, err, "Move must not report success when the destination is not the moved file")
+}
+
+// TestMoveSuccess checks the nominal move: the object returned is the
+// moved file (same ID as the source).
+func TestMoveSuccess(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
+			// After the move, the moved file is at the destination
+			_, _ = fmt.Fprintln(w, kdriveMockItem(111, "file2"))
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	dstObj, err := f.Move(context.Background(), srcObj, "file2")
+	require.NoError(t, err)
+	moved, ok := dstObj.(*Object)
+	require.True(t, ok)
+	assert.Equal(t, "111", moved.id, "the returned object must be the moved file")
+}
+
+// TestFormatModTime checks that pre-1970 times format as negative
+// unix timestamps instead of wrapping through a uint64 cast
+func TestFormatModTime(t *testing.T) {
+	for _, test := range []struct {
+		in   time.Time
+		want string
+	}{
+		{in: time.Unix(1700000000, 0), want: "1700000000"},
+		{in: time.Unix(0, 0), want: "0"},
+		{in: time.Unix(-1577923200, 0), want: "-1577923200"}, // Sat Jan 2 1965
+	} {
+		t.Run(test.want, func(t *testing.T) {
+			assert.Equal(t, test.want, formatModTime(test.in))
+		})
+	}
+}
+
+// TestObjectSetMetaDataIsDir checks that directories are rejected
+func TestObjectSetMetaDataIsDir(t *testing.T) {
+	o := &Object{remote: "dir"}
+	info := &api.Item{ID: 42, Name: "dir", Type: "dir"}
+	err := o.setMetaData(info)
+	assert.Equal(t, fs.ErrorIsDir, err)
+}
+
+// TestShouldRetryLockError checks that transient locks held by server
+// side copies are retried while locks held by applications and other
+// errors are not
+func TestShouldRetryLockError(t *testing.T) {
+	for _, test := range []struct {
+		name      string
+		body      string
+		status    int
+		wantRetry bool
+	}{
+		{
+			name:      "lock held by internal copy",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy","context":{"values":["internal_copy"]}}}`,
+			status:    http.StatusForbidden,
+			wantRetry: true,
+		},
+		{
+			name:      "lock held by an application",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by onlyoffice","context":{"values":["onlyoffice"]}}}`,
+			status:    http.StatusForbidden,
+			wantRetry: false,
+		},
+		{
+			name:      "lock without context",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy"}}`,
+			status:    http.StatusForbidden,
+			wantRetry: false,
+		},
+		{
+			name:      "other api error",
+			body:      `{"result":"error","error":{"code":"object_not_found","description":"File not found"}}`,
+			status:    http.StatusNotFound,
+			wantRetry: false,
+		},
+		{
+			name:      "server error",
+			status:    http.StatusServiceUnavailable,
+			wantRetry: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var apiErr error
+			if test.body != "" {
+				e := new(api.ResultStatus)
+				require.NoError(t, json.Unmarshal([]byte(test.body), e))
+				apiErr = e
+			}
+			resp := &http.Response{StatusCode: test.status, Header: http.Header{}}
+			got, gotErr := shouldRetry(context.Background(), resp, apiErr)
+			assert.Equal(t, test.wantRetry, got)
+			assert.Equal(t, apiErr, gotErr)
+		})
+	}
+}

backend/kdrive/kdrive_test.go

@@ -0,0 +1,25 @@
+// Test kdrive filesystem interface
+package kdrive
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestKdrive:",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+			MaxChunkSize: maxChunkSize,
+		},
+	})
+}
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setChunkSize(cs)
+}

backend/kdrive/khash/khash.go

@@ -0,0 +1,67 @@
+// Package khash provides the kDrive specific hash handling.
+//
+// kDrive uses XXH3-64 for file hashing. Files uploaded in chunks (by
+// rclone, or by the kDrive apps which each use their own chunk size)
+// get a "nested" hash: the XXH3 of the concatenated lowercase hex
+// strings of the individual chunk hashes. Such a nested hash can't be
+// compared with a plain XXH3 of the file contents.
+//
+// Hash formats returned by the kDrive API:
+//   - "xxh3:HASH" for a plain XXH3 of the file contents
+//   - "N:xxh3:HASH" for the nested hash of a chunked upload
+package khash
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/zeebo/xxh3"
+)
+
+// ParseHash parses a kDrive hash string from the API and returns the hex hash value
+// along with whether it is a nested (chunked) hash.
+func ParseHash(hashStr string) (hexHash string, isNested bool, err error) {
+	if hashStr == "" {
+		return "", false, nil
+	}
+
+	// Check for nested hash format: "N:xxh3:HASH"
+	if strings.HasPrefix(hashStr, "N:xxh3:") {
+		hashStr = strings.TrimPrefix(hashStr, "N:xxh3:")
+		isNested = true
+	} else {
+		hashStr = strings.TrimPrefix(hashStr, "xxh3:")
+	}
+
+	// Validate hex string
+	if _, decodeErr := hex.DecodeString(hashStr); decodeErr != nil {
+		return "", false, fmt.Errorf("invalid hash format: %w", decodeErr)
+	}
+
+	return hashStr, isNested, nil
+}
+
+// NestedChunkHash computes the kDrive nested hash from the hashes of the
+// file chunks, in order.
+//
+// kDrive stores the nested hash of a chunked file as the XXH3 of the
+// concatenated lowercase hex strings of the individual chunk hashes.
+// Each chunk hash may be passed with or without the "xxh3:" prefix.
+func NestedChunkHash(chunkHashes []string) (string, error) {
+	if len(chunkHashes) == 0 {
+		return "", errors.New("no chunk hashes to nest")
+	}
+	hexHashes := make([]string, len(chunkHashes))
+	for i, chunkHash := range chunkHashes {
+		hexHash, _, err := ParseHash(chunkHash)
+		if err != nil {
+			return "", fmt.Errorf("invalid chunk hash %q: %w", chunkHash, err)
+		}
+		hexHashes[i] = hexHash
+	}
+	hasher := xxh3.New()
+	_, _ = hasher.Write([]byte(strings.Join(hexHashes, "")))
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}

backend/kdrive/khash/khash_test.go

@@ -0,0 +1,150 @@
+package khash
+
+import "testing"
+
+func TestParseHash(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        string
+		expectedHash string
+		isNested     bool
+		expectError  bool
+	}{
+		{
+			name:         "simple hash with prefix",
+			input:        "xxh3:1db797b96febd334",
+			expectedHash: "1db797b96febd334",
+			isNested:     false,
+		},
+		{
+			name:         "nested hash",
+			input:        "N:xxh3:877abf3579f0a5c0",
+			expectedHash: "877abf3579f0a5c0",
+			isNested:     true,
+		},
+		{
+			name:         "plain hex",
+			input:        "1db797b96febd334",
+			expectedHash: "1db797b96febd334",
+			isNested:     false,
+		},
+		{
+			name:         "empty string",
+			input:        "",
+			expectedHash: "",
+			isNested:     false,
+		},
+		{
+			name:        "invalid hex",
+			input:       "xxh3:notahexvalue",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hash, isNested, err := ParseHash(tt.input)
+
+			if tt.expectError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if hash != tt.expectedHash {
+				t.Errorf("Hash: got %q, expected %q", hash, tt.expectedHash)
+			}
+			if isNested != tt.isNested {
+				t.Errorf("IsNested: got %t, expected %t", isNested, tt.isNested)
+			}
+		})
+	}
+}
+
+func TestNestedChunkHash(t *testing.T) {
+	// Golden values: the nested hash is the XXH3 of the concatenated
+	// lowercase hex strings of the chunk hashes, in order
+	chunk1 := "xxh3:d447b1ea40e6988b" // XXH3 of "hello world"
+	chunk2 := "2d06800538d394c2"      // XXH3 of ""
+
+	tests := []struct {
+		name        string
+		chunkHashes []string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:        "two chunks in order",
+			chunkHashes: []string{chunk1, chunk2},
+			expected:    "6d51edccc9c05a8b",
+		},
+		{
+			name:        "order matters",
+			chunkHashes: []string{chunk2, chunk1},
+			expected:    "596c127c7bcc8820",
+		},
+		{
+			name:        "single chunk",
+			chunkHashes: []string{chunk1},
+			expected:    "b73081dff8fe443c",
+		},
+		{
+			name:        "mixed prefixes",
+			chunkHashes: []string{chunk1, "xxh3:" + chunk2, chunk1},
+			expected:    "ca0b13abcdcd1661",
+		},
+		{
+			name:        "no chunks",
+			chunkHashes: []string{},
+			expectError: true,
+		},
+		{
+			name:        "invalid chunk hash",
+			chunkHashes: []string{chunk1, "notahexvalue"},
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nested, err := NestedChunkHash(tt.chunkHashes)
+
+			if tt.expectError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if nested != tt.expected {
+				t.Errorf("NestedChunkHash: got %q, expected %q", nested, tt.expected)
+			}
+		})
+	}
+}
+
+func TestNestedChunkHashAcceptsPrefixedHashes(t *testing.T) {
+	plain := []string{"d447b1ea40e6988b", "2d06800538d394c2"}
+	prefixed := []string{"xxh3:d447b1ea40e6988b", "N:xxh3:2d06800538d394c2"}
+
+	gotPlain, err := NestedChunkHash(plain)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	gotPrefixed, err := NestedChunkHash(prefixed)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if gotPlain != gotPrefixed {
+		t.Errorf("Prefixed hashes should give the same nested hash: %q != %q", gotPlain, gotPrefixed)
+	}
+}

backend/kdrive/multipart.go

@@ -0,0 +1,383 @@
+package kdrive
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/backend/kdrive/api"
+	"github.com/rclone/rclone/backend/kdrive/khash"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/chunksize"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/zeebo/xxh3"
+	"golang.org/x/text/unicode/norm"
+)
+
+const (
+	// maxUploadParts is the maximum number of chunks the kDrive API accepts per file
+	maxUploadParts = 10000
+
+	// maxChunkSize is the largest chunk size the kDrive API accepts
+	maxChunkSize = 1 * 1000 * 1000 * 1000
+
+	// minChunkSize is the smallest chunk size the kDrive API accepts
+	minChunkSize = 1 * 1000 * 1000
+)
+
+// checkChunkSize checks the user supplied chunk_size against the
+// limits the kDrive API accepts
+func checkChunkSize(chunkSize fs.SizeSuffix) error {
+	if chunkSize < minChunkSize {
+		return fmt.Errorf("chunk_size %s is too small: the kDrive API requires chunks of at least 1 MB", chunkSize)
+	}
+	if chunkSize > maxChunkSize {
+		return fmt.Errorf("chunk_size %s is too large: the kDrive API accepts chunks of at most 1 GB (1000000000 bytes)", chunkSize)
+	}
+	return nil
+}
+
+// setChunkSize sets the chunk size, returning the old value
+func (f *Fs) setChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkChunkSize(cs)
+	if err == nil {
+		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	}
+	return
+}
+
+// formatModTime formats the modification time of an upload as the
+// unix timestamp string the kDrive API expects
+func formatModTime(modTime time.Time) string {
+	return fmt.Sprintf("%d", modTime.Unix())
+}
+
+// uploadSession implements fs.ChunkWriter for kdrive multipart uploads
+type uploadSession struct {
+	f         *Fs
+	token     string
+	uploadURL string
+	fileInfo  *api.Item
+	fileSize  int64 // total size of the file being uploaded
+	chunkSize int64 // size of the chunks of this session
+
+	mu          sync.Mutex
+	chunkHashes map[int64]string // chunk number -> hash of the chunk contents
+}
+
+// OpenChunkWriter returns the chunk size and a ChunkWriter
+//
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+	fileSize := src.Size()
+	if fileSize < 0 {
+		return info, nil, errors.New("kdrive can't upload files with unknown size")
+	}
+
+	dir, leaf := path.Split(remote)
+	dir = strings.TrimSuffix(dir, "/")
+
+	// Create parent directories if they don't exist
+	parentID, err := f.dirCache.FindDir(ctx, dir, true)
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to find parent directory: %w", err)
+	}
+
+	// Calculate the chunk size, honouring any fs.ChunkOption hint
+	chunkSize := f.opt.ChunkSize
+	for _, opt := range options {
+		if chunkOpt, ok := opt.(*fs.ChunkOption); ok && chunkOpt.ChunkSize > 0 {
+			chunkSize = fs.SizeSuffix(chunkOpt.ChunkSize)
+		}
+	}
+	chunkSize = chunksize.Calculator(src, fileSize, maxUploadParts, chunkSize)
+	if chunkSize < minChunkSize {
+		chunkSize = minChunkSize
+	}
+	if chunkSize > maxChunkSize {
+		chunkSize = maxChunkSize
+	}
+
+	// Calculate the total number of chunks
+	totalChunks := max((fileSize+int64(chunkSize)-1)/int64(chunkSize), 1)
+
+	sessionReq := struct {
+		Conflict       string `json:"conflict"`
+		DirectoryID    string `json:"directory_id"`
+		FileName       string `json:"file_name"`
+		LastModifiedAt string `json:"last_modified_at"`
+		TotalChunks    int64  `json:"total_chunks"`
+		TotalSize      int64  `json:"total_size"`
+	}{
+		Conflict:       "version",
+		DirectoryID:    parentID,
+		FileName:       f.opt.Enc.FromStandardName(norm.NFC.String(leaf)),
+		LastModifiedAt: formatModTime(src.ModTime(ctx)),
+		TotalChunks:    totalChunks,
+		TotalSize:      fileSize,
+	}
+
+	// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   fmt.Sprintf("/3/drive/%s/upload/session/start", f.opt.DriveID),
+	}
+	var sessionResp api.SessionStartResponse
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = f.srv.CallJSON(ctx, &opts, &sessionReq, &sessionResp)
+		callErr = sessionResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to start upload session: %w", err)
+	}
+
+	info = fs.ChunkWriterInfo{
+		ChunkSize:   int64(chunkSize),
+		Concurrency: f.opt.UploadConcurrency,
+	}
+
+	writer = &uploadSession{
+		f:           f,
+		token:       sessionResp.Data.Token,
+		uploadURL:   sessionResp.Data.UploadURL,
+		fileSize:    fileSize,
+		chunkSize:   int64(chunkSize),
+		chunkHashes: make(map[int64]string),
+	}
+
+	fs.Debugf(f, "open chunk writer: started upload session for %d chunks of %v", totalChunks, chunkSize)
+	return info, writer, nil
+}
+
+// WriteChunk uploads a single chunk
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
+func (u *uploadSession) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
+	if chunkNumber < 0 {
+		return -1, fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
+	}
+
+	// Calculate the size of this chunk from the session parameters so
+	// the chunk can be streamed to the API in a single pass
+	chunkLen := u.chunkSize
+	if remaining := u.fileSize - int64(chunkNumber)*u.chunkSize; remaining < chunkLen {
+		chunkLen = remaining
+	}
+	if chunkLen <= 0 {
+		return 0, nil
+	}
+
+	sourceChunkNumber := chunkNumber + 1 // kDrive API uses 1-based numbering
+
+	// The chunk_hash parameter is optional, so the chunk is streamed
+	// in a single pass, hashing the contents on the way with a
+	// TeeReader. The server returns the hash of the chunk it received,
+	// which is checked below. Retries seek back to the chunk start
+	// and hash with a fresh hasher.
+	var chunkHash string
+	err = u.f.pacer.Call(func() (bool, error) {
+		if _, seekErr := reader.Seek(0, io.SeekStart); seekErr != nil {
+			return false, fmt.Errorf("failed to rewind chunk %d: %w", sourceChunkNumber, seekErr)
+		}
+		chunkHasher := xxh3.New()
+
+		// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
+		chunkOpts := rest.Opts{
+			Method:  "POST",
+			RootURL: u.uploadURL,
+			Path:    fmt.Sprintf("/3/drive/%s/upload/session/%s/chunk", u.f.opt.DriveID, u.token),
+			Parameters: url.Values{
+				"chunk_number": {fmt.Sprintf("%d", sourceChunkNumber)},
+				"chunk_size":   {fmt.Sprintf("%d", chunkLen)},
+				"with":         {"hash"},
+			},
+			Body:          io.TeeReader(reader, chunkHasher),
+			ContentLength: &chunkLen,
+		}
+
+		var chunkResp api.ChunkUploadResponse
+		resp, callErr := u.f.srv.CallJSON(ctx, &chunkOpts, nil, &chunkResp)
+		callErr = chunkResp.ResultStatus.Update(callErr)
+		if callErr != nil {
+			return shouldRetry(ctx, resp, callErr)
+		}
+
+		chunkHash = hex.EncodeToString(chunkHasher.Sum(nil))
+
+		// The server hashed the chunk it received, so compare with what we
+		// sent to detect any corruption immediately
+		if chunkResp.Data.Hash != "" {
+			if serverHash, _, parseErr := khash.ParseHash(chunkResp.Data.Hash); parseErr == nil {
+				if !strings.EqualFold(chunkHash, serverHash) {
+					return false, fmt.Errorf("chunk %d hash mismatch: client=%s, server=%s", sourceChunkNumber, chunkHash, serverHash)
+				}
+			} else {
+				fs.Debugf(u, "Failed to parse server chunk hash %q: %v", chunkResp.Data.Hash, parseErr)
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return -1, err
+	}
+
+	u.mu.Lock()
+	u.chunkHashes[int64(chunkNumber)] = chunkHash
+	u.mu.Unlock()
+
+	fs.Debugf(u, "uploaded chunk %d (size: %d, hash: %s)", sourceChunkNumber, chunkLen, chunkHash)
+	return chunkLen, nil
+}
+
+// Close finalizes the upload session, verifies the hash of the uploaded
+// file and returns the created file info
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
+func (u *uploadSession) Close(ctx context.Context) error {
+	// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   fmt.Sprintf("/3/drive/%s/upload/session/%s/finish", u.f.opt.DriveID, u.token),
+	}
+	var resp *http.Response
+	var finishResp api.SessionFinishResponse
+	err := u.f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &finishResp)
+		callErr = finishResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to finish upload session: %w", err)
+	}
+
+	u.fileInfo = &finishResp.Data.File
+	fs.Debugf(u, "multipart upload completed: file id %d", finishResp.Data.File.ID)
+
+	return u.verifyHash(ctx)
+}
+
+// verifyHash checks the hash of the uploaded file against the hashes of
+// the chunks we uploaded, deleting the file if it is corrupted
+func (u *uploadSession) verifyHash(ctx context.Context) (err error) {
+	remoteHash := u.fileInfo.Hash
+	if remoteHash == "" {
+		// Hash might be missing in the finish response, try to fetch it
+		obj := &Object{
+			fs: u.f,
+			id: strconv.Itoa(u.fileInfo.ID),
+		}
+		var hashErr error
+		if remoteHash, hashErr = obj.retrieveHash(ctx); hashErr != nil {
+			// skip verification
+			fs.Debugf(u, "Failed to retrieve hash for verification: %v", hashErr)
+			return nil
+		}
+	}
+	parsedRemoteHash, isNested, err := khash.ParseHash(remoteHash)
+	if err != nil {
+		fs.Debugf(u, "Failed to parse server hash %q: %v", remoteHash, err)
+		return nil
+	}
+
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	// Rebuild the hash the server should have stored from the chunk
+	// hashes we uploaded, in order
+	chunkCount := int64(len(u.chunkHashes))
+	if chunkCount == 0 {
+		return nil
+	}
+	chunkHashes := make([]string, 0, chunkCount)
+	for i := int64(0); i < chunkCount; i++ {
+		chunkHash, ok := u.chunkHashes[i]
+		if !ok {
+			fs.Debugf(u, "Chunk %d hash missing, skipping hash verification", i)
+			return nil
+		}
+		chunkHashes = append(chunkHashes, chunkHash)
+	}
+
+	var localHash string
+	if isNested {
+		if localHash, err = khash.NestedChunkHash(chunkHashes); err != nil {
+			fs.Debugf(u, "Failed to compute nested hash: %v", err)
+			return nil
+		}
+	} else if chunkCount == 1 {
+		localHash, _, _ = khash.ParseHash(chunkHashes[0])
+	} else {
+		// The server returned a simple hash for a multi chunk upload,
+		// nothing to compare it against
+		fs.Debugf(u, "Server returned a simple hash for a %d chunk upload, skipping hash verification", chunkCount)
+		return nil
+	}
+
+	if !strings.EqualFold(localHash, parsedRemoteHash) {
+		err = fmt.Errorf(
+			"multipart upload hash mismatch: local=%s, remote=%s",
+			localHash, parsedRemoteHash,
+		)
+
+		// Remove the corrupted file
+		obj := &Object{
+			fs: u.f,
+			id: strconv.Itoa(u.fileInfo.ID),
+		}
+		if delErr := obj.Remove(ctx); delErr != nil {
+			fs.Errorf(nil, "Failed to remove corrupted file after hash mismatch: %v", delErr)
+		} else {
+			fs.Debugf(nil, "Removed corrupted file after hash mismatch")
+		}
+
+		return err
+	}
+	fs.Debugf(u, "Multipart upload hash verified: %s", localHash)
+
+	return nil
+}
+
+// Abort the upload session
+// @see  https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
+func (u *uploadSession) Abort(ctx context.Context) error {
+	// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
+	opts := rest.Opts{
+		Method: "DELETE",
+		Path:   fmt.Sprintf("/2/drive/%s/upload/session/%s", u.f.opt.DriveID, u.token),
+	}
+	var resp *http.Response
+	var cancelResp api.SessionCancelResponse
+	err := u.f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &cancelResp)
+		callErr = cancelResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to cancel upload session: %w", err)
+	}
+
+	fs.Debugf(u, "upload session cancelled")
+	return nil
+}
+
+// String implements fmt.Stringer
+func (u *uploadSession) String() string {
+	return "kdrive upload session"
+}
+
+// Verify interface compliance
+var _ fs.ChunkWriter = (*uploadSession)(nil)

backend/local/clone_darwin.go

@@ -6,11 +6,13 @@ package local
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"runtime"
 
 	"github.com/go-darwin/apfs"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/random"
 )
 
 // Copy src to this remote using server-side copy operations.
@@ -63,7 +65,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = Clone(srcPath, dstPath)
+	err = cloneOver(srcPath, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -79,6 +81,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
+// cloneOver clones src to dst, replacing dst if it already exists.
+//
+// COPYFILE_CLONE implies COPYFILE_EXCL so Clone fails if dst exists.
+// Cloning to a temporary name in the same directory and renaming it
+// over dst avoids this and leaves dst untouched if the clone fails.
+func cloneOver(src, dst string) error {
+	if _, err := os.Lstat(dst); os.IsNotExist(err) {
+		return Clone(src, dst)
+	}
+	tmp := filepath.Join(filepath.Dir(dst), "."+random.String(16)+".rclone-clone")
+	err := Clone(src, tmp)
+	if err == nil {
+		err = os.Rename(tmp, dst)
+	}
+	if err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("failed to clone %q to %q: %w", src, dst, err)
+	}
+	return nil
+}
+
 // Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)
 // note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical
 // https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html

backend/local/local.go

@@ -12,6 +12,7 @@ import (
 	"path"
 	"path/filepath"
 	"runtime"
+	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -36,6 +37,10 @@ import (
 const (
 	devUnset   = 0xdeadbeefcafebabe                                     // a device id meaning it is unset
 	useReadDir = (runtime.GOOS == "windows" || runtime.GOOS == "plan9") // these OSes read FileInfos directly
+	// maxLinkTargetSize is the largest symlink target accepted when
+	// translating a .rclonelink object, comfortably above any OS path
+	// limit (Windows allows 32767 UTF-16 units).
+	maxLinkTargetSize = 128 * 1024
 )
 
 // timeType allows the user to choose what exactly ModTime() returns
@@ -82,8 +87,10 @@ Metadata is supported on files and directories.
 When restoring metadata with ` + "`--metadata`" + ` rclone applies the
 "mode", "uid" and "gid" from the source. These come from the source
 remote which may not be trusted, so restoring metadata as root from an
-untrusted source can change file ownership and is not recommended. The
-setuid, setgid and sticky bits are not restored by default - see the
+untrusted source can change file ownership and is not recommended. A
+user other than root keeps its own uid and sets the gid only to a group
+it is a member of, as rsync does. The setuid, setgid and sticky bits are
+not restored by default - see the
 ` + "`--local-metadata-restore-special-bits`" + ` flag.
 `,
 		},
@@ -247,6 +254,13 @@ deduplicated by default (i.e. having two identical files does not consume more
 storage than having just one.)  However, for use cases where data redundancy is
 preferable, --local-no-clone can be used to disable cloning and force "deep" copies.
 
+Cloning also preserves macOS-specific metadata such as extended attributes,
+which a normal copy does not, so this metadata is lost when cloning is disabled.
+
+If the destination file already exists, cloning replaces it with a new file,
+even with --inplace, so hard links to it are broken. Use --local-no-clone to
+update existing files in place.
+
 Currently, cloning is only supported when using APFS on macOS (support for other
 platforms may be added in the future.)`,
 				Default:  false,
@@ -438,6 +452,9 @@ var (
 	errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
 	errLinksNeedsSuffix  = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
 	errPathEscapes       = errors.New("file name is not a path within the local root - check the encoding")
+	errSymlinkLoop       = errors.New("loop detected: points to a parent directory")
+	errLinkTargetTooLong = errors.New("symlink target is too long to be a path")
+	errLinkNameInvalid   = errors.New("translated link needs a file name before the \"" + fs.LinkSuffix + "\" suffix")
 )
 
 // NewFs constructs an Fs from the path
@@ -565,30 +582,26 @@ func (f *Fs) caseInsensitive() bool {
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
-// translateLink checks whether the remote is a translated link
-// and returns a new path, removing the suffix as needed,
-// It also returns whether this is a translated link at all
-//
-// for regular files, localPath is returned unchanged
-func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
-	isTranslatedLink = strings.HasSuffix(remote, fs.LinkSuffix)
-	newLocalPath = strings.TrimSuffix(localPath, fs.LinkSuffix)
-	return newLocalPath, isTranslatedLink
-}
-
 // newObject makes a half completed Object
 func (f *Fs) newObject(remote string) (*Object, error) {
-	translatedLink := false
-	localPath, err := f.localPath(remote)
+	name, translatedLink := remote, false
+	if f.opt.TranslateSymlinks {
+		name, translatedLink = strings.CutSuffix(remote, fs.LinkSuffix)
+	}
+	if translatedLink {
+		// Check this makes a valid native file name
+		native := f.nativePath(name)
+		switch native[strings.LastIndex(native, string(filepath.Separator))+1:] {
+		case "", ".", "..":
+			return nil, fserrors.NoRetryError(fmt.Errorf("%q: %w", remote, errLinkNameInvalid))
+		}
+	}
+	// Check the name after any suffix removal
+	localPath, err := f.localPath(name)
 	if err != nil {
 		return nil, err
 	}
 
-	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for localPath
-		localPath, translatedLink = translateLink(remote, localPath)
-	}
-
 	return &Object{
 		fs:             f,
 		remote:         remote,
@@ -637,15 +650,23 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 }
 
 // Create new directory object from the info passed in
+//
+// Directories are never translated links, so the name is used as is
+// whatever suffix it has.
 func (f *Fs) newDirectory(dir string, fi os.FileInfo) (*Directory, error) {
-	o, err := f.newObject(dir)
+	localPath, err := f.localPath(dir)
 	if err != nil {
 		return nil, err
 	}
-	o.setMetadata(fi)
-	return &Directory{
-		Object: *o,
-	}, nil
+	d := &Directory{
+		Object: Object{
+			fs:     f,
+			remote: dir,
+			path:   localPath,
+		},
+	}
+	d.setMetadata(fi)
+	return d, nil
 }
 
 // List the objects and directories in dir into entries.  The
@@ -668,6 +689,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if err != nil {
 		return nil, fs.ErrorDirNotFound
 	}
+	var parents []os.FileInfo // fsDirPath and its parents, read on first use
 
 	fd, err := os.Open(fsDirPath)
 	if err != nil {
@@ -748,11 +770,28 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if f.opt.FollowSymlinks && (mode&symlinkFlag) != 0 {
 				localPath := filepath.Join(fsDirPath, name)
 				fi, err = os.Stat(localPath)
+				// A directory symlink pointing to a parent would be followed
+				// until the OS returns ELOOP, which can take a very long time
+				if err == nil && fi.IsDir() {
+					if parents == nil {
+						parents = statParents(fsDirPath)
+					}
+					if slices.ContainsFunc(parents, func(parent os.FileInfo) bool { return os.SameFile(fi, parent) }) {
+						// Quietly skip loops the directory filters exclude as
+						// the layer above wouldn't recurse into them
+						if useFilter {
+							if include, dirErr := filter.IncludeDirectory(ctx, f)(newRemote); dirErr == nil && !include {
+								continue
+							}
+						}
+						err = errSymlinkLoop
+					}
+				}
 				// Quietly skip errors on excluded files and directories
 				if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
 					continue
 				}
-				if os.IsNotExist(err) || isCircularSymlinkError(err) {
+				if os.IsNotExist(err) || isCircularSymlinkError(err) || errors.Is(err, errSymlinkLoop) {
 					// Skip bad symlinks and circular symlinks
 					err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
 					fs.Errorf(newRemote, "Listing error: %v", err)
@@ -785,6 +824,12 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					continue
 				}
 				fso, err := f.newObjectWithInfo(newRemote, fi)
+				if errors.Is(err, errLinkNameInvalid) {
+					// Skip a badly named ".rclonelink" file
+					fs.Errorf(newRemote, "Listing error: %v", err)
+					_ = accounting.Stats(ctx).Error(err)
+					continue
+				}
 				if err != nil {
 					return nil, err
 				}
@@ -797,6 +842,21 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return entries, nil
 }
 
+// statParents returns the info for dirPath and each of its parents up
+// to the root of the file system, skipping any which can't be read.
+func statParents(dirPath string) (parents []os.FileInfo) {
+	for {
+		if fi, err := os.Stat(dirPath); err == nil {
+			parents = append(parents, fi)
+		}
+		parent := filepath.Dir(dirPath)
+		if parent == dirPath {
+			return parents
+		}
+		dirPath = parent
+	}
+}
+
 func (f *Fs) cleanRemote(dir, filename string) (remote string) {
 	if f.opt.UTFNorm {
 		filename = norm.NFC.String(filename)
@@ -814,14 +874,19 @@ func (f *Fs) cleanRemote(dir, filename string) (remote string) {
 	return
 }
 
+// nativePath returns name in the OS encoding with OS path separators,
+// relative to f.root, without cleaning it.
+func (f *Fs) nativePath(name string) string {
+	return filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
+}
+
 // localPath returns the OS path for the object called name, which is
 // always underneath f.root.
 //
 // It returns errPathEscapes if name resolves outside f.root which can
 // happen depending on the encoding.
 func (f *Fs) localPath(name string) (string, error) {
-	native := filepath.FromSlash(f.opt.Enc.FromStandardPath(name))
-	localPath := filepath.Join(f.root, native)
+	localPath := filepath.Join(f.root, f.nativePath(name))
 	rel, err := filepath.Rel(f.root, localPath)
 	if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) {
 		return "", fserrors.NoRetryError(fmt.Errorf("%q: %w", name, errPathEscapes))
@@ -895,13 +960,13 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
 	if err != nil {
 		return nil, err
 	}
-	fi, err := f.lstat(localPath)
+	fi, err := f.lstatRoot(localPath)
 	if errors.Is(err, os.ErrNotExist) {
 		err := f.Mkdir(ctx, dir)
 		if err != nil {
 			return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
 		}
-		fi, err = f.lstat(localPath)
+		fi, err = f.lstatRoot(localPath)
 		if err != nil {
 			return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
 		}
@@ -933,20 +998,34 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
 // Rmdir removes the directory
 //
 // If it isn't empty it will return an error
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	localPath, err := f.localPath(dir)
 	if err != nil {
 		return err
 	}
-	if fi, err := os.Stat(localPath); err != nil {
+	// When translating symlinks go through os.Root so a symlink planted at
+	// any parent can't redirect the removal out of the destination, and
+	// don't follow the leaf as a symlink is a file in this mode. The root
+	// itself is the trusted destination and is removed directly.
+	stat, removeFn, chmod := os.Stat, os.Remove, os.Chmod
+	if f.opt.TranslateSymlinks && localPath != f.root {
+		var root *os.Root
+		root, localPath, err = f.osRoot(localPath)
+		if err != nil {
+			return err
+		}
+		defer fs.CheckClose(root, &err)
+		stat, removeFn, chmod = root.Lstat, root.Remove, root.Chmod
+	}
+	if fi, err := stat(localPath); err != nil {
 		return err
 	} else if !fi.IsDir() {
 		return fs.ErrorIsFile
 	}
-	err = os.Remove(localPath)
+	err = removeFn(localPath)
 	if runtime.GOOS == "windows" && errors.Is(err, iofs.ErrPermission) { // https://github.com/golang/go/issues/26295
-		if os.Chmod(localPath, 0o600) == nil {
-			err = os.Remove(localPath)
+		if chmod(localPath, 0o600) == nil {
+			err = removeFn(localPath)
 		}
 	}
 	return err
@@ -1123,9 +1202,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	// Create parent of destination
+	// Create parent of destination. The root itself is the trusted
+	// destination so its parent, which lies outside it, is created directly.
 	dstParentPath := filepath.Dir(dstPath)
-	err = file.MkdirAll(dstParentPath, 0777)
+	if dstPath == f.root {
+		err = file.MkdirAll(dstParentPath, 0777)
+	} else {
+		err = f.mkdirAll(dstParentPath)
+	}
 	if err != nil {
 		return err
 	}
@@ -1542,10 +1626,26 @@ func (f *Fs) mkdirAll(localPath string) (err error) {
 	return root.MkdirAll(rel, 0777)
 }
 
-// openFile opens localPath for writing. When translating symlinks it goes
-// through os.Root so a symlink planted at the path, or at any parent, is never
-// followed out of the destination. The returned file is independent of the
-// root, which is closed before returning.
+// lstatRoot returns the info for localPath without following a symlink at
+// the leaf. When translating symlinks it goes through os.Root so a symlink
+// planted at any parent can't make a path outside the destination look like
+// it exists, which would let its metadata be written there.
+func (f *Fs) lstatRoot(localPath string) (fi os.FileInfo, err error) {
+	if !f.opt.TranslateSymlinks {
+		return f.lstat(localPath)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return nil, err
+	}
+	defer fs.CheckClose(root, &err)
+	return root.Lstat(rel)
+}
+
+// openFile opens localPath with the given flags and perm. When translating
+// symlinks it goes through os.Root so a symlink planted at the path, or at any
+// parent, is never followed out of the destination. The returned file is
+// independent of the root, which is closed before returning.
 func (f *Fs) openFile(localPath string, flags int, perm os.FileMode) (fi *os.File, err error) {
 	if !f.opt.TranslateSymlinks {
 		return file.OpenFile(localPath, flags, perm)
@@ -1614,6 +1714,21 @@ func (f *Fs) chtimes(localPath string, atime, mtime time.Time) (err error) {
 	return root.Chtimes(rel, atime, mtime)
 }
 
+// remove removes the file, symlink or empty directory at localPath. When
+// translating symlinks it goes through os.Root so a symlink planted at any
+// parent can't redirect the removal out of the destination.
+func (f *Fs) remove(localPath string) (err error) {
+	if !f.opt.TranslateSymlinks {
+		return remove(localPath, os.Remove)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(root, &err)
+	return remove(rel, root.Remove)
+}
+
 // mkdirAll makes all the directories needed to store the object
 func (o *Object) mkdirAll() error {
 	return o.fs.mkdirAll(filepath.Dir(o.path))
@@ -1701,6 +1816,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 		out = f
 	} else {
+		// The target is buffered in memory, so stop reading just past
+		// the longest acceptable one rather than trust the source
+		in = io.LimitReader(in, maxLinkTargetSize+1)
 		out = nopWriterCloser{&symlinkData}
 	}
 
@@ -1717,6 +1835,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	if o.translatedLink {
+		if err == nil && symlinkData.Len() > maxLinkTargetSize {
+			err = fserrors.NoRetryError(errLinkTargetTooLong)
+		}
 		if err == nil {
 			// Use the contents of the copied object to create a symlink,
 			// without following or creating it through a planted symlink
@@ -1732,7 +1853,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
-		if removeErr := os.Remove(o.path); removeErr != nil {
+		if removeErr := o.fs.remove(o.path); removeErr != nil {
 			fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
 		}
 		return err
@@ -1880,7 +2001,7 @@ func (o *Object) lstat() error {
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	o.clearHashCache()
-	return remove(o.path)
+	return o.fs.remove(o.path)
 }
 
 // Metadata returns metadata for an object

backend/local/local_internal_test.go

@@ -3,6 +3,7 @@ package local
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -23,6 +24,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
@@ -97,6 +99,31 @@ func TestVerifyCopy(t *testing.T) {
 	assert.Error(t, err)
 }
 
+// Test server-side copy over an existing file
+func TestCopyOverExisting(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	doCopy := r.Flocal.Features().Copy
+	if doCopy == nil {
+		t.Skip("local backend has no server-side copy on this platform")
+	}
+	t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	t2 := fstest.Time("2002-02-03T04:05:06.499999999Z")
+	file1 := r.WriteFile("a.txt", "hello1", t1)
+	file2 := r.WriteFile("b.txt", "HELLO2 longer", t2)
+
+	src, err := r.Flocal.NewObject(ctx, file1.Path)
+	require.NoError(t, err)
+	dst, err := doCopy(ctx, src, file2.Path)
+	require.NoError(t, err)
+	assert.Equal(t, file2.Path, dst.Remote())
+
+	file2.Size = file1.Size
+	file2.Hashes = file1.Hashes
+	file2.ModTime = file1.ModTime
+	r.CheckLocalItems(t, file1, file2)
+}
+
 // skipIfNoSymlinks skips the test if this process can't create symlinks.
 //
 // Windows grants the privilege only to an elevated process or one running with
@@ -270,6 +297,29 @@ func linksMode(f *Fs) {
 	f.lstat = os.Lstat
 }
 
+// TestSymlinkTargetTooLong checks that a .rclonelink object whose body is
+// far bigger than any path is refused without being buffered in memory,
+// and that nothing is created at the destination.
+func TestSymlinkTargetTooLong(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// A source which never ends, so the read must be bounded
+	src := object.NewStaticObjectInfo("big"+fs.LinkSuffix, fstest.Time("2001-02-03T04:05:10Z"), -1, true, nil, nil)
+	in := readers.NewCountingReader(readers.NewPatternReader(1 << 40))
+	_, err := f.Put(ctx, in, src)
+	require.ErrorIs(t, err, errLinkTargetTooLong)
+	assert.True(t, fserrors.IsNoRetryError(err))
+	assert.LessOrEqual(t, in.BytesRead(), uint64(maxLinkTargetSize+1), "read more of the body than needed")
+
+	_, err = os.Lstat(filepath.Join(f.root, "big"))
+	assert.True(t, os.IsNotExist(err), "nothing should be created for a refused symlink")
+}
+
 // TestSymlinkEscapeWriteThroughBlocked mirrors the GHSA-cf44-9pgv-m4xc PoC: a
 // malicious --links source serves "pwn.rclonelink" whose body is a path outside
 // the destination, plus a sibling "pwn/authkeys" that sorts after it and would
@@ -510,6 +560,115 @@ func TestDirBTimeThroughPlantedSymlinkBlocked(t *testing.T) {
 	require.Equal(t, before, readBTime(), "btime escaped through planted symlink to %q", evilDir)
 }
 
+// TestMkdirMetadataThroughPlantedSymlinkBlocked checks MkdirMetadata can't
+// be fooled into treating a directory outside the root, reached through a
+// planted symlink, as the directory to apply metadata to.
+func TestMkdirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	require.NoError(t, os.Mkdir(filepath.Join(evilDir, "sub"), 0700))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: an existing in-tree directory is found as normal.
+	require.NoError(t, f.Mkdir(ctx, "sub"))
+	_, err := f.MkdirMetadata(ctx, "sub", nil)
+	require.NoError(t, err)
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// "pwn/sub" exists outside the root but must not be found through the
+	// planted symlink, and must not be created there either.
+	_, err = f.MkdirMetadata(ctx, "pwn/sub", nil)
+	require.Error(t, err, "finding a directory through a planted symlink should be refused")
+	_, err = f.MkdirMetadata(ctx, "pwn/new", nil)
+	require.Error(t, err, "making a directory through a planted symlink should be refused")
+	_, err = os.Lstat(filepath.Join(evilDir, "new"))
+	require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evilDir)
+}
+
+// TestRemoveThroughPlantedSymlinkBlocked checks Remove and Rmdir can't
+// delete anything outside the root through a planted symlink.
+func TestRemoveThroughPlantedSymlinkBlocked(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		// os.Root.Symlink makes a file symlink to a directory outside
+		// the root, which Windows can't traverse, so the planted link
+		// can't reach the outside files.
+		t.Skip("planted directory symlinks can't be traversed on Windows")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilFile := filepath.Join(evil, "secret.txt")
+	require.NoError(t, os.WriteFile(evilFile, []byte("keep me"), 0600))
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: in-tree removals work as normal.
+	require.NoError(t, putFile(ctx, f, "file.txt", "data"))
+	o, err := f.NewObject(ctx, "file.txt")
+	require.NoError(t, err)
+	require.NoError(t, o.Remove(ctx))
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	require.NoError(t, f.Rmdir(ctx, "dir"))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evil))
+
+	// Removing the outside file and directory named through the planted
+	// symlink must be refused.
+	o, err = f.NewObject(ctx, "pwn/secret.txt")
+	require.NoError(t, err)
+	err = o.Remove(ctx)
+	require.Error(t, err, "removing through a planted symlink should be refused")
+	_, err = os.Lstat(evilFile)
+	require.NoError(t, err, "a file was removed outside the destination at %q", evilFile)
+
+	err = f.Rmdir(ctx, "pwn/secret.d")
+	require.Error(t, err, "removing a directory through a planted symlink should be refused")
+	_, err = os.Lstat(evilDir)
+	require.NoError(t, err, "a directory was removed outside the destination at %q", evilDir)
+}
+
+// TestDirMoveThroughPlantedSymlinkBlocked checks DirMove can't move a
+// directory out of the root through a planted symlink.
+func TestDirMoveThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: an in-tree directory move works as normal.
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	require.NoError(t, f.DirMove(ctx, f, "dir", "sub/moved"))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evil))
+
+	err := f.DirMove(ctx, f, "sub/moved", "pwn/moved")
+	require.Error(t, err, "moving a directory through a planted symlink should be refused")
+	_, err = os.Lstat(filepath.Join(evil, "moved"))
+	require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evil)
+	_, err = os.Lstat(filepath.Join(f.root, "sub", "moved"))
+	require.NoError(t, err, "the directory should still be in the destination")
+}
+
 // TestEncodingEscapeBlocked checks that a name from a malicious source can't
 // be decoded into path syntax which writes outside the destination.
 func TestEncodingEscapeBlocked(t *testing.T) {
@@ -621,6 +780,154 @@ func TestLocalPath(t *testing.T) {
 	}
 }
 
+// TestLinkSuffixEscapeBlocked checks that in --links mode a name which
+// passes the root check with its ".rclonelink" suffix can't escape the
+// destination once the suffix is removed: "...rclonelink" must not
+// become "..", the parent of the root.
+func TestLinkSuffixEscapeBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+	outer := t.TempDir()
+	newLinksFs := func(dir, encoding string) *Fs {
+		fRaw, err := NewFs(ctx, "local", filepath.Join(outer, dir), configmap.Simple{"encoding": encoding})
+		require.NoError(t, err)
+		f := fRaw.(*Fs)
+		linksMode(f)
+		require.NoError(t, f.Mkdir(ctx, ""))
+		return f
+	}
+	f := newLinksFs("dst", encoder.OS.String())
+	s := newLinksFs("slash", "Slash")
+
+	// Stamp the parent of the destinations so a write to it shows up.
+	before := fstest.Time("2011-12-25T12:59:59Z")
+	require.NoError(t, os.Chtimes(outer, before, before))
+	parentUnchanged := func() {
+		t.Helper()
+		fi, err := os.Lstat(outer)
+		require.NoError(t, err)
+		assert.True(t, fi.ModTime().Equal(before), "the parent of the destination was modified")
+	}
+
+	// A directory called "...rclonelink" is created under its own name
+	// and its metadata goes there, not to the parent of the root which
+	// trimming the suffix off its path would address.
+	remote := ".." + fs.LinkSuffix
+	metadata := fs.Metadata{"mtime": "2001-02-03T04:05:06Z"}
+	_, err := f.MkdirMetadata(ctx, remote, metadata)
+	require.NoError(t, err)
+	parentUnchanged()
+
+	// A file called "...rclonelink" is a link called "..", which the
+	// default encoding stores as fullwidth dots like any other name.
+	require.NoError(t, putFile(ctx, f, remote, "target"))
+	fi, err := os.Lstat(filepath.Join(f.root, "．．"))
+	require.NoError(t, err)
+	assert.NotZero(t, fi.Mode()&os.ModeSymlink)
+	parentUnchanged()
+
+	// Ordinary translated links still resolve to the file without the suffix.
+	o, err := f.newObject("sub/link" + fs.LinkSuffix)
+	require.NoError(t, err)
+	assert.True(t, o.translatedLink)
+	assert.Equal(t, filepath.Join(f.root, "sub", "link"), o.path)
+
+	// A name which merely contains the suffix is a regular file.
+	o, err = f.newObject("sub/link" + fs.LinkSuffix + ".txt")
+	require.NoError(t, err)
+	assert.False(t, o.translatedLink)
+	assert.Equal(t, filepath.Join(f.root, "sub", "link"+fs.LinkSuffix+".txt"), o.path)
+
+	// An encoding without Dot leaves ".." as a real ".." once the suffix
+	// is gone, so the name is refused instead.
+	require.ErrorIs(t, putFile(ctx, s, remote, "PWNED"), errLinkNameInvalid)
+	_, err = s.MkdirMetadata(ctx, remote, metadata)
+	require.NoError(t, err)
+	parentUnchanged()
+
+	// As is any other name which doesn't leave a file name once the
+	// suffix is removed and the encoding applied.
+	for _, name := range []string{
+		remote,
+		"sub/.." + fs.LinkSuffix,
+		"sub/." + fs.LinkSuffix,
+		"sub/．．" + fs.LinkSuffix,
+		"sub/sub2/．．" + fs.LinkSuffix,
+		"sub/．" + fs.LinkSuffix,
+		"sub/" + fs.LinkSuffix,
+		fs.LinkSuffix,
+	} {
+		_, err = s.newObject(name)
+		assert.ErrorIs(t, err, errLinkNameInvalid, name)
+		assert.True(t, fserrors.IsNoRetryError(err), name)
+	}
+
+	// The root check still applies to the name of the link.
+	_, err = s.newObject("../link" + fs.LinkSuffix)
+	require.ErrorIs(t, err, errPathEscapes)
+
+	if runtime.GOOS == "windows" {
+		// A backslash is a path separator on Windows, so an encoding
+		// without BackSlash can hide ".." behind one.
+		w := newLinksFs("win", "Slash,Dot")
+		_, err = w.newObject(`sub\..` + fs.LinkSuffix)
+		assert.ErrorIs(t, err, errLinkNameInvalid)
+	}
+}
+
+// TestLinkSuffixListSkipsInvalidName checks that with --links a regular
+// file whose name is just the suffix is reported as an error and skipped
+// rather than failing the listing of its whole directory.
+func TestLinkSuffixListSkipsInvalidName(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+	when := fstest.Time("2001-02-03T04:05:06Z")
+	r.WriteFile("sub/"+fs.LinkSuffix, "not a link", when)
+	r.WriteFile("sub/file.txt", "hello", when)
+
+	accounting.Stats(ctx).ResetErrors()
+	entries, err := f.List(ctx, "sub")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	assert.Equal(t, "sub/file.txt", entries[0].Remote())
+	assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors())
+	accounting.Stats(ctx).ResetErrors()
+}
+
+// TestLinkSuffixDirectory checks that in --links mode a directory whose
+// name ends in ".rclonelink" is an ordinary directory. Only files are
+// translated into links, so the suffix must stay in its path rather
+// than making it address a sibling.
+func TestLinkSuffixDirectory(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	dir := "sub" + fs.LinkSuffix
+	want := filepath.Join(f.root, dir)
+	mtime := fstest.Time("2001-02-03T04:05:06Z")
+
+	d, err := f.MkdirMetadata(ctx, dir, fs.Metadata{"mtime": mtime.Format(time.RFC3339Nano)})
+	require.NoError(t, err)
+	assert.Equal(t, want, d.(*Directory).path)
+	fi, err := os.Lstat(want)
+	require.NoError(t, err)
+	assert.True(t, fi.IsDir())
+	assert.True(t, fi.ModTime().Equal(mtime), "metadata was not written to the directory")
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	got, ok := entries[0].(*Directory)
+	require.True(t, ok)
+	assert.Equal(t, dir, got.Remote())
+	assert.Equal(t, want, got.path)
+	assert.False(t, got.translatedLink)
+}
+
 func TestHashWithTypeNone(t *testing.T) {
 	ctx := context.Background()
 	r := fstest.NewRun(t)
@@ -1176,3 +1483,121 @@ func TestCopySymlink(t *testing.T) {
 	want = fstest.NewItem("dst2/file.txt", "hello world", when)
 	fstest.CompareItems(t, []fs.DirEntry{dst}, []fstest.Item{want}, nil, f.precision, "")
 }
+
+// TestCopyLinksLoop checks that -L/--copy-links follows a symlink to a
+// sibling directory but skips symlinks which point to the directory
+// being listed or one of its parents, as these would loop forever.
+func TestCopyLinksLoop(t *testing.T) {
+	skipIfNoSymlinks(t)
+	dir := t.TempDir()
+
+	// folder-A/link-to-B and folder-B/link-to-A point at each other
+	// and folder-A/link-to-parent points at dir
+	require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-A"), 0700))
+	require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-B"), 0700))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-A", "file-A"), []byte("A"), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-B", "file-B"), []byte("B"), 0600))
+	require.NoError(t, os.Symlink(filepath.Join("..", "folder-B"), filepath.Join(dir, "folder-A", "link-to-B")))
+	require.NoError(t, os.Symlink(filepath.Join("..", "folder-A"), filepath.Join(dir, "folder-B", "link-to-A")))
+	require.NoError(t, os.Symlink("..", filepath.Join(dir, "folder-A", "link-to-parent")))
+
+	wantRoot := []string{
+		"folder-A",
+		"folder-A/file-A",
+		"folder-A/link-to-B",
+		"folder-A/link-to-B/file-B",
+		"folder-B",
+		"folder-B/file-B",
+		"folder-B/link-to-A",
+		"folder-B/link-to-A/file-A",
+	}
+	for _, test := range []struct {
+		name       string
+		root       string
+		exclude    string
+		want       []string
+		wantErrors int64
+	}{{
+		name: "Root",
+		root: dir,
+		want: wantRoot,
+		// folder-A/link-to-parent, folder-A/link-to-B/link-to-A,
+		// folder-B/link-to-A/link-to-B, folder-B/link-to-A/link-to-parent
+		wantErrors: 4,
+	}, {
+		// link-to-parent points above the root of the Fs
+		name: "SubDir",
+		root: filepath.Join(dir, "folder-A"),
+		want: []string{
+			"file-A",
+			"link-to-B",
+			"link-to-B/file-B",
+		},
+		// link-to-parent, link-to-B/link-to-A
+		wantErrors: 2,
+	}, {
+		// Excluded loops aren't reported
+		name:    "Excluded",
+		root:    dir,
+		exclude: "link-to-parent/**",
+		want:    wantRoot,
+		// folder-A/link-to-B/link-to-A, folder-B/link-to-A/link-to-B
+		wantErrors: 2,
+	}} {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := context.Background()
+			if test.exclude != "" {
+				var fi *filter.Filter
+				ctx, fi = filter.AddConfig(ctx)
+				require.NoError(t, fi.AddRule("- "+test.exclude))
+			}
+			f, err := NewFs(ctx, "local", test.root, configmap.Simple{
+				"copy_links": "true",
+			})
+			require.NoError(t, err)
+
+			accounting.Stats(ctx).ResetErrors()
+			defer accounting.Stats(ctx).ResetErrors()
+
+			var got []string
+			err = walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {
+				for _, entry := range entries {
+					got = append(got, entry.Remote())
+				}
+				// Stop a runaway listing rather than waiting for ELOOP
+				if len(got) > 100 {
+					return errors.New("too many entries - symlink loop not detected")
+				}
+				return nil
+			})
+			require.NoError(t, err)
+
+			sort.Strings(got)
+			assert.Equal(t, test.want, got)
+			assert.Equal(t, test.wantErrors, accounting.Stats(ctx).GetErrors(), "global errors found")
+		})
+	}
+
+	// A single List reports and skips the link to a parent, but still
+	// returns the link to a sibling
+	t.Run("List", func(t *testing.T) {
+		ctx := context.Background()
+		f, err := NewFs(ctx, "local", dir, configmap.Simple{
+			"copy_links": "true",
+		})
+		require.NoError(t, err)
+
+		accounting.Stats(ctx).ResetErrors()
+		defer accounting.Stats(ctx).ResetErrors()
+
+		entries, err := f.List(ctx, "folder-A")
+		require.NoError(t, err)
+		var got []string
+		for _, entry := range entries {
+			got = append(got, entry.Remote())
+		}
+		sort.Strings(got)
+		assert.Equal(t, []string{"folder-A/file-A", "folder-A/link-to-B"}, got)
+		assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
+	})
+}

backend/local/metadata.go

@@ -5,7 +5,9 @@ import (
 	"math"
 	"os"
 	"runtime"
+	"slices"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -121,13 +123,15 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	uid, hasUID := o.parseMetadataInt(m, "uid", 10)
 	gid, hasGID := o.parseMetadataInt(m, "gid", 10)
 	if hasUID {
-		// FIXME should read UID and GID of current user and only attempt to set it if different
 		if !hasGID {
 			gid = uid
 		}
 		if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
 			fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
+		} else if setUID, setGID := ownerToSet(uid, gid); setUID < 0 && setGID < 0 {
+			fs.Debugf(o, "Not setting ownership %d.%d: only root can", uid, gid)
 		} else {
+			uid, gid = setUID, setGID
 			if o.translatedLink {
 				err = os.Lchown(o.path, uid, gid)
 			} else {
@@ -169,3 +173,27 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	// FIXME not parsing rdev yet
 	return outErr
 }
+
+// processGroups are the supplementary groups of this process
+var processGroups = sync.OnceValue(func() []int {
+	groups, _ := os.Getgroups()
+	return groups
+})
+
+// ownerToSet returns the uid and gid to pass to chown for the wanted
+// ownership: unchanged for root, else -1 (leave it as it is) for a uid
+// other than our own and for a group we are not a member of, which
+// only root may set.
+func ownerToSet(uid, gid int) (setUID, setGID int) {
+	euid := os.Geteuid()
+	if euid <= 0 {
+		return uid, gid
+	}
+	if uid != euid {
+		uid = -1
+	}
+	if gid != os.Getegid() && !slices.Contains(processGroups(), gid) {
+		gid = -1
+	}
+	return uid, gid
+}

backend/local/metadata_owner_unix_test.go

@@ -0,0 +1,50 @@
+//go:build !windows && !plan9 && !js
+
+package local
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetadataOwnershipAsNonRoot(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("needs a user other than root")
+	}
+	uid, gid := os.Geteuid(), os.Getegid()
+	other := uid + 1
+	notMember := gid + 1
+	for slices.Contains(processGroups(), notMember) {
+		notMember++
+	}
+
+	// only what a non-root user may set: its own uid, its groups
+	setUID, setGID := ownerToSet(other, notMember)
+	assert.Equal(t, [2]int{-1, -1}, [2]int{setUID, setGID})
+	setUID, setGID = ownerToSet(other, gid)
+	assert.Equal(t, [2]int{-1, gid}, [2]int{setUID, setGID})
+	setUID, setGID = ownerToSet(uid, notMember)
+	assert.Equal(t, [2]int{uid, -1}, [2]int{setUID, setGID})
+
+	// a file of another user (e.g. root:root) is copied with our ownership
+	// instead of failing with "operation not permitted"
+	r := fstest.NewRun(t)
+	r.WriteFile("owned.txt", "payload", time.Now())
+	f := r.Flocal.(*Fs)
+	obj, err := f.NewObject(context.Background(), "owned.txt")
+	require.NoError(t, err)
+	require.NoError(t, obj.(*Object).writeMetadataToFile(fs.Metadata{"uid": "0", "gid": "0"}))
+	fi, err := os.Stat(filepath.Join(f.root, "owned.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, uint32(uid), fi.Sys().(*syscall.Stat_t).Uid)
+}

backend/local/remove_other.go

@@ -2,9 +2,7 @@
 
 package local
 
-import "os"
-
-// Removes name, retrying on a sharing violation
-func remove(name string) error {
-	return os.Remove(name)
+// remove removes name with removeFn, retrying on a sharing violation
+func remove(name string, removeFn func(string) error) error {
+	return removeFn(name)
 }

backend/local/remove_test.go

@@ -38,7 +38,7 @@ func TestRemove(t *testing.T) {
 		require.NoError(t, fd.Close())
 	})
 	// delete the open file
-	err = remove(name)
+	err = remove(name, os.Remove)
 	require.NoError(t, err)
 	// check it no longer exists
 	assert.False(t, exists())

backend/local/remove_windows.go

@@ -10,12 +10,12 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-// Removes name, retrying on a sharing violation
-func remove(name string) (err error) {
+// remove removes name with removeFn, retrying on a sharing violation
+func remove(name string, removeFn func(string) error) (err error) {
 	const maxTries = 10
 	var sleepTime = 1 * time.Millisecond
 	for i := range maxTries {
-		err = os.Remove(name)
+		err = removeFn(name)
 		if err == nil {
 			break
 		}

backend/local/xattr.go

@@ -4,6 +4,7 @@ package local
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"syscall"
 
@@ -95,6 +96,11 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 	if !xattrSupported || o.fs.xattrSupported.Load() == 0 {
 		return nil
 	}
+	// When translating symlinks (-l/--links) write the xattrs through a
+	// descriptor opened under os.Root so a symlink planted in the destination
+	// can't redirect the write outside it (CWE-59). A translated link is
+	// itself the leaf, so it keeps the non-following path based write.
+	var fd *os.File
 	for k, value := range metadata {
 		k = strings.ToLower(k)
 		if _, found := systemMetadataInfo[k]; found {
@@ -102,9 +108,27 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 		}
 		k = xattrPrefix + k
 		v := []byte(value)
-		if o.fs.opt.FollowSymlinks {
+		switch {
+		case o.fs.opt.TranslateSymlinks && !o.translatedLink:
+			if fd == nil {
+				fd, err = o.fs.openFile(o.path, os.O_RDONLY, 0)
+				if os.IsPermission(err) {
+					// A write only file can't be opened for reading but its
+					// owner may still set xattrs on it, so try opening it for
+					// writing instead (which can't succeed for a directory).
+					if wfd, werr := o.fs.openFile(o.path, os.O_WRONLY, 0); werr == nil {
+						fd, err = wfd, nil
+					}
+				}
+				if err != nil {
+					return fmt.Errorf("failed to open for xattr: %w", err)
+				}
+				defer fs.CheckClose(fd, &err)
+			}
+			err = xattr.FSet(fd, k, v)
+		case o.fs.opt.FollowSymlinks:
 			err = xattr.Set(o.path, k, v)
-		} else {
+		default:
 			err = xattr.LSet(o.path, k, v)
 		}
 		if err != nil {

backend/local/xattr_internal_test.go

@@ -0,0 +1,96 @@
+//go:build !openbsd && !plan9
+
+package local
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/xattr"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestXattrThroughPlantedSymlinkBlocked checks user metadata (xattrs)
+// can't be written outside the root through a planted symlink with
+// --links, while in-tree writes still work.
+func TestXattrThroughPlantedSymlinkBlocked(t *testing.T) {
+	if !xattrSupported {
+		t.Skip("xattrs not supported on this OS")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	metadata := fs.Metadata{
+		"pwned": "attacker-controlled-value",
+	}
+
+	// Control: an in-tree directory gets the xattr as normal.
+	_, err := f.MkdirMetadata(ctx, "sub", metadata)
+	require.NoError(t, err)
+	if f.xattrSupported.Load() == 0 {
+		t.Skip("xattrs not supported on this filesystem")
+	}
+	v, err := xattr.LGet(filepath.Join(f.root, "sub"), "user.pwned")
+	require.NoError(t, err)
+	assert.Equal(t, "attacker-controlled-value", string(v))
+
+	// A directory outside the destination, with a pre-existing child,
+	// whose xattrs the attacker targets.
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	evilSub := filepath.Join(evilDir, "sub")
+	require.NoError(t, os.Mkdir(evilSub, 0700))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// The source now presents "pwn/sub" as a directory with attacker-chosen
+	// user metadata. Applying it must not reach through the planted symlink.
+	_, err = f.MkdirMetadata(ctx, "pwn/sub", metadata)
+	require.Error(t, err, "applying metadata through a planted symlink should be refused")
+
+	_, err = xattr.LGet(evilSub, "user.pwned")
+	require.Error(t, err, "xattr escaped through planted symlink to %q", evilSub)
+}
+
+// TestXattrWriteOnlyFile checks user metadata can still be set on a
+// write-only file with --links, where the confined open can't read it.
+func TestXattrWriteOnlyFile(t *testing.T) {
+	if !xattrSupported {
+		t.Skip("xattrs not supported on this OS")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	require.NoError(t, putFile(ctx, f, "wo.txt", "write only"))
+	path := filepath.Join(f.root, "wo.txt")
+	require.NoError(t, os.Chmod(path, 0200))
+
+	obj, err := f.NewObject(ctx, "wo.txt")
+	require.NoError(t, err)
+	o := obj.(*Object)
+	err = o.setXattr(fs.Metadata{"potato": "chips"})
+	require.NoError(t, err)
+	if f.xattrSupported.Load() == 0 {
+		t.Skip("xattrs not supported on this filesystem")
+	}
+	// Reading xattrs needs read permission
+	require.NoError(t, os.Chmod(path, 0600))
+	v, err := xattr.LGet(path, "user.potato")
+	require.NoError(t, err)
+	assert.Equal(t, "chips", string(v))
+}

backend/netstorage/netstorage.go

@@ -208,6 +208,24 @@ func (d Du) GetName() xml.Name {
 
 //------------------------------------------------------------------------------
 
+// apiHeaders are the NetStorage API request headers which carry the
+// account name, the request signature and the signed action. They must
+// not be forwarded when a redirect leaves the configured host.
+//
+// The action header is sent uncanonicalised, hence the "*" prefix.
+var apiHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "*X-Akamai-ACS-Action"}
+
+// newClient makes the http client used for all NetStorage requests.
+//
+// The signer adds the API headers to every request, so the client
+// strips them on cross host redirects and refuses an HTTPS to HTTP
+// downgrade.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(apiHeaders...)
+	return client
+}
+
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 //
@@ -236,11 +254,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("couldn't parse URL %q: %w", opt.Endpoint, err)
 	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(root))
+	u, err := rest.URLJoinRoot(base, root)
 	if err != nil {
-		return nil, fmt.Errorf("couldn't join URL %q and %q: %w", base.String(), root, err)
+		return nil, err
 	}
-	client := fshttp.NewClient(ctx)
+	client := newClient(ctx)
 
 	f := &Fs{
 		name:        name,
@@ -573,10 +591,12 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 				fs.Errorf(nil, "Unable to parse URL %q: %v", f.endpointURL, err)
 				return fs.ErrorDirNotFound
 			}
-			resumeURL, err := rest.URLJoin(u, rest.URLPathEscape(resumeStart))
+			// A bad resume start must fail the listing rather than
+			// return fs.ErrorDirNotFound which sync treats as an empty
+			// directory
+			resumeURL, err := rest.URLJoinRoot(u, resumeStart)
 			if err != nil {
-				fs.Errorf(nil, "Unable to join URL %q for resumeStart %s: %v", f.endpointURL, resumeStart, err)
-				return fs.ErrorDirNotFound
+				return fmt.Errorf("unable to join URL %q for resumeStart %q: %w", f.endpointURL, resumeStart, err)
 			}
 			URL = resumeURL.String()
 		}

backend/netstorage/netstorage_internal_test.go

@@ -0,0 +1,205 @@
+package netstorage
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestListResumeCannotChangeHost(t *testing.T) {
+	// The resume start returned by a list request comes from the
+	// server. It must not be able to move the signed continuation
+	// request to another host.
+	var requests int
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests++
+		http.Error(w, "not found", http.StatusNotFound)
+	}))
+	defer other.Close()
+	otherHost := strings.TrimPrefix(other.URL, "http://")
+
+	var listRequests int
+	configured := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		action := r.Header.Get("X-Akamai-ACS-Action")
+		switch {
+		case strings.Contains(action, "action=stat"):
+			_, _ = w.Write([]byte(`<stat directory="/123456/dir"><file type="dir" name="dir" mtime="0"/></stat>`))
+		case strings.Contains(action, "action=list"):
+			listRequests++
+			_, _ = w.Write([]byte(`<list><file type="file" name="123456/dir/a.txt" size="1" mtime="0"/><resume start="//` + otherHost + `/123456/dir/b.txt"/></list>`))
+		default:
+			http.Error(w, "unexpected action "+action, http.StatusBadRequest)
+		}
+	}))
+	defer configured.Close()
+
+	m := configmap.Simple{
+		"type":     "netstorage",
+		"protocol": "http",
+		"host":     strings.TrimPrefix(configured.URL, "http://") + "/123456/",
+		"account":  "account",
+		"secret":   obscure.MustObscure("secret"),
+	}
+	f, err := NewFs(context.Background(), "TestNetStorage", "dir", m)
+	require.NoError(t, err)
+
+	// The listing must fail loudly rather than with
+	// fs.ErrorDirNotFound which sync treats as an empty directory
+	err = f.(*Fs).ListR(context.Background(), "", func(entries fs.DirEntries) error { return nil })
+	assert.ErrorContains(t, err, "must not change the scheme, host or user")
+	assert.NotErrorIs(t, err, fs.ErrorDirNotFound)
+	assert.Equal(t, 1, listRequests, "list request sent to the configured host")
+	assert.Equal(t, 0, requests, "request sent to the other host")
+}
+
+// apiTestHeaders is a deliberately literal copy of apiHeaders, without
+// the "*" prefix, so that a header dropped from the production list
+// shows up as a leak in the cross host tests rather than silently
+// losing coverage.
+var apiTestHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "X-Akamai-ACS-Action"}
+
+// statXML is a minimal stat response saying the root is a directory
+const statXML = `<?xml version="1.0" encoding="UTF-8"?>
+<stat directory="/dir"><file type="dir" name="dir" mtime="1"/></stat>`
+
+// newTestFs makes an Fs via NewFs pointing at serverURL.
+//
+// NewFs makes a stat request for the root and treats any failure as
+// the root not existing, so callers check what the servers received
+// or make a further request rather than relying on its error.
+func newTestFs(ctx context.Context, t *testing.T, serverURL string) *Fs {
+	u, err := url.Parse(serverURL)
+	require.NoError(t, err)
+	m := configmap.Simple{
+		"type":     "netstorage",
+		"protocol": u.Scheme,
+		"host":     u.Host + "/dir",
+		"account":  "test-account",
+		"secret":   obscure.MustObscure("test-secret"),
+	}
+	f, err := NewFs(ctx, "netstorage-test", "", m)
+	require.NoError(t, err)
+	return f.(*Fs)
+}
+
+// assertAPIHeaders checks all the API headers are present (want) or
+// absent (!want) in r.
+func assertAPIHeaders(t *testing.T, r *http.Request, want bool) {
+	for _, header := range apiTestHeaders {
+		if want {
+			assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
+		} else {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+	}
+}
+
+// TestRedirectStripsAPIHeadersOnCrossHost checks the API headers are
+// sent to the configured host but not to another host it redirects to.
+func TestRedirectStripsAPIHeadersOnCrossHost(t *testing.T) {
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAPIHeaders(t, r, false)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer other.Close()
+
+	var originHits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAPIHeaders(t, r, true)
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	newTestFs(context.Background(), t, origin.URL)
+	assert.Equal(t, int32(1), originHits.Load())
+	assert.Equal(t, int32(1), otherHits.Load())
+}
+
+// TestUploadStripsAPIHeadersOnCrossHost checks an upload, which
+// carries the API headers on the PUT as well as in the trailers, does
+// not send them to another host the configured host redirects to.
+func TestUploadStripsAPIHeadersOnCrossHost(t *testing.T) {
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAPIHeaders(t, r, false)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer other.Close()
+
+	var originHits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAPIHeaders(t, r, true)
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusMovedPermanently)
+	}))
+	defer origin.Close()
+
+	ctx := context.Background()
+	f := newTestFs(ctx, t, origin.URL)
+	o := &Object{fs: f, remote: "file"}
+	src := object.NewStaticObjectInfo("file", time.Now(), 4, true, nil, f)
+	err := o.netStorageUploadRequest(ctx, strings.NewReader("data"), src)
+	require.NoError(t, err)
+	// NewFs stat, PUT and the stat after the upload
+	assert.Equal(t, int32(3), originHits.Load())
+	assert.Equal(t, int32(3), otherHits.Load())
+}
+
+// TestRedirectKeepsAPIHeadersOnSameHost checks the API headers are
+// still sent after a redirect which stays on the configured host.
+func TestRedirectKeepsAPIHeadersOnSameHost(t *testing.T) {
+	var hits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		assertAPIHeaders(t, r, true)
+		if !strings.HasSuffix(r.URL.Path, "/moved") {
+			http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
+			return
+		}
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer origin.Close()
+
+	newTestFs(context.Background(), t, origin.URL)
+	assert.Equal(t, int32(2), hits.Load())
+}
+
+// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
+// plaintext HTTP is not followed.
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+	var targetHits atomic.Int32
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		targetHits.Add(1)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer target.Close()
+
+	origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.InsecureSkipVerify = true
+	f := newTestFs(ctx, t, origin.URL)
+	_, err := f.netStorageStatRequest(ctx, f.url(""), true)
+	require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	assert.Equal(t, int32(0), targetHits.Load())
+}

backend/onedrive/metadata.go

@@ -20,7 +20,7 @@ import (
 const (
 	dirMimeType   = "inode/directory"
 	timeFormatIn  = time.RFC3339
-	timeFormatOut = "2006-01-02T15:04:05.999Z" // mS for OneDrive Personal, otherwise only S
+	timeFormatOut = "2006-01-02T15:04:05.999Z" // keeps mS if present, though OneDrive only stores S
 )
 
 // system metadata keys which this backend owns
@@ -32,17 +32,17 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 		ReadOnly: true,
 	},
 	"mtime": {
-		Help:    "Time of last modification with S accuracy (mS for OneDrive Personal).",
+		Help:    "Time of last modification with S accuracy.",
 		Type:    "RFC 3339",
 		Example: "2006-01-02T15:04:05Z",
 	},
 	"btime": {
-		Help:    "Time of file birth (creation) with S accuracy (mS for OneDrive Personal).",
+		Help:    "Time of file birth (creation) with S accuracy.",
 		Type:    "RFC 3339",
 		Example: "2006-01-02T15:04:05Z",
 	},
 	"utime": {
-		Help:     "Time of upload with S accuracy (mS for OneDrive Personal).",
+		Help:     "Time of upload with S accuracy.",
 		Type:     "RFC 3339",
 		Example:  "2006-01-02T15:04:05Z",
 		ReadOnly: true,
@@ -113,7 +113,7 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 		ReadOnly: true,
 	},
 	"shared-time": {
-		Help:     "Time when the item was shared, with S accuracy (mS for OneDrive Personal).",
+		Help:     "Time when the item was shared, with S accuracy.",
 		Type:     "RFC 3339",
 		Example:  "2006-01-02T15:04:05Z",
 		ReadOnly: true,

backend/onedrive/onedrive.go

@@ -41,6 +41,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -56,6 +57,7 @@ const (
 	driveTypeSharepoint         = "documentLibrary"
 	defaultChunkSize            = 10 * fs.Mebi
 	chunkSizeMultiple           = 320 * fs.Kibi
+	defaultTenantAPIVersion     = "v2.0"
 	// maxSinglePartSize is the size at which Graph stops accepting an upload in a
 	// single request (PUT /items/{id}/content). Microsoft documents this as
 	// "250 MB", see
@@ -166,7 +168,7 @@ See: https://github.com/rclone/rclone/issues/1716
 			Name: "tenant_url",
 			Help: `The tenant URL for non-admin OneDrive access.
 
-Set this to your SharePoint tenant URL to use the SharePoint v2.0 API
+Set this to your SharePoint tenant URL to use the SharePoint API
 endpoint instead of the standard Microsoft Graph API. This allows
 accessing business OneDrive without admin consent.
 
@@ -177,6 +179,16 @@ for "driveAccessToken" in the network requests. Look for the
 Example: https://your-tenant.sharepoint.com/_api`,
 			Default:  "",
 			Advanced: true,
+		}, {
+			Name: "tenant_api_version",
+			Help: `The SharePoint API version to use with tenant_url.
+
+Set this to the SharePoint API version matching the browser-extracted
+access token. For example, use v2.1 with a driveAccessTokenV21 token.
+
+This only applies when tenant_url is set.`,
+			Default:  defaultTenantAPIVersion,
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
@@ -283,16 +295,13 @@ cases, rclone will fall back to normal copy (which will be slightly slower).`,
 			Default: false,
 			Help: `Remove all versions on modifying operations.
 
-Onedrive for business creates versions when rclone uploads new files
+Onedrive creates versions when rclone uploads new files
 overwriting an existing one and when it sets the modification time.
 
 These versions take up space out of the quota.
 
 This flag checks for versions after file upload and setting
 modification time and removes all but the last version.
-
-**NB** Onedrive personal can't currently delete versions so don't use
-this flag there.
 `,
 			Advanced: true,
 		}, {
@@ -331,14 +340,18 @@ This works with OneDrive for Business, SharePoint document libraries, and OneDri
 				Help:  "Creates a read-write link to the item.",
 			}, {
 				Value: "embed",
-				Help:  "Creates an embeddable link to the item.",
+				Help:  "Creates an embeddable link to the item.\nOnly available in OneDrive personal.",
 			}},
 		}, {
 			Name:    "link_password",
 			Default: "",
 			Help: `Set the password for links created by the link command.
 
-At the time of writing this only works with OneDrive personal paid accounts.
+At the time of writing this works with OneDrive for Business and
+OneDrive personal paid accounts.
+
+OneDrive personal free accounts can't set a password or an expiry time
+(with --expire) on links.
 `,
 			Advanced:  true,
 			Sensitive: true,
@@ -426,21 +439,12 @@ Setting this flag speeds up these things greatly:
     rclone size onedrive:
     rclone rc vfs/refresh recursive=true
 
-**However** the delta listing API **only** works at the root of the
-drive. If you use it not at the root then it recurses from the root
-and discards all the data that is not under the directory you asked
-for. So it will be correct but may not be very efficient.
-
-This is why this flag is not set as the default.
-
-As a rule of thumb if nearly all of your data is under rclone's root
-directory (the |root/directory| in |onedrive:root/directory|) then
-using this flag will be a big performance win. If your data is
-mostly not under the root then using this flag will be a big
-performance loss.
-
-It is recommended if you are mounting your onedrive at the root
-(or near the root when using crypt) and using rclone |rc vfs/refresh|.
+Rclone asks for the delta listing of the directory being listed. If
+the drive only supports delta listings at the root of the drive (as
+some older OneDrive for Business and SharePoint drives do) then
+rclone lists from the root and discards all the data that is not
+under the directory you asked for. So it will be correct but may not
+be very efficient.
 `, "|", "`"),
 			Advanced: true,
 		}, {
@@ -509,12 +513,20 @@ func getRegionURL(m configmap.Mapper) (region, graphURL string) {
 	// Check if tenant_url is provided for non-admin mode
 	tenantURL, _ := m.Get("tenant_url")
 	if tenantURL != "" {
-		graphURL = tenantURL + "/v2.0"
+		tenantAPIVersion, _ := m.Get("tenant_api_version")
+		graphURL = tenantAPIEndpoint(tenantURL, tenantAPIVersion)
 	}
 
 	return region, graphURL
 }
 
+func tenantAPIEndpoint(tenantURL, tenantAPIVersion string) string {
+	if tenantAPIVersion == "" {
+		tenantAPIVersion = defaultTenantAPIVersion
+	}
+	return strings.TrimRight(tenantURL, "/") + "/" + strings.TrimLeft(tenantAPIVersion, "/")
+}
+
 // Config for chooseDrive
 type chooseDriveOpt struct {
 	opts         rest.Opts
@@ -808,6 +820,7 @@ type Options struct {
 	UploadCutoff            fs.SizeSuffix        `config:"upload_cutoff"`
 	ChunkSize               fs.SizeSuffix        `config:"chunk_size"`
 	TenantURL               string               `config:"tenant_url"`
+	TenantAPIVersion        string               `config:"tenant_api_version"`
 	DriveID                 string               `config:"drive_id"`
 	DriveType               string               `config:"drive_type"`
 	RootFolderID            string               `config:"root_folder_id"`
@@ -832,19 +845,33 @@ type Options struct {
 
 // Fs represents a remote OneDrive
 type Fs struct {
-	name         string             // name of this remote
-	root         string             // the path we are working on
-	opt          Options            // parsed options
-	ci           *fs.ConfigInfo     // global config
-	features     *fs.Features       // optional features
-	srv          *rest.Client       // the connection to the OneDrive server
-	unAuth       *rest.Client       // no authentication connection to the OneDrive server
-	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
-	tokenRenewer *oauthutil.Renew   // renew the token on expiry
-	driveID      string             // ID to use for querying Microsoft Graph
-	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
-	hashType     hash.Type          // type of the hash we are using
+	name         string                 // name of this remote
+	root         string                 // the path we are working on
+	opt          Options                // parsed options
+	ci           *fs.ConfigInfo         // global config
+	features     *fs.Features           // optional features
+	srv          *rest.Client           // the connection to the OneDrive server
+	unAuth       *rest.Client           // no authentication connection to the OneDrive server
+	dirCache     *dircache.DirCache     // Map of directory path to directory id
+	pacer        *fs.Pacer              // pacer for API calls
+	tokenRenewer *oauthutil.Renew       // renew the token on expiry
+	tokenSource  *oauthutil.TokenSource // source of the API token
+	sharePoint   *sharePointClients     // authenticated clients for SharePoint hosts
+	driveID      string                 // ID to use for querying Microsoft Graph
+	driveType    string                 // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
+	hashType     hash.Type              // type of the hash we are using
+}
+
+// sharePointClients holds the clients authenticated for SharePoint hosts
+type sharePointClients struct {
+	mu    sync.Mutex
+	hosts map[string]sharePointAuth
+}
+
+// sharePointAuth is a client authenticated for a SharePoint host
+type sharePointAuth struct {
+	client      *rest.Client
+	tokenSource oauth2.TokenSource
 }
 
 // Object describes a OneDrive object
@@ -1064,6 +1091,154 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
+// errPreauthRetired explains an authentication failure on a URL which
+// SharePoint used to return pre-authenticated.
+var errPreauthRetired = errors.New("SharePoint refused to authenticate a request - Microsoft is retiring pre-authenticated (tempauth) URLs, see https://rclone.org/onedrive/#pre-authenticated-urls")
+
+// sharePointDomains are the domains of SharePoint Online in the
+// different Microsoft clouds
+var sharePointDomains = []string{".sharepoint.com", ".sharepoint.us", ".sharepoint-mil.us", ".sharepoint.cn", ".sharepoint.de"}
+
+// stripTempauth removes the tempauth parameter from the URLs the API
+// returns, which simulates the retirement of pre-authenticated URLs
+// in tests.
+var stripTempauth = false
+
+// returnedURL processes a URL returned by the API before use
+func returnedURL(rawURL string) string {
+	if !stripTempauth {
+		return rawURL
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	q := u.Query()
+	q.Del("tempauth")
+	u.RawQuery = q.Encode()
+	return u.String()
+}
+
+// urlAuth says how a URL returned by the API is authenticated
+type urlAuth int
+
+const (
+	urlAuthNone       urlAuth = iota // call without authentication
+	urlAuthAPI                       // call with the API token
+	urlAuthSharePoint                // call with a token for the SharePoint host
+)
+
+// apiHost returns the host the API calls made with f.srv go to.
+func (f *Fs) apiHost() string {
+	endpoint := graphAPIEndpoint[f.opt.Region]
+	if f.opt.TenantURL != "" {
+		endpoint = f.opt.TenantURL
+	}
+	u, err := url.Parse(endpoint)
+	if err != nil || u.Scheme != "https" {
+		return ""
+	}
+	return u.Host
+}
+
+// urlAuthFor returns how to authenticate a URL returned by the API,
+// such as an upload session or an async job monitor.
+//
+// Pre-authenticated URLs fail if they are sent an Authorization
+// header and tokens must not be sent to other hosts, so only HTTPS
+// URLs without tempauth on the API host or a SharePoint host are
+// authenticated.
+func (f *Fs) urlAuthFor(rawURL string) urlAuth {
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Scheme != "https" || u.Query().Has("tempauth") {
+		return urlAuthNone
+	}
+	if apiHost := f.apiHost(); apiHost != "" && strings.EqualFold(u.Host, apiHost) {
+		return urlAuthAPI
+	}
+	if isSharePointHost(u.Hostname()) {
+		return urlAuthSharePoint
+	}
+	return urlAuthNone
+}
+
+// isSharePointHost returns true if host is in a SharePoint Online domain.
+func isSharePointHost(host string) bool {
+	host = strings.ToLower(host)
+	for _, domain := range sharePointDomains {
+		if strings.HasSuffix(host, domain) {
+			return true
+		}
+	}
+	return false
+}
+
+// checkSharePointRedirect stops the SharePoint token being sent to a
+// host which isn't SharePoint, as the token is added to every request
+// the client makes.
+func checkSharePointRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if req.URL.Scheme != "https" || !isSharePointHost(req.URL.Hostname()) {
+		return fmt.Errorf("refusing to send a SharePoint token on a redirect to %s://%s", req.URL.Scheme, req.URL.Host)
+	}
+	return nil
+}
+
+// clientForURL returns the client to call a URL returned by the API with.
+func (f *Fs) clientForURL(ctx context.Context, rawURL string) (*rest.Client, error) {
+	switch f.urlAuthFor(rawURL) {
+	case urlAuthAPI:
+		return f.srv, nil
+	case urlAuthSharePoint:
+		u, _ := url.Parse(rawURL)
+		return f.sharePointClient(ctx, u.Host)
+	}
+	return f.unAuth, nil
+}
+
+// sharePointClient returns a client authenticated for the SharePoint
+// host, which upload session and async job monitor URLs need once
+// they are no longer pre-authenticated.
+//
+// The token is got from the existing credentials, which Entra ID
+// allows for SharePoint as the user has consented to the Graph file
+// scopes.
+func (f *Fs) sharePointClient(ctx context.Context, host string) (*rest.Client, error) {
+	if f.tokenSource == nil || f.sharePoint == nil {
+		return nil, fmt.Errorf("%w: no credentials to get a SharePoint token with", errPreauthRetired)
+	}
+	host = strings.ToLower(host)
+	f.sharePoint.mu.Lock()
+	defer f.sharePoint.mu.Unlock()
+	sp, ok := f.sharePoint.hosts[host]
+	if !ok {
+		tokenSource := f.tokenSource.ScopedTokenSource([]string{"https://" + host + "/.default"})
+		client := oauth2.NewClient(oauthutil.Context(ctx, fshttp.NewClient(ctx)), tokenSource)
+		client.CheckRedirect = checkSharePointRedirect
+		sp = sharePointAuth{client: rest.NewClient(client), tokenSource: tokenSource}
+		f.sharePoint.hosts[host] = sp
+		fs.Debugf(f, "Using a SharePoint token for %s", host)
+	}
+	// Get the token now so failures are reported before the client is used
+	if _, err := sp.tokenSource.Token(); err != nil {
+		return nil, fmt.Errorf("%w: failed to get a SharePoint token: %w", errPreauthRetired, err)
+	}
+	return sp.client, nil
+}
+
+// checkPreauthRetired adds an explanation to a 401 error from a URL
+// returned by the API which needs a SharePoint token, as these were
+// pre-authenticated until Microsoft retired them.
+func (f *Fs) checkPreauthRetired(rawURL string, resp *http.Response, err error) error {
+	if err == nil || resp == nil || resp.StatusCode != http.StatusUnauthorized ||
+		f.driveType == driveTypePersonal || errors.Is(err, errPreauthRetired) || f.urlAuthFor(rawURL) != urlAuthSharePoint {
+		return err
+	}
+	return fmt.Errorf("%w: %w", errPreauthRetired, err)
+}
+
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
 	const minChunkSize = fs.SizeSuffixBase
 	if cs%chunkSizeMultiple != 0 {
@@ -1115,7 +1290,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	rootURL := graphAPIEndpoint[opt.Region] + "/v1.0" + "/drives/" + opt.DriveID
 
 	if opt.TenantURL != "" {
-		rootURL = opt.TenantURL + "/v2.0" + "/drives/" + opt.DriveID
+		rootURL = tenantAPIEndpoint(opt.TenantURL, opt.TenantAPIVersion) + "/drives/" + opt.DriveID
 	}
 
 	oauthConfig, err := makeOauthConfig(ctx, opt)
@@ -1132,16 +1307,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:      name,
-		root:      root,
-		opt:       *opt,
-		ci:        ci,
-		driveID:   opt.DriveID,
-		driveType: opt.DriveType,
-		srv:       rest.NewClient(oAuthClient).SetRoot(rootURL),
-		unAuth:    rest.NewClient(client).SetRoot(rootURL),
-		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		hashType:  QuickXorHashType,
+		name:        name,
+		root:        root,
+		opt:         *opt,
+		ci:          ci,
+		driveID:     opt.DriveID,
+		driveType:   opt.DriveType,
+		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
+		tokenSource: ts,
+		sharePoint:  &sharePointClients{hosts: make(map[string]sharePointAuth)},
+		unAuth:      rest.NewClient(client).SetRoot(rootURL),
+		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		hashType:    QuickXorHashType,
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:          true,
@@ -1402,6 +1579,8 @@ func (f *Fs) itemToDirEntry(ctx context.Context, dir string, info *api.Item) (en
 		id := info.GetID()
 		f.dirCache.Put(remote, id)
 		d := f.newDir(id, remote)
+		// OneDrive reports the total size of a folder's contents
+		d.size = info.GetSize()
 		d.items = folder.ChildCount
 		f.setSystemMetadata(info, d.meta, remote, dirMimeType)
 		entry = d
@@ -1490,10 +1669,6 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		return err
 	}
 
-	// ListR only works at the root of a onedrive, not on a folder
-	// So we have to filter things outside of the root which is
-	// inefficient.
-
 	list := list.NewHelper(callback)
 
 	// list a folder conventionally - used for shared folders
@@ -1569,17 +1744,26 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		return nil
 	}
 
-	opts := rest.Opts{
-		Method: "GET",
-		Path:   "/root/delta",
-		Parameters: map[string][]string{
-			// "token": {token},
+	listDelta := func(opts rest.Opts) error {
+		opts.Parameters = url.Values{
 			"$top": {fmt.Sprintf("%d", f.opt.ListChunk)},
-		},
+		}
+		var result api.DeltaResponse
+		return f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
 	}
 
-	var result api.DeltaResponse
-	err = f._listAll(ctx, "", false, false, fn, &opts, &result, &result.Value, &result.NextLink)
+	err = listDelta(f.newOptsCall(directoryID, "GET", "/delta"))
+	// Some drives only support delta listings at the root of the
+	// drive, in which case list the whole drive and filter out the
+	// items outside dir.
+	var apiErr *api.Error
+	if err != nil && len(seen) == 0 && errors.As(err, &apiErr) {
+		fs.Debugf(f, "Delta listing of directory failed, listing from the root of the drive instead: %v", err)
+		err = listDelta(rest.Opts{
+			Method: "GET",
+			Path:   "/root/delta",
+		})
+	}
 	if err != nil {
 		return err
 	}
@@ -1706,11 +1890,25 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 		var resp *http.Response
 		var err error
 		var body []byte
+		var redirected bool
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = http.Get(location)
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
+			if err != nil {
+				return false, err
+			}
+			client, err := f.clientForURL(ctx, location)
+			if err != nil {
+				return false, err
+			}
+			resp, err = client.Do(req)
 			if err != nil {
 				return fserrors.ShouldRetry(err), err
 			}
+			redirected = resp.Request.URL.String() != req.URL.String()
+			if resp.StatusCode == http.StatusUnauthorized {
+				_ = resp.Body.Close()
+				return false, f.checkPreauthRetired(location, resp, fmt.Errorf("async job monitor: HTTP error %s", resp.Status))
+			}
 			body, err = rest.ReadBody(resp)
 			return fserrors.ShouldRetry(err), err
 		})
@@ -1723,6 +1921,11 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 		if err != nil {
 			return fmt.Errorf("async status result not JSON: %q: %w", body, err)
 		}
+		// An authenticated monitor redirects to the new item when the
+		// job has finished instead of returning its status
+		if status.Status == "" && redirected {
+			status.Status = "completed"
+		}
 
 		switch status.Status {
 		case "failed":
@@ -1822,7 +2025,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	// read location header
-	location := resp.Header.Get("Location")
+	location := returnedURL(resp.Header.Get("Location"))
 	if location == "" {
 		return nil, errors.New("didn't receive location header in copy response")
 	}
@@ -1833,6 +2036,10 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		fs.Debugf(src, "Server-side copy failed - file not shared between drives")
 		return nil, fs.ErrorCantCopy
 	}
+	if errors.Is(err, errPreauthRetired) {
+		fs.Debugf(src, "Server-side copy can't be monitored - copying another way: %v", err)
+		return nil, fs.ErrorCantCopy
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -2507,16 +2714,23 @@ func (o *Object) openContentPrefer(ctx context.Context, options ...fs.OpenOption
 	return o.openWithRedirect(ctx, &opts)
 }
 
-// openWithRedirect downloads via /content style endpoints that 302 to a preauthenticated URL.
+// openWithRedirect downloads via /content style endpoints that 302 to
+// a download URL, which is pre-authenticated or needs a SharePoint
+// token.
 func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.ReadCloser, err error) {
 	var resp *http.Response
-	// Make a note of the redirect target as we need to call it without Auth
+	// Make a note of the redirect target as it needs different auth
 	var redirectReq *http.Request
 	opts.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 		if len(via) >= 10 {
 			return errors.New("stopped after 10 redirects")
 		}
-		// Preauthenticated download URLs must not carry the Graph Authorization header.
+		if stripTempauth {
+			if u, err := url.Parse(returnedURL(req.URL.String())); err == nil {
+				req.URL = u
+			}
+		}
+		// Download URLs must not carry the Graph Authorization header.
 		req.Header.Del("Authorization")
 		// Keep Prefer on the redirect when forcing infected download; some SharePoint
 		// endpoints honor it only on the final download request.
@@ -2555,17 +2769,24 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
 		return nil, err
 	}
 	if redirectReq != nil {
+		redirectURL := redirectReq.URL.String()
+		var client *rest.Client
+		client, err = o.fs.clientForURL(ctx, redirectURL)
+		if err != nil {
+			return nil, err
+		}
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.unAuth.Do(redirectReq)
+			resp, err = client.Do(redirectReq)
 			if err != nil {
 				return shouldRetry(ctx, resp, err)
 			}
-			// unAuth.Do does not check status; treat non-2xx as failure so a malware
+			// Do does not check status; treat non-2xx as failure so a malware
 			// JSON body is not written out as file content.
 			if resp.StatusCode < 200 || resp.StatusCode > 299 {
 				body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
 				_ = resp.Body.Close()
 				err = fmt.Errorf("HTTP error %d (%s) %s", resp.StatusCode, resp.Status, strings.TrimSpace(string(body)))
+				err = o.fs.checkPreauthRetired(redirectURL, resp, err)
 				if strings.Contains(string(body), "malwareDetected") || resp.Header.Get("X-Virus-Infected") != "" {
 					err = malwareDownloadError(o.fs.opt.AVOverride, err)
 				}
@@ -2587,7 +2808,19 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
 		// Overwrite size with actual size since size readings from Onedrive is unreliable.
 		o.size = resp.ContentLength
 	}
-	return resp.Body, err
+	return o.checkRange(resp, opts.Options)
+}
+
+// checkRange returns the body of a download, checking it is the range
+// requested in options.
+func (o *Object) checkRange(resp *http.Response, options []fs.OpenOption) (in io.ReadCloser, err error) {
+	// The size isn't passed as it is unreliable on OneDrive
+	err = rest.CheckContentRange(resp, options, -1)
+	if err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
+	return resp.Body, nil
 }
 
 // malwareDownloadError formats an error when the server blocks a malware-flagged file.
@@ -2627,12 +2860,16 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
 	}
 	var info api.UploadFragmentResponse
 	var resp *http.Response
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return 0, err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
+		resp, err = client.CallJSON(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return 0, err
+		return 0, o.fs.checkPreauthRetired(url, resp, err)
 	}
 	if len(info.NextExpectedRanges) != 1 {
 		return 0, fmt.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
@@ -2656,6 +2893,10 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 	var resp *http.Response
 	var body []byte
 	skip := int64(0)
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return nil, err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		toSend := chunkSize - skip
 		opts := rest.Opts{
@@ -2667,7 +2908,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			Options:       options,
 		}
 		_, _ = chunk.Seek(skip, io.SeekStart)
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = client.Call(ctx, &opts)
 		if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
 			fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
 			pos, posErr := o.getPosition(ctx, url)
@@ -2706,7 +2947,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 		}
 		return false, nil
 	})
-	return info, err
+	return info, o.fs.checkPreauthRetired(url, resp, err)
 }
 
 // cancelUploadSession cancels an upload session
@@ -2717,11 +2958,15 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
 		NoResponse: true,
 	}
 	var resp *http.Response
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = client.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
-	return
+	return o.fs.checkPreauthRetired(url, resp, err)
 }
 
 // uploadMultipart uploads a file using multipart upload
@@ -2739,7 +2984,20 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 	if err != nil {
 		return nil, err
 	}
-	uploadURL := session.UploadURL
+	uploadURL := returnedURL(session.UploadURL)
+
+	// Check a session which needs SharePoint authentication can get it
+	// before reading any data, so it can be uploaded in a single part
+	// instead if it can't.
+	if o.fs.urlAuthFor(uploadURL) == urlAuthSharePoint {
+		if _, err = o.getPosition(ctx, uploadURL); err != nil {
+			if !errors.Is(err, errPreauthRetired) || size >= int64(maxSinglePartSize) {
+				return nil, err
+			}
+			fs.Debugf(o, "Uploading in a single part as the upload session can't be used: %v", err)
+			return o.uploadSinglepart(ctx, in, src, options...)
+		}
+	}
 
 	// Cancel the session if something went wrong
 	defer atexit.OnError(&err, func() {
@@ -2918,7 +3176,7 @@ func (o *Object) ID() string {
 func (f *Fs) parseNormalizedID(ID string) (string, string, string) {
 	var rootURL string
 	if f.opt.TenantURL != "" {
-		rootURL = f.opt.TenantURL + "/v2.0/drives"
+		rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
 	} else {
 		rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
 	}
@@ -3086,9 +3344,15 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 				}
 			case <-tickerC:
 				fs.Debugf(f, "Checking for changes on remote")
-				nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
+				var newDeltaToken string
+				newDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
 				if err != nil {
+					// Keep the token we already have: replacing it with the
+					// empty token from a failed poll would reset the delta
+					// listing to the start and lose every change in between.
 					fs.Infof(f, "Change notify listener failure: %s", err)
+				} else {
+					nextDeltaToken = newDeltaToken
 				}
 			}
 		}
@@ -3096,7 +3360,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 }
 
 func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
-	delta, err := f.changeNotifyNextChange(ctx, "latest")
+	delta, err := f.changeNotifyNextChange(ctx, "latest", "")
 	if err != nil {
 		return
 	}
@@ -3108,18 +3372,37 @@ func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken str
 	return
 }
 
-func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta api.DeltaResponse, err error) {
-	opts := f.buildDriveDeltaOpts(token)
+// changeNotifyNextChange fetches one page of the drive delta listing.
+//
+// Pass the delta token to resume from in token, or the @odata.nextLink URL
+// returned by the previous page in nextLink. Exactly one of the two should
+// be set.
+func (f *Fs) changeNotifyNextChange(ctx context.Context, token, nextLink string) (delta api.DeltaResponse, err error) {
+	opts := f.buildDriveDeltaOpts(token, nextLink)
 
-	_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
+	err = f.pacer.Call(func() (bool, error) {
+		var resp *http.Response
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
+		return shouldRetry(ctx, resp, err)
+	})
 
 	return
 }
 
-func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
+// buildDriveDeltaOpts builds the request for one page of the drive delta
+// listing. nextLink, when set, is the complete @odata.nextLink URL from the
+// previous page and is used as-is, overriding the drive root.
+func (f *Fs) buildDriveDeltaOpts(token, nextLink string) rest.Opts {
+	if nextLink != "" {
+		return rest.Opts{
+			Method:  "GET",
+			RootURL: nextLink,
+		}
+	}
+
 	var rootURL string
 	if f.opt.TenantURL != "" {
-		rootURL = f.opt.TenantURL + "/v2.0/drives"
+		rootURL = tenantAPIEndpoint(f.opt.TenantURL, f.opt.TenantAPIVersion) + "/drives"
 	} else {
 		rootURL = graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
 	}
@@ -3132,46 +3415,58 @@ func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
 	}
 }
 
-func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
-	delta, err := f.changeNotifyNextChange(ctx, deltaToken)
-	if err != nil {
-		return
-	}
-	parsedURL, err := url.Parse(delta.DeltaLink)
-	if err != nil {
-		return
-	}
-	nextDeltaToken = parsedURL.Query().Get("token")
-
-	for _, item := range delta.Value {
-		isDriveRootFolder := item.GetParentReference().ID == ""
-		if isDriveRootFolder {
-			continue
-		}
-
-		fullPath, err := getItemFullPath(&item)
+// changeNotifyRunner polls for changes and calls notifyFunc for each one.
+//
+// It walks every page of the change set and returns the token to resume from
+// next time. The returned token is only valid when err is nil: the caller
+// must keep the token it passed in on error, otherwise a failed poll would
+// reset the delta listing to the start and lose the changes in between.
+func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (string, error) {
+	token, nextLink := deltaToken, ""
+	for {
+		delta, err := f.changeNotifyNextChange(ctx, token, nextLink)
 		if err != nil {
-			fs.Errorf(f, "Could not get item full path: %s", err)
-			continue
+			return "", err
 		}
 
-		if fullPath == f.root {
-			continue
+		for _, item := range delta.Value {
+			isDriveRootFolder := item.GetParentReference().ID == ""
+			if isDriveRootFolder {
+				continue
+			}
+
+			fullPath, err := getItemFullPath(&item)
+			if err != nil {
+				fs.Errorf(f, "Could not get item full path: %s", err)
+				continue
+			}
+
+			if fullPath == f.root {
+				continue
+			}
+
+			relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
+			if !insideRoot {
+				continue
+			}
+
+			if item.GetFile() != nil {
+				notifyFunc(relName, fs.EntryObject)
+			} else if item.GetFolder() != nil {
+				notifyFunc(relName, fs.EntryDirectory)
+			}
 		}
 
-		relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
-		if !insideRoot {
-			continue
-		}
-
-		if item.GetFile() != nil {
-			notifyFunc(relName, fs.EntryObject)
-		} else if item.GetFolder() != nil {
-			notifyFunc(relName, fs.EntryDirectory)
+		if delta.NextLink == "" {
+			// end of the change set - the deltaLink carries the resume token
+			parsedURL, err := url.Parse(delta.DeltaLink)
+			if err != nil {
+				return "", err
+			}
+			return parsedURL.Query().Get("token"), nil
 		}
+		nextLink = delta.NextLink
 	}
-
-	return
 }
 
 func getItemFullPath(item *api.Item) (fullPath string, err error) {

backend/onedrive/onedrive_internal_test.go

@@ -3,19 +3,27 @@ package onedrive
 import (
 	"context"
 	"encoding/json"
+	"flag"
 	"fmt"
+	"net/http"
+	"net/http/httptest"
 	"slices"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/backend/onedrive/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -37,11 +45,49 @@ const (
 	// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1
 )
 
+func init() {
+	flag.BoolVar(&stripTempauth, "strip-tempauth", false, "Remove tempauth from URLs the API returns to simulate the retirement of pre-authenticated URLs")
+}
+
 // TestMain drives the tests
 func TestMain(m *testing.M) {
 	fstest.TestMain(m)
 }
 
+func TestTenantAPIEndpoint(t *testing.T) {
+	for _, tc := range []struct {
+		name             string
+		tenantURL        string
+		tenantAPIVersion string
+		want             string
+	}{
+		{
+			name:      "default",
+			tenantURL: "https://example-my.sharepoint.com/_api",
+			want:      "https://example-my.sharepoint.com/_api/v2.0",
+		}, {
+			name:             "explicit v2.1",
+			tenantURL:        "https://example-my.sharepoint.com/_api",
+			tenantAPIVersion: "v2.1",
+			want:             "https://example-my.sharepoint.com/_api/v2.1",
+		}, {
+			name:             "future version",
+			tenantURL:        "https://example-my.sharepoint.com/_api",
+			tenantAPIVersion: "v2.2",
+			want:             "https://example-my.sharepoint.com/_api/v2.2",
+		}, {
+			name:             "trailing slash",
+			tenantURL:        "https://example-my.sharepoint.com/_api/",
+			tenantAPIVersion: "/v2.1",
+			want:             "https://example-my.sharepoint.com/_api/v2.1",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.want, tenantAPIEndpoint(tc.tenantURL, tc.tenantAPIVersion))
+		})
+	}
+}
+
 // sharingRefused caches whether the remote refuses sharing invitations.
 var sharingRefused *bool
 
@@ -574,3 +620,252 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// newChangeNotifyTestFs returns an Fs whose delta listing is served by
+// handler on a local test server, plus the server itself. The tenant URL
+// is pointed at the server so no request leaves the test.
+func newChangeNotifyTestFs(t *testing.T, handler http.Handler) (*Fs, *httptest.Server) {
+	t.Helper()
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+
+	ctx := context.Background()
+	f := &Fs{
+		ci:      fs.GetConfig(ctx),
+		root:    "",
+		driveID: "testdrive",
+		srv:     rest.NewClient(server.Client()).SetRoot(server.URL),
+		pacer:   fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+	}
+	f.opt.TenantURL = server.URL
+	f.opt.TenantAPIVersion = defaultTenantAPIVersion
+	return f, server
+}
+
+// deltaItem builds a delta entry for a file named name inside a folder.
+func deltaItem(name, folder string) api.Item {
+	return api.Item{
+		ID:   name,
+		Name: name,
+		File: &api.FileFacet{},
+		ParentReference: &api.ItemReference{
+			ID:   "parentid",
+			Path: "driveid:/" + folder,
+		},
+	}
+}
+
+// TestChangeNotifyRetriesAndKeepsToken covers a transient failure of the
+// delta poll: the request must go through the pacer so it is retried, and
+// a poll which fails outright must not hand back an empty token, since the
+// caller stores whatever it returns. An empty token would reset the delta
+// listing to the start and silently lose every change in between.
+func TestChangeNotifyRetriesAndKeepsToken(t *testing.T) {
+	ctx := context.Background()
+	var attempts atomic.Int32
+	var tokens []string
+	fail := true
+	f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		attempts.Add(1)
+		tokens = append(tokens, r.URL.Query().Get("token"))
+		if fail {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusServiceUnavailable)
+			_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{
+				"code":    "serviceNotAvailable",
+				"message": "Service unavailable",
+			}})
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(api.DeltaResponse{
+			Value:     []api.Item{deltaItem("file1.txt", "folder")},
+			DeltaLink: "https://example.com/delta?token=newtoken",
+		})
+	}))
+
+	var notified []string
+	notifyFunc := func(path string, entryType fs.EntryType) {
+		notified = append(notified, path)
+	}
+
+	// The poll fails, but is retried by the pacer first
+	token, err := f.changeNotifyRunner(ctx, notifyFunc, "tok1")
+	require.Error(t, err)
+	assert.Empty(t, token, "a failed poll must not return a token")
+	assert.Equal(t, fs.GetConfig(ctx).LowLevelRetries, int(attempts.Load()), "the failed poll should have been retried by the pacer")
+	assert.Equal(t, slices.Repeat([]string{"tok1"}, int(attempts.Load())), tokens, "every retry should resume from the same token")
+	assert.Empty(t, notified)
+
+	// Keeping the previous token (as ChangeNotify does) resumes correctly
+	fail = false
+	attempts.Store(0)
+	tokens = nil
+	token, err = f.changeNotifyRunner(ctx, notifyFunc, "tok1")
+	require.NoError(t, err)
+	assert.Equal(t, "newtoken", token)
+	assert.Equal(t, []string{"tok1"}, tokens, "the second poll should resume from the retained token")
+	assert.Equal(t, []string{"folder/file1.txt"}, notified)
+}
+
+// TestChangeNotifyFollowsNextLink covers a change set which spans more than
+// one page. Graph returns @odata.nextLink and no @odata.deltaLink until the
+// last page, so a runner which stops after the first page both loses the
+// remaining changes and ends up with an empty resume token.
+func TestChangeNotifyFollowsNextLink(t *testing.T) {
+	ctx := context.Background()
+	var urls []string
+	var server *httptest.Server
+	f, server := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		urls = append(urls, r.URL.RequestURI())
+		w.Header().Set("Content-Type", "application/json")
+		switch len(urls) {
+		case 1:
+			_ = json.NewEncoder(w).Encode(api.DeltaResponse{
+				Value:    []api.Item{deltaItem("page1.txt", "folder")},
+				NextLink: server.URL + "/nextpage?$skiptoken=abc",
+			})
+		case 2:
+			_ = json.NewEncoder(w).Encode(api.DeltaResponse{
+				Value:     []api.Item{deltaItem("page2.txt", "folder")},
+				DeltaLink: "https://example.com/delta?token=lasttoken",
+			})
+		default:
+			t.Errorf("unexpected extra request to %s", r.URL)
+		}
+	}))
+
+	var notified []string
+	token, err := f.changeNotifyRunner(ctx, func(path string, entryType fs.EntryType) {
+		notified = append(notified, path)
+	}, "tok1")
+	require.NoError(t, err)
+
+	assert.Equal(t, "lasttoken", token, "the resume token should come from the last page")
+	assert.Equal(t, []string{"folder/page1.txt", "folder/page2.txt"}, notified, "changes from every page should be notified")
+	require.Len(t, urls, 2, "the nextLink page should have been fetched")
+	assert.Contains(t, urls[0], "/testdrive/root/delta")
+	assert.Contains(t, urls[0], "token=tok1")
+	assert.Equal(t, "/nextpage?$skiptoken=abc", urls[1], "the nextLink should be used as-is, without the delta token")
+}
+
+// TestListDirectorySize checks that a listed directory carries the size
+// OneDrive reports for it, the total size of its contents, instead of -1.
+func TestListDirectorySize(t *testing.T) {
+	ctx := context.Background()
+	f, _ := newChangeNotifyTestFs(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/items/rootid/children", r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(api.ListChildrenResponse{
+			Value: []api.Item{{
+				ID:     "dirid",
+				Name:   "photos",
+				Size:   2646204760,
+				Folder: &api.FolderFacet{ChildCount: 3},
+			}},
+		})
+	}))
+	f.dirCache = dircache.New("", "rootid", f)
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	require.Len(t, entries, 1)
+	dir, ok := entries[0].(fs.Directory)
+	require.True(t, ok, "entry should be a directory")
+	assert.Equal(t, "photos", dir.Remote())
+	assert.Equal(t, int64(2646204760), dir.Size())
+	assert.Equal(t, int64(3), dir.Items())
+}
+
+// TestWaitForJobUsesConfiguredClient checks that the async job monitor is
+// polled with the configured HTTP client and request context rather than
+// http.DefaultClient, so proxy, TLS and timeout options and cancellation apply.
+func TestWaitForJobUsesConfiguredClient(t *testing.T) {
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.UserAgent = "rclone-waitforjob-test"
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "rclone-waitforjob-test", r.UserAgent())
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(api.AsyncOperationStatus{Status: "deleteFailed"})
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(ctx))
+
+	err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "deleteFailed")
+}
+
+// TestWaitForJobCancel checks that cancelling the context stops a poll of
+// the async job monitor which is not answering.
+func TestWaitForJobCancel(t *testing.T) {
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-release:
+		}
+	}))
+	t.Cleanup(server.Close)
+	t.Cleanup(func() { close(release) })
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"}) }()
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, context.DeadlineExceeded)
+	case <-time.After(5 * time.Second):
+		t.Fatal("waitForJob did not return after the context was cancelled")
+	}
+}
+
+// TestWaitForJobUnauthorized checks that a monitor URL which refuses
+// the request fails straight away instead of polling until the timeout.
+func TestWaitForJobUnauthorized(t *testing.T) {
+	var polls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		polls.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusUnauthorized)
+		_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "unauthenticated", "message": "no token"}})
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+	f.driveType = driveTypeBusiness
+
+	err := f.waitForJob(context.Background(), server.URL+"/monitor", &Object{remote: "file.txt"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "401")
+	assert.Equal(t, int32(1), polls.Load())
+}
+
+// TestWaitForJobRedirectCompleted checks that a monitor which
+// redirects to the new item, as it does when authenticated, is taken
+// to mean the job has finished.
+func TestWaitForJobRedirectCompleted(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/monitor":
+			http.Redirect(w, r, "/item", http.StatusSeeOther)
+		case "/item":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"id":"newid","name":"file.txt","size":5,"file":{}}`)
+		default:
+			t.Errorf("unexpected request %s", r.URL)
+		}
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt", hasMetaData: true})
+	require.NoError(t, err)
+}