docs: bring the Proton Drive handoff up to date after the final review

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

.claude/settings.json

@@ -0,0 +1,25 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git status:*)", "Bash(git diff:*)", "Bash(git log:*)", "Bash(git show:*)",
+      "Bash(git rev-parse:*)", "Bash(git add:*)", "Bash(git commit:*)",
+      "Bash(go mod download:*)", "Bash(go build:*)", "Bash(go test:*)", "Bash(go vet:*)",
+      "Bash(gofmt -l:*)",
+      "Bash(golangci-lint run:*)"
+    ],
+    "ask": [
+      "Bash(git push:*)", "Bash(git merge:*)", "Bash(git rebase:*)", "Bash(git switch:*)",
+      "Bash(git restore:*)", "Bash(git commit --amend:*)", "Bash(rm:*)",
+      "Bash(go get:*)", "Bash(go mod tidy:*)", "Bash(go install:*)",
+      "Bash(make:*)", "Bash(docker:*)"
+    ],
+    "deny": [
+      "Bash(git push --force:*)", "Bash(git push -f:*)", "Bash(git push --force-with-lease:*)",
+      "Bash(git reset --hard:*)", "Bash(git clean:*)", "Bash(git add -f:*)", "Bash(git add --force:*)",
+      "Bash(git commit --no-verify:*)", "Bash(git commit -n:*)",
+      "Bash(rm -r:*)", "Bash(rm -R:*)", "Bash(rm -rf:*)", "Bash(rm -fr:*)", "Bash(rm -Rf:*)",
+      "Bash(rm -fR:*)", "Bash(rm --recursive:*)", "Bash(sudo:*)",
+      "Read(./.env)", "Read(./.env.*)", "Read(./secrets/**)"
+    ]
+  }
+}

.codex/config.toml

@@ -0,0 +1,2 @@
+sandbox_mode = "workspace-write"
+approval_policy = "on-request"

.codex/rules/project.rules

@@ -0,0 +1,20 @@
+# Rendered from docs/PERMISSIONS.md - change both in the same commit.
+# Deny
+prefix_rule(pattern = ["git", "push", ["--force", "-f", "--force-with-lease"]], decision = "forbidden",
+            justification = "Force-push rewrites shared history.")
+prefix_rule(pattern = ["git", "reset", "--hard"], decision = "forbidden", justification = "Discards work.")
+prefix_rule(pattern = ["git", "clean"], decision = "forbidden", justification = "Deletes untracked files.")
+prefix_rule(pattern = ["git", "add", ["-f", "--force"]], decision = "forbidden",
+            justification = "Stages ignored files, e.g. secrets.")
+prefix_rule(pattern = ["git", "commit", ["--no-verify", "-n"]], decision = "forbidden", justification = "Skips hooks.")
+prefix_rule(pattern = ["rm", ["-r", "-R", "-rf", "-fr", "-Rf", "-fR", "--recursive"]], decision = "forbidden",
+            justification = "Recursive delete.")
+prefix_rule(pattern = ["sudo"], decision = "forbidden", justification = "No privilege escalation.")
+# Ask
+prefix_rule(pattern = ["git", ["push", "merge", "rebase", "switch", "restore"]], decision = "prompt")
+prefix_rule(pattern = ["git", "commit", "--amend"], decision = "prompt")
+prefix_rule(pattern = ["rm"], decision = "prompt")
+prefix_rule(pattern = ["go", ["get", "install"]], decision = "prompt")
+prefix_rule(pattern = ["go", "mod", "tidy"], decision = "prompt")
+prefix_rule(pattern = ["make"], decision = "prompt")
+prefix_rule(pattern = ["docker"], decision = "prompt")

.gemini/settings.json

@@ -0,0 +1,20 @@
+{
+  "tools": {
+    "allowed": [
+      "run_shell_command(git status)", "run_shell_command(git diff)", "run_shell_command(git log)",
+      "run_shell_command(git show)", "run_shell_command(git branch --list)",
+      "run_shell_command(git add)", "run_shell_command(git commit)",
+      "run_shell_command(go mod download)", "run_shell_command(go build)",
+      "run_shell_command(go test)", "run_shell_command(go vet)", "run_shell_command(gofmt -l)",
+      "run_shell_command(golangci-lint run)"
+    ],
+    "exclude": [
+      "run_shell_command(git push --force)", "run_shell_command(git push -f)",
+      "run_shell_command(git push --force-with-lease)", "run_shell_command(git reset --hard)",
+      "run_shell_command(git clean)", "run_shell_command(git add -f)", "run_shell_command(git add --force)",
+      "run_shell_command(git commit --no-verify)", "run_shell_command(git commit -n)",
+      "run_shell_command(rm -r)", "run_shell_command(rm -R)", "run_shell_command(rm -rf)",
+      "run_shell_command(rm -fr)", "run_shell_command(rm -Rf)", "run_shell_command(sudo)"
+    ]
+  }
+}

.gitattributes

@@ -8,6 +8,7 @@ lib/http/templates/** text eol=lf
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true
+cmd/bisync/testdata/*/golden/** linguist-generated=true
 
 # Don't fiddle with the line endings of test data
 **/testdata/** -text

.gitea/workflows/ci.yml

@@ -0,0 +1,56 @@
+# Fork CI for the Proton Drive backend. Every step is hermetic: the
+# tests run against the fake server in backend/protondrive/protontest.
+name: ci
+
+on:
+  push:
+  pull_request:
+
+env:
+  # Never pick up a config holding a real Proton remote.
+  RCLONE_CONFIG: /notfound
+
+jobs:
+  protondrive:
+    runs-on: ubuntu-latest
+    # The race suite takes about 6 minutes on a fast machine and 7 on
+    # two cores; the slow runner may take several times that.
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          # The line the code was verified with, not go.mod's minimum,
+          # so govulncheck sees a patched standard library.
+          go-version: '1.27.x'
+          # Caches the module and build caches, keyed on go.sum.
+          cache: true
+
+      - name: Build
+        run: go build ./...
+
+      - name: Vet
+        run: go vet ./backend/protondrive/...
+
+      - name: Lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.15.0
+          args: ./backend/protondrive/...
+
+      - name: Test
+        # Go's default 10 minute limit per test binary is too short
+        # for the race suite on a slow runner.
+        run: go test -race -timeout 40m ./backend/protondrive/...
+
+      - name: Fuzz
+        run: |
+          for target in $(go test -list '^Fuzz' ./backend/protondrive/ | grep '^Fuzz'); do
+            go test -run '^$' -fuzz "^${target}\$" -fuzztime 20s ./backend/protondrive/
+          done
+
+      - name: Vulnerability check
+        run: go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./backend/protondrive/...

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -48,14 +48,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -64,14 +64,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
           - job_name: go1.26
             os: ubuntu-latest
-            go: '~1.26.9'
+            go: '~1.26.10'
             quicktest: true
             racequicktest: true
 
@@ -177,9 +177,12 @@ jobs:
           # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
           # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
           unset HOMEBREW_NO_INSTALL_FROM_API
-          brew untap --force homebrew/core
-          brew untap --force homebrew/cask
-          brew untap --force aws/tap
+          # brew untap fails if the tap isn't installed, which varies between runner images
+          for tap in homebrew/core homebrew/cask aws/tap; do
+            if brew tap | grep -qx "$tap"; then
+              brew untap --force "$tap"
+            fi
+          done
           brew update
           brew install --cask macfuse
           brew install git-annex git-annex-remote-rclone
@@ -284,7 +287,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.27.2'
+          go-version: '~1.27.3'
           check-latest: true
           cache: false
           
@@ -399,7 +402,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.27.2'
+          go-version: '~1.27.3'
           # Caching is handled explicitly below to share the module cache
           # with the other jobs - see the build job for the rationale.
           cache: false

.gitignore

@@ -19,4 +19,12 @@ Thumbs.db
 __pycache__
 .DS_Store
 resource_windows_*.syso
+/librclone.h
 .devcontainer
+/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json
+.env
+.env.*
+CLAUDE.local.md
+.claude/settings.local.json
+.worktrees/
+.superpowers/

AGENTS.md

@@ -134,3 +134,16 @@ Uses golangci-lint v2 with config in `.golangci.yml`. Enabled linters: errcheck,
 - Command docs are in the command source code (e.g., `cmd/ls/ls.go`).
 - Don't commit autogenerated doc changes from `make backenddocs` or `make commanddocs`.
 - Website docs are in `docs/content/` as markdown, built with Hugo (`make serve` to preview).
+
+## Fork Workflow
+
+This fork carries agent tooling that upstream does not. Never include `docs/specs/`, `docs/plans/`, `docs/{ARCHITECTURE,DECISIONS,TODO,PERMISSIONS}.md`, `GEMINI.md`, `.claude/`, `.codex/`, `.gemini/` or this section in a PR to rclone/rclone.
+
+- Required skill: `superpowers` - check at session start, install if missing. There is no product UI here, so `impeccable` is not needed.
+- Specs go in `docs/specs/NNN-slug.md` (permanent). Plans go in `docs/plans/NNN-slug.md` and are removed with `git rm` before merge. This overrides the superpowers default paths.
+- Paused or in-flight work is listed under "Now" in `docs/TODO.md` with the handoff to read first.
+- Non-trivial work: brainstorm → spec → plan → TDD → review. For trivial fixes say "trivial, skipping spec" and proceed.
+- Log things noticed but deliberately not fixed in `docs/TODO.md`, record decisions in `docs/DECISIONS.md`, and keep `docs/ARCHITECTURE.md` in step with structural changes - in the same commit as the code.
+- `docs/PERMISSIONS.md` is the source for the `.claude/`, `.codex/` and `.gemini/` permission files - change them together.
+- Never push, force-push or commit directly to `master`. Ask before adding or upgrading a dependency (`go get`, `go mod tidy`).
+- Done means run in this session and seen passing: `go build`, `go test` on the touched packages, and `golangci-lint run` on them.

GEMINI.md

@@ -0,0 +1,8 @@
+# GEMINI.md
+
+**`AGENTS.md` is the single source of truth for this repository.** Read
+`./AGENTS.md` before doing anything else and follow it exactly.
+
+This file exists only so Gemini CLI finds the rules. It contains no project
+rules of its own - add those to `AGENTS.md`. If anything here ever conflicts
+with `AGENTS.md`, `AGENTS.md` wins.

Makefile

@@ -285,7 +285,7 @@ endif
 
 # Fetch the binary builds from GitHub actions
 fetch_binaries:
-	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
+	rclone -P sync --exclude "/testbuilds/**" --exclude "/index.{html,json}" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
 	cd docs && hugo server --logLevel info -w --disableFastRender --ignoreCache

README.md

@@ -24,16 +24,19 @@ directories to and from different cloud storage providers.
 ## Storage providers
 
 - 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
+- Abrha Storage [:page_facing_up:](https://rclone.org/s3/#abrhastorage)
 - Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
 - Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
 - Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
 - ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
+- Autonomys Auto Drive [:page_facing_up:](https://rclone.org/s3/#autonomys)
 - Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
 - Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
 - Box [:page_facing_up:](https://rclone.org/box/)
 - Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
 - China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
 - Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
+- Clever Cloud Cellar [:page_facing_up:](https://rclone.org/s3/#clever-cloud-cellar)
 - Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
 - Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
 - Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
@@ -71,8 +74,11 @@ directories to and from different cloud storage providers.
 - Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
 - IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
 - Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
+- Infomaniak Swiss Backup (S3) [:page_facing_up:](https://rclone.org/s3/#infomaniak)
+- Infomaniak Swiss Backup (Swift) [:page_facing_up:](https://rclone.org/swift/)
 - Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
 - IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
+- kDrive [:page_facing_up:](https://rclone.org/kdrive/)
 - Koofr [:page_facing_up:](https://rclone.org/koofr/)
 - Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
 - Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
@@ -88,6 +94,7 @@ directories to and from different cloud storage providers.
 - Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
 - Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
 - Minio [:page_facing_up:](https://rclone.org/s3/#minio)
+- Neon Object Storage [:page_facing_up:](https://rclone.org/s3/#neon)
 - Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
 - Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
 - OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)

RELEASE.md

@@ -32,10 +32,14 @@ This file describes how to make the various kinds of releases
 - git commit -a -v -m "Version v1.XX.0"
 - make check # if this fails, fix, git commit --amend and run it again
 - make retag # moves the tag onto the Version commit
-- git push origin # without --follow-tags so it doesn't push the tag if it fails
+- git push origin # without --follow-tags so it doesn't push the tag yet
+- Wait for the GitHub actions build of master to complete and check it is
+  green - this is the first time CI sees any commits which were only local,
+  such as security fixes, so it must pass before the tag is pushed
+  - if it fails: fix, git commit, make retag, git push origin and wait again
 - git push --follow-tags origin
-- \# Wait for the GitHub builds to complete then...
-- \# Don't commit anything until after make upload_github as these steps
+- \# Wait for the GitHub builds of the tag to complete then...
+- \# Don't commit or push anything until after make upload_github as these steps
   use the tag on HEAD to find the version
 - make fetch_binaries
 - make tarball
@@ -62,10 +66,14 @@ This file describes how to make the various kinds of releases
 Early in the next release cycle update the dependencies.
 
 - Active Pins
-  - `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
-    - See: [#9539](https://github.com/rclone/rclone/issues/9539)
+  - `github.com/coreos/go-systemd/v22` v22.6.0 - v22.7.0 fails to compile on netbsd
+  - `github.com/hanwen/go-fuse/v2` v2.11.0 - v2.12.0 fails to compile on 32
+    bit linux, awaiting fix for
+    [go-fuse/issues/628](https://github.com/hanwen/go-fuse/issues/628)
 - Review Active Pins in go.mod and remove if possible
 - `make updatedirect`
+- The update ignores the pins so put them back, eg
+  `go get github.com/coreos/go-systemd/v22@v22.6.0` then `go mod tidy`
 - `make GOTAGS=cmount`
 - `make compiletest`
 - Fix anything which doesn't compile at this point and commit changes here

VERSION

@@ -1 +1 @@
-v1.76.0
+v1.77.0

backend/all/all.go

@@ -39,6 +39,7 @@ import (
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/internxt"
 	_ "github.com/rclone/rclone/backend/jottacloud"
+	_ "github.com/rclone/rclone/backend/kdrive"
 	_ "github.com/rclone/rclone/backend/koofr"
 	_ "github.com/rclone/rclone/backend/linkbox"
 	_ "github.com/rclone/rclone/backend/local"

backend/archive/archive.go

@@ -194,8 +194,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 		// Path to the archive
 		archiveRemote := remote[:len(foundArchive.remote)]
-		// Remote is archive leaf name
-		foundArchive.remote = path.Base(archiveRemote)
+		// Remote is archive leaf name. This must understand remote
+		// names as the archive may be in the root, e.g. "remote:file.zip"
+		_, foundArchive.remote, err = fspath.Split(archiveRemote)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse archive path %q: %w", archiveRemote, err)
+		}
 		foundArchive.prefix = ""
 		// Point remote to archive file
 		remote = archiveRemote

backend/archive/archive_internal_test.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"testing"
 
+	_ "github.com/rclone/rclone/backend/alias"
 	"github.com/rclone/rclone/backend/archive/archiver"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
@@ -323,6 +324,50 @@ func TestArchiveUncleanRoot(t *testing.T) {
 	}
 }
 
+// TestArchiveInRemoteRoot checks that an archive in the root of a
+// remote, with no "/" between the remote name and the archive name, can
+// be read, as can a path within it.
+func TestArchiveInRemoteRoot(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create("sub/a.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test.zip"), buf.Bytes(), 0600))
+	t.Chdir(dir)
+
+	for _, test := range []struct {
+		name   string
+		remote string
+	}{
+		{name: "plain", remote: ":local:test.zip"},
+		{name: "slash in remote name", remote: ":alias,remote='" + dir + "':test.zip"},
+	} {
+		for _, sub := range []struct {
+			path string
+			want string
+		}{
+			{path: "", want: "sub"},
+			{path: "/sub", want: "a.txt"},
+		} {
+			t.Run(test.name+sub.path, func(t *testing.T) {
+				f, err := cache.Get(ctx, ":archive:"+test.remote+sub.path)
+				require.NoError(t, err)
+				entries, err := f.List(ctx, "")
+				require.NoError(t, err)
+				require.Len(t, entries, 1)
+				assert.Equal(t, sub.want, entries[0].Remote())
+			})
+		}
+	}
+}
+
 // escapingObject is an object whose remote is not where it was asked for.
 type escapingObject struct {
 	fs.Object

backend/azurefiles/azurefiles.go

@@ -79,7 +79,7 @@ This is required and is the name of the share to access.
 			Help: `Upload chunk size.
 
 Note that this is stored in memory and there may be up to
-"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
+"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
 in memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
@@ -95,7 +95,7 @@ links and these uploads do not fully utilize your bandwidth, then
 increasing this may help to speed up the transfers.
 
 Note that chunks are stored in memory and there may be up to
-"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
+"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
 in memory.`,
 			Default:  16,
 			Advanced: true,

backend/b2/b2.go

@@ -277,7 +277,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
 			Name: "sse_customer_key",
 			Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
 
-Alternatively you can provide --sse-customer-key-base64.`,
+Alternatively you can provide --b2-sse-customer-key-base64.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -288,7 +288,7 @@ Alternatively you can provide --sse-customer-key-base64.`,
 			Name: "sse_customer_key_base64",
 			Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
 
-Alternatively you can provide --sse-customer-key.`,
+Alternatively you can provide --b2-sse-customer-key.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",

backend/box/box.go

@@ -27,7 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/golang-jwt/jwt/v4"
+	"github.com/golang-jwt/jwt/v5"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -77,7 +77,10 @@ var (
 )
 
 type boxCustomClaims struct {
-	jwt.StandardClaims
+	jwt.RegisteredClaims
+	// Audience overrides RegisteredClaims.Audience so it is sent as
+	// a single string rather than an array of strings
+	Audience   string `json:"aud,omitempty"`
 	BoxSubType string `json:"box_sub_type,omitempty"`
 }
 
@@ -223,7 +226,7 @@ func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
 
 	if configFileCredentials == "" {
 		configFile, _ := m.Get("box_config_file")
-		configFileBytes, err = os.ReadFile(configFile)
+		configFileBytes, err = os.ReadFile(env.ShellExpand(configFile))
 		if err != nil {
 			return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 		}
@@ -243,15 +246,13 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	}
 
 	claims = &boxCustomClaims{
-		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
-		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
-		StandardClaims: jwt.StandardClaims{
-			Id:        val,
+		RegisteredClaims: jwt.RegisteredClaims{
+			ID:        val,
 			Issuer:    boxConfig.BoxAppSettings.ClientID,
 			Subject:   boxConfig.EnterpriseID,
-			Audience:  tokenURL,
-			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
 		},
+		Audience:   tokenURL,
 		BoxSubType: boxSubType,
 	}
 	return claims, nil

backend/box/box_internal_test.go

@@ -0,0 +1,118 @@
+package box
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/box/api"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/lib/jwtutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// decodeJWTPart decodes one base64url encoded part of a JWT into a map
+func decodeJWTPart(t *testing.T, part string) map[string]any {
+	data, err := base64.RawURLEncoding.DecodeString(part)
+	require.NoError(t, err)
+	var result map[string]any
+	require.NoError(t, json.Unmarshal(data, &result))
+	return result
+}
+
+// Test the JWT assertion sent to the token endpoint is in the format Box expects
+func TestJWTAssertion(t *testing.T) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+	boxConfig := &api.ConfigJSON{
+		BoxAppSettings: api.AppSettings{
+			ClientID:     "client-id",
+			ClientSecret: "client-secret",
+			AppAuth: api.AppAuth{
+				PublicKeyID: "public-key-id",
+			},
+		},
+		EnterpriseID: "enterprise-id",
+	}
+
+	var form map[string][]string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "POST", r.Method)
+		assert.NoError(t, r.ParseForm())
+		form = r.PostForm
+		_, _ = w.Write([]byte(`{"access_token":"access-token","token_type":"bearer","expires_in":3600}`))
+	}))
+	defer ts.Close()
+
+	claims, err := getClaims(boxConfig, "enterprise")
+	require.NoError(t, err)
+	m := configmap.Simple{}
+	start := time.Now()
+	err = jwtutil.Config("box", "test", ts.URL, *claims, getSigningHeaders(boxConfig), getQueryParams(boxConfig), privateKey, m, ts.Client())
+	require.NoError(t, err)
+
+	// Check the token was stored
+	assert.Contains(t, m["token"], `"access_token":"access-token"`)
+
+	// Check the form parameters
+	assert.Equal(t, []string{"urn:ietf:params:oauth:grant-type:jwt-bearer"}, form["grant_type"])
+	assert.Equal(t, []string{"client-id"}, form["client_id"])
+	assert.Equal(t, []string{"client-secret"}, form["client_secret"])
+	require.Len(t, form["assertion"], 1)
+	parts := strings.Split(form["assertion"][0], ".")
+	require.Len(t, parts, 3)
+
+	// Check the signature independently of the JWT library
+	signature, err := base64.RawURLEncoding.DecodeString(parts[2])
+	require.NoError(t, err)
+	hash := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
+	require.NoError(t, rsa.VerifyPKCS1v15(&privateKey.PublicKey, crypto.SHA256, hash[:], signature))
+
+	// Check the header
+	assert.Equal(t, map[string]any{
+		"alg": "RS256",
+		"typ": "JWT",
+		"kid": "public-key-id",
+	}, decodeJWTPart(t, parts[0]))
+
+	// Check the claims - aud must be a string, not an array
+	payload := decodeJWTPart(t, parts[1])
+	exp, ok := payload["exp"].(float64)
+	require.True(t, ok, "exp should be a number")
+	assert.InDelta(t, float64(start.Add(45*time.Second).Unix()), exp, 5)
+	jti, ok := payload["jti"].(string)
+	require.True(t, ok, "jti should be a string")
+	assert.Len(t, jti, 40)
+	delete(payload, "exp")
+	delete(payload, "jti")
+	assert.Equal(t, map[string]any{
+		"aud":          tokenURL,
+		"iss":          "client-id",
+		"sub":          "enterprise-id",
+		"box_sub_type": "enterprise",
+	}, payload)
+}
+
+// Test box_config_file has ~ and environment variables expanded
+func TestGetBoxConfigShellExpand(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"enterpriseID":"enterprise-id"}`), 0600))
+	t.Setenv("RCLONE_TEST_BOX_CONFIG_DIR", dir)
+	m := configmap.Simple{
+		"box_config_file": "$RCLONE_TEST_BOX_CONFIG_DIR/config.json",
+	}
+	boxConfig, err := getBoxConfig(m)
+	require.NoError(t, err)
+	assert.Equal(t, "enterprise-id", boxConfig.EnterpriseID)
+}

backend/chunker/chunker.go

@@ -1502,11 +1502,21 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
 func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
 	oldFsObject, err := f.NewObject(ctx, remote)
 	if err == nil {
-		oldObject := oldFsObject.(*Object)
-		for _, chunk := range oldObject.chunks {
-			if err := chunk.Remove(ctx); err != nil {
-				fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
-			}
+		removeStaleChunks(ctx, oldFsObject.(*Object), nil)
+	}
+}
+
+func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
+	currentRemotes := make(map[string]struct{}, len(currentChunks))
+	for _, chunk := range currentChunks {
+		currentRemotes[chunk.Remote()] = struct{}{}
+	}
+	for _, chunk := range oldObject.chunks {
+		if _, found := currentRemotes[chunk.Remote()]; found {
+			continue
+		}
+		if err := chunk.Remove(ctx); err != nil {
+			fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
 		}
 	}
 }
@@ -1687,12 +1697,20 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		// metadata format which might involve unsupported chunk types.
 		return nil, fmt.Errorf("can't %s this file: %w", opName, err)
 	}
+	// Snapshot destination chunks before its metadata is replaced.
+	var oldObject *Object
+	if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
+		oldObject = oldFsObject.(*Object)
+	}
 	if !o.isComposite() {
 		fs.Debugf(o, "%s non-chunked object...", opName)
 		oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
 		if err != nil {
 			return nil, err
 		}
+		if oldObject != nil {
+			removeStaleChunks(ctx, oldObject, nil)
+		}
 		return f.newObject("", oResult, nil), nil
 	}
 
@@ -1758,6 +1776,9 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		silentlyRemove(ctx, newObj)
 		return nil, err
 	}
+	if oldObject != nil {
+		removeStaleChunks(ctx, oldObject, newChunks)
+	}
 	return newObj, nil
 }
 

backend/chunker/chunker_internal_test.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	_ "github.com/rclone/rclone/backend/memory"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/fspath"
@@ -877,6 +878,104 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	_ = operations.Purge(ctx, f.base, dir)
 }
 
+func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
+	ctx := context.Background()
+	const dir = "serveroverwrite"
+	remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
+	fsResult, err := fs.NewFs(ctx, remote)
+	require.NoError(t, err)
+	copyFs, ok := fsResult.(*Fs)
+	require.True(t, ok)
+	oldMoveChunkSize := moveFs.opt.ChunkSize
+	moveFs.opt.ChunkSize = fs.SizeSuffix(3)
+	defer func() {
+		moveFs.opt.ChunkSize = oldMoveChunkSize
+		_ = operations.Purge(ctx, copyFs.base, "")
+		_ = operations.Purge(ctx, moveFs.base, dir)
+	}()
+
+	type operation struct {
+		name    string
+		fs      *Fs
+		enabled bool
+		do      copyMoveFn
+	}
+	operations := []operation{
+		{name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
+		{name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
+	}
+	testCases := []struct {
+		name           string
+		sourceContents string
+		targetContents string
+	}{
+		{name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
+		{name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
+	}
+	transactionModes := []struct {
+		name        string
+		useNoRename bool
+	}{
+		{name: "Rename"},
+		{name: "NoRename", useNoRename: true},
+	}
+
+	for _, operation := range operations {
+		t.Run(operation.name, func(t *testing.T) {
+			if !operation.enabled {
+				t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
+			}
+			for _, transactionMode := range transactionModes {
+				t.Run(transactionMode.name, func(t *testing.T) {
+					f := operation.fs
+					if transactionMode.useNoRename && !f.useMeta {
+						t.Skip("Can't test norename transactions without metadata")
+					}
+					oldUseNoRename := f.useNoRename
+					f.useNoRename = transactionMode.useNoRename
+					defer func() { f.useNoRename = oldUseNoRename }()
+					for _, testCase := range testCases {
+						t.Run(testCase.name, func(t *testing.T) {
+							prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
+							source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
+							target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
+							targetObject := target.(*Object)
+							require.NotEmpty(t, targetObject.chunks)
+
+							oldChunks := append([]fs.Object(nil), targetObject.chunks...)
+							result, err := operation.do(ctx, source, target.Remote())
+							require.NoError(t, err)
+
+							reader, err := result.Open(ctx)
+							require.NoError(t, err)
+							contents, err := io.ReadAll(reader)
+							require.NoError(t, err)
+							require.NoError(t, reader.Close())
+							assert.Equal(t, testCase.sourceContents, string(contents))
+
+							resultObject := result.(*Object)
+							currentChunks := make(map[string]struct{}, len(resultObject.chunks))
+							for _, chunk := range resultObject.chunks {
+								currentChunks[chunk.Remote()] = struct{}{}
+							}
+							removed := 0
+							for _, chunk := range oldChunks {
+								if _, found := currentChunks[chunk.Remote()]; found {
+									continue
+								}
+								_, err := f.base.NewObject(ctx, chunk.Remote())
+								assert.Equal(t, fs.ErrorObjectNotFound, err)
+								removed++
+							}
+							assert.Positive(t, removed)
+						})
+					}
+				})
+			}
+		})
+	}
+}
+
 // Test that md5all creates metadata even for small files
 func testMD5AllSlow(t *testing.T, f *Fs) {
 	ctx := context.Background()
@@ -950,6 +1049,9 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("ChunkerServerSideMove", func(t *testing.T) {
 		testChunkerServerSideMove(t, f)
 	})
+	t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
+		testChunkerServerSideOverwrite(t, f)
+	})
 	t.Run("MD5AllSlow", func(t *testing.T) {
 		testMD5AllSlow(t, f)
 	})

backend/crypt/crypt.go

@@ -1197,7 +1197,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash
-	if srcObj.Fs().Features().IsLocal {
+	if srcObj != nil && srcObj.Fs().Features().IsLocal {
 		// Read the data and encrypt it to calculate the hash
 		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
 		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)

backend/crypt/crypt_internal_test.go

@@ -93,6 +93,16 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
 }
 
+// Test ObjectInfo.Hash doesn't panic when the source is an
+// fs.OverrideRemote which wraps an fs.ObjectInfo that isn't an fs.Object
+func TestObjectInfoHashOverrideRemoteNotObject(t *testing.T) {
+	src := object.NewStaticObjectInfo("file", time.Now(), 3, true, nil, nil)
+	oi := (&Fs{}).newObjectInfo(fs.NewOverrideRemote(src, "new_remote"), nonce{})
+	gotHash, err := oi.Hash(context.Background(), hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, "", gotHash)
+}
+
 func testComputeHash(t *testing.T, f *Fs) {
 	var (
 		contents = random.String(100)

backend/drime/drime.go

@@ -321,10 +321,13 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
 		}
 		return false
 	})
+	if err != nil {
+		return nil, err
+	}
 	if !found {
 		return nil, fs.ErrorObjectNotFound
 	}
-	return info, err
+	return info, nil
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -546,6 +549,11 @@ OUTER:
 		if err != nil {
 			return found, fmt.Errorf("couldn't list files: %w", err)
 		}
+		// Drime won't page past a fixed number of entries (20,000 for
+		// rclone) and serves the last page it allows again instead
+		if result.CurrentPage != page {
+			return found, errors.New("couldn't list files: directory has more entries than drime will list")
+		}
 		for _, item := range result.Data {
 			if item.Type == api.ItemTypeFolder {
 				if filesOnly {

backend/drime/drime_internal_test.go

@@ -0,0 +1,158 @@
+package drime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/drime/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testPerPage = 5  // entries per page the test server returns
+	testLimit   = 20 // entries the test server will page through
+)
+
+// listingServer serves /drive/file-entries from entries the way Drime
+// does.
+//
+// It returns testPerPage entries per page and won't page past testLimit
+// entries. Asking for a later page returns the last page it allows
+// again, with last_page one beyond it.
+//
+// It counts the requests made in requests.
+func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		assert.Equal(t, "/drive/file-entries", r.URL.Path)
+		params := r.URL.Query()
+		page, err := strconv.Atoi(params.Get("page"))
+		require.NoError(t, err)
+		maxPage := testLimit / testPerPage
+		currentPage := min(page, maxPage)
+		start := min((currentPage-1)*testPerPage, len(entries))
+		end := min(start+testPerPage, len(entries))
+		lastPage := currentPage
+		if page > maxPage || end < len(entries) {
+			lastPage = currentPage + 1
+		}
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"current_page": currentPage,
+			"last_page":    lastPage,
+			"per_page":     testPerPage,
+			"total":        (currentPage + 1) * testPerPage,
+			"data":         entries[start:end],
+		}))
+	}))
+}
+
+// makeEntries makes n file entries
+func makeEntries(n int) []api.Item {
+	entries := make([]api.Item, n)
+	for i := range entries {
+		entries[i] = api.Item{
+			ID:   json.Number(strconv.Itoa(1000 + i)),
+			Name: fmt.Sprintf("file%05d", i),
+			Type: "text",
+		}
+	}
+	return entries
+}
+
+// testFs makes an Fs which uses server as its API
+func testFs(t *testing.T, server *httptest.Server) *Fs {
+	t.Helper()
+	f := &Fs{
+		opt: Options{ListChunk: 1000},
+		srv: rest.NewClient(server.Client()).SetRoot(server.URL),
+	}
+	f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+	f.srv.SetErrorHandler(errorHandler)
+	return f
+}
+
+// listNames lists a directory with listAll and returns the names found
+func listNames(ctx context.Context, f *Fs) (names []string, err error) {
+	_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
+		names = append(names, item.Name)
+		return false
+	})
+	return names, err
+}
+
+// TestListAll checks that listAll returns every entry exactly once
+func TestListAll(t *testing.T) {
+	for _, test := range []struct {
+		name    string
+		entries int
+	}{
+		{name: "Empty", entries: 0},
+		{name: "OnePage", entries: 3},
+		{name: "ExactPages", entries: 10},
+		{name: "PartialLastPage", entries: 13},
+		{name: "AtLimit", entries: testLimit},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			var requests atomic.Int64
+			entries := makeEntries(test.entries)
+			server := listingServer(t, entries, &requests)
+			defer server.Close()
+			f := testFs(t, server)
+
+			names, err := listNames(ctx, f)
+			require.NoError(t, err)
+			var want []string
+			for _, entry := range entries {
+				want = append(want, entry.Name)
+			}
+			assert.ElementsMatch(t, want, names)
+		})
+	}
+}
+
+// TestListAllTooManyEntries checks that listAll returns an error for a
+// directory with more entries than the server will list, rather than
+// looping forever.
+func TestListAllTooManyEntries(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	var requests atomic.Int64
+	server := listingServer(t, makeEntries(2*testLimit), &requests)
+	defer server.Close()
+	f := testFs(t, server)
+
+	_, err := listNames(ctx, f)
+	require.Error(t, err)
+	assert.NoError(t, ctx.Err(), "listing didn't finish")
+	assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
+}
+
+// TestGetItemListError checks that getItem returns listing errors
+// rather than fs.ErrorObjectNotFound.
+func TestGetItemListError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
+	}))
+	defer server.Close()
+	f := testFs(t, server)
+
+	_, err := f.getItem(context.Background(), "1000", "1", "file00000")
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
+	assert.Contains(t, err.Error(), "access denied")
+}

backend/drive/drive.go

@@ -568,7 +568,7 @@ recommended usage is using the flag form --drive-size-as-quota when
 doing rclone ls/lsl/lsf/lsjson/etc only.
 
 If you do use this flag for syncing (not recommended) then you will
-need to use --ignore size also.`,
+need to use --ignore-size also.`,
 			Advanced: true,
 			Hide:     fs.OptionHideConfigurator,
 		}, {
@@ -2298,15 +2298,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
 		for len(overflow) > 0 {
-			mu.Lock()
-			// only fill half of the channel to prevent entries being put into overflow again
-			l := min(len(overflow), listRInputBuffer/2)
-			wg.Add(l)
-			for _, d := range overflow[:l] {
-				in <- d
+			if !refillListRInput(&mu, &in, &overflow, &wg) {
+				return
 			}
-			overflow = overflow[l:]
-			mu.Unlock()
 
 			// wait again for the completion of all directories
 			wg.Wait()
@@ -2354,6 +2348,25 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	return nil
 }
 
+// refillListRInput moves queued directory jobs back onto in.
+// It returns false when a listing error has already closed in.
+// Sending on that nil channel would block forever while holding mu.
+func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
+	mu.Lock()
+	defer mu.Unlock()
+	if *in == nil {
+		return false
+	}
+	// only fill half of the channel to prevent entries being put into overflow again
+	l := min(len(*overflow), listRInputBuffer/2)
+	wg.Add(l)
+	for _, d := range (*overflow)[:l] {
+		*in <- d
+	}
+	*overflow = (*overflow)[l:]
+	return true
+}
+
 const shortcutSeparator = '\t'
 
 // joinID adds an actual drive ID to the shortcut ID it came from

backend/drive/drive_internal_test.go

@@ -916,3 +916,51 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
+	mu := stdsync.Mutex{}
+	wg := stdsync.WaitGroup{}
+	in := make(chan listREntry, listRInputBuffer)
+	overflow := []listREntry{{id: "folder", path: "folder"}}
+
+	mu.Lock()
+	close(in)
+	in = nil
+	mu.Unlock()
+
+	done := make(chan bool, 1)
+	go func() {
+		done <- refillListRInput(&mu, &in, &overflow, &wg)
+	}()
+	select {
+	case ok := <-done:
+		require.False(t, ok, "refill continued after the input channel was closed")
+	case <-time.After(time.Second):
+		t.Fatal("refill blocked after the input channel was closed")
+	}
+	require.Len(t, overflow, 1)
+
+	finished := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(time.Second):
+		t.Fatal("refill incremented the wait group after the input channel was closed")
+	}
+}
+
+func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
+	mu := stdsync.Mutex{}
+	wg := stdsync.WaitGroup{}
+	in := make(chan listREntry, listRInputBuffer)
+	overflow := []listREntry{{id: "folder", path: "folder"}}
+
+	require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
+	require.Empty(t, overflow)
+	got := <-in
+	require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
+	wg.Done()
+}

backend/ftp/ftp.go

@@ -584,8 +584,9 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
 //
 // It nils the pointed to connection out so it can't be reused
 //
-// if err is not nil then it checks the connection is alive using a
-// NOOP request
+// if err is not nil then it discards the connection unless the error
+// carries an FTP reply code, in which case it checks the connection
+// is alive using a NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	if f.opt.Concurrency > 0 {
 		defer f.tokens.Put()
@@ -599,14 +600,17 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	*pc = nil
 	if err != nil {
-		// If not a regular FTP error code then check the connection
-		if tpErr := textprotoError(err); tpErr != nil {
-			nopErr := c.NoOp()
-			if nopErr != nil {
-				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
-				_ = c.Quit()
-				return
-			}
+		// An error without an FTP reply code means the control
+		// connection can no longer be trusted, so discard it.
+		if textprotoError(err) == nil {
+			fs.Debugf(f, "Discarding connection after non-FTP error: %v", err)
+			_ = c.Quit()
+			return
+		}
+		if nopErr := c.NoOp(); nopErr != nil {
+			fs.Debugf(f, "Connection failed, closing: %v", nopErr)
+			_ = c.Quit()
+			return
 		}
 	}
 	f.poolMu.Lock()
@@ -1265,15 +1269,18 @@ func (o *Object) Storable() bool {
 
 // ftpReadCloser implements io.ReadCloser for FTP objects.
 type ftpReadCloser struct {
-	rc  io.ReadCloser
-	c   *ftp.ServerConn
-	f   *Fs
-	err error // errors found during read
+	rc     io.ReadCloser
+	c      *ftp.ServerConn
+	f      *Fs
+	err    error // errors found during read
+	closed bool  // set by Close so a second call is a no-op
+	read   int64 // bytes returned by Read
 }
 
 // Read bytes into p
 func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 	n, err = f.rc.Read(p)
+	f.read += int64(n)
 	if err != nil && err != io.EOF {
 		f.err = err // store any errors for Close to examine
 	}
@@ -1282,6 +1289,10 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
+	if f.closed {
+		return nil
+	}
+	f.closed = true
 	var err error
 	errchan := make(chan error, 1)
 	go func() {
@@ -1305,6 +1316,7 @@ func (f *ftpReadCloser) Close() error {
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
+		f.c = nil // never pool a connection after Quit
 		f.f.putFtpConnection(nil, nil)
 	} else {
 		f.f.putFtpConnection(&f.c, nil)
@@ -1316,6 +1328,12 @@ func (f *ftpReadCloser) Close() error {
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend, ftp.StatusRequestedFileActionOK:
 			err = nil
+		case ftp.StatusCanNotOpenDataConnection:
+			// Egnyte answers 425 when a RETR is closed before the first
+			// read; a transfer cut after data was read stays an error
+			if f.read == 0 {
+				err = nil
+			}
 		}
 	}
 	return err

backend/ftp/ftp_internal_test.go

@@ -1,12 +1,19 @@
 package ftp
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
+	"io"
+	"net"
+	"net/textproto"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
@@ -144,3 +151,159 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// fakeFTPServer is a minimal FTP control-connection server which answers
+// every command with 200 OK.
+type fakeFTPServer struct {
+	ln   net.Listener
+	mu   sync.Mutex
+	conn net.Conn
+}
+
+func startFakeFTPServer(t *testing.T) *fakeFTPServer {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	s := &fakeFTPServer{ln: ln}
+	go s.serve()
+	t.Cleanup(func() { s.close() })
+	return s
+}
+
+func (s *fakeFTPServer) serve() {
+	conn, err := s.ln.Accept()
+	if err != nil {
+		return
+	}
+	s.mu.Lock()
+	s.conn = conn
+	s.mu.Unlock()
+	defer func() { _ = conn.Close() }()
+	_, _ = fmt.Fprint(conn, "220 fake FTP server\r\n")
+	scanner := bufio.NewScanner(conn)
+	for scanner.Scan() {
+		_, _ = fmt.Fprint(conn, "200 OK\r\n")
+	}
+}
+
+// killConnection closes the server side of the control connection,
+// simulating a server that dropped an idle connection.
+func (s *fakeFTPServer) killConnection() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.conn != nil {
+		_ = s.conn.Close()
+	}
+}
+
+func (s *fakeFTPServer) close() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.conn != nil {
+		_ = s.conn.Close()
+	}
+	_ = s.ln.Close()
+}
+
+// TestPutFtpConnectionDiscardsDeadConnection checks that a connection
+// returned with a raw network error is closed and never re-pooled, while
+// a connection returned with a plain FTP reply-code error is probed with
+// NOOP and pooled while the probe succeeds.
+func TestPutFtpConnectionDiscardsDeadConnection(t *testing.T) {
+	t.Run("network error discards", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, errors.New("unexpected EOF"))
+		assert.Nil(t, c)
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("FTP reply error with live connection pools", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
+		assert.Nil(t, c)
+		require.Len(t, f.pool, 1)
+		assert.NoError(t, f.pool[0].NoOp())
+		_ = f.pool[0].Quit()
+	})
+
+	t.Run("FTP reply error with dead connection discards", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		server.killConnection()
+		// Let the FIN propagate so the NOOP probe observes the dead connection.
+		time.Sleep(100 * time.Millisecond)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
+		assert.Nil(t, c)
+		assert.Empty(t, f.pool)
+	})
+}
+
+// fakeDataReader stands in for the RETR data stream of ftpReadCloser.
+type fakeDataReader struct {
+	data     []byte
+	closeErr error
+}
+
+func (r *fakeDataReader) Read(p []byte) (int, error) {
+	if len(r.data) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, r.data)
+	r.data = r.data[n:]
+	return n, nil
+}
+
+func (r *fakeDataReader) Close() error { return r.closeErr }
+
+// TestFtpReadCloserClose checks that an early-closed RETR answered with 425
+// is masked only when nothing was read, that a connection which got QUIT is
+// never pooled, and that a second Close does not touch the pool.
+func TestFtpReadCloserClose(t *testing.T) {
+	err425 := &textproto.Error{Code: ftp.StatusCanNotOpenDataConnection, Msg: "Can't open data connection."}
+	newReader := func(t *testing.T, data []byte, closeErr error) (*ftpReadCloser, *Fs) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		return &ftpReadCloser{rc: &fakeDataReader{data: data, closeErr: closeErr}, c: c, f: f}, f
+	}
+
+	t.Run("425 before any read is masked and the connection is not pooled", func(t *testing.T) {
+		rc, f := newReader(t, nil, err425)
+		assert.NoError(t, rc.Close())
+		assert.Empty(t, f.pool)
+		assert.NoError(t, rc.Close())
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("425 after data was read stays an error", func(t *testing.T) {
+		rc, f := newReader(t, []byte("partial"), err425)
+		buf := make([]byte, 4)
+		n, err := rc.Read(buf)
+		require.NoError(t, err)
+		require.Equal(t, 4, n)
+		err = rc.Close()
+		require.Error(t, err)
+		assert.Equal(t, ftp.StatusCanNotOpenDataConnection, textprotoError(err).Code)
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("clean close pools the connection once", func(t *testing.T) {
+		rc, f := newReader(t, []byte("data"), nil)
+		_, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		assert.NoError(t, rc.Close())
+		require.Len(t, f.pool, 1)
+		assert.NoError(t, rc.Close())
+		assert.Len(t, f.pool, 1)
+		_ = f.pool[0].Quit()
+	})
+}

backend/hasher/hasher_internal_test.go

@@ -122,6 +122,49 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {
 	_ = operations.Purge(ctx, f, dirName)
 }
 
+func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {
+	ctx := context.Background()
+
+	const dirName = "setmodtime_hash_1"
+	const fileName = dirName + "/file_setmodtime_1"
+	const longTime = fs.ModTimeNotSupported
+	hashType := f.keepHashes.GetOne()
+
+	// upload a file to hasher via Put
+	src := putFile(ctx, t, f, fileName, "setmodtime content")
+	dst, err := f.NewObject(ctx, fileName)
+	require.NoError(t, err)
+	require.NotNil(t, dst)
+	_ = src
+
+	// verify hash was stored after Put
+	if f.opt.MaxAge <= 0 {
+		t.Skip("hash cache is disabled")
+	}
+	hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+	require.NoError(t, err)
+	require.NotEmpty(t, hash1)
+
+	// change only the modification time, as operations does when it
+	// syncs the modtime of an otherwise unchanged destination file
+	newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")
+	require.NoError(t, dst.SetModTime(ctx, newTime))
+
+	// the cached hash must survive a pure modtime change
+	hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+	assert.NoError(t, err)
+	assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")
+
+	// and it must be found via the fingerprint-checked lookup too
+	dst2, err := f.NewObject(ctx, fileName)
+	require.NoError(t, err)
+	hashVal, err := dst2.Hash(ctx, hashType)
+	assert.NoError(t, err)
+	assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")
+
+	_ = operations.Purge(ctx, f, dirName)
+}
+
 // InternalTest dispatches all internal tests
 func (f *Fs) InternalTest(t *testing.T) {
 	if !kv.Supported() {
@@ -129,6 +172,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	}
 	t.Run("UploadFromCrypt", f.testUploadFromCrypt)
 	t.Run("UpdateStoresHash", f.testUpdateStoresHash)
+	t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/hasher/kv.go

@@ -206,6 +206,34 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
 	return err
 }
 
+// kvUpdateFingerprint: rewrite the fingerprint of an existing record
+type kvUpdateFingerprint struct {
+	key   string
+	oldFp string
+	newFp string
+	age   time.Duration
+}
+
+func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {
+	data := b.Get([]byte(op.key))
+	if len(data) == 0 {
+		return nil
+	}
+	var r hashRecord
+	if err := r.decode(op.key, data); err != nil {
+		return nil
+	}
+	if r.Fp != op.oldFp || time.Since(r.Created) > op.age {
+		return nil
+	}
+	r.Fp = op.newFp
+	data, err := r.encode(op.key)
+	if err != nil {
+		return err
+	}
+	return b.Put([]byte(op.key), data)
+}
+
 // kvDump: dump the database.
 // Note: long dump can cause concurrent operations to fail.
 type kvDump struct {

backend/hasher/object.go

@@ -56,6 +56,23 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {
 	return o.f.putRawHashes(ctx, key, fp, hashes)
 }
 
+// updateFingerprint rewrites the fingerprint of the cached hash
+// record for remote from oldFp to newFp, keeping the hashes. It is
+// used when only the modification time of an object changes, so the
+// cached hashes remain valid under the new fingerprint.
+func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {
+	if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {
+		return nil
+	}
+	key := path.Join(f.Fs.Root(), remote)
+	return f.db.Do(true, &kvUpdateFingerprint{
+		key:   key,
+		oldFp: oldFp,
+		newFp: newFp,
+		age:   time.Duration(f.opt.MaxAge),
+	})
+}
+
 // set hashes for a path without any validation
 func (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {
 	return f.db.Do(true, &kvPut{
@@ -196,14 +213,27 @@ func (o *Object) Remove(ctx context.Context) error {
 }
 
 // SetModTime sets the modification time of the file.
-// Also prunes the cache entry when modtime changes so that
-// touching a file will trigger checksum recalculation even
-// on backends that don't provide modTime with fingerprint.
+//
+// When the fingerprint includes the modtime, the contents have not
+// changed, so the cached hashes are carried over to the fingerprint
+// for the new modtime. Otherwise the cache entry is pruned when the
+// modtime changes so that touching a file will trigger checksum
+// recalculation even on backends that don't provide modTime with
+// fingerprint.
 func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
-	if mtime != o.Object.ModTime(ctx) {
-		_ = o.f.pruneHash(o.Remote())
+	if mtime == o.Object.ModTime(ctx) {
+		return o.Object.SetModTime(ctx, mtime)
 	}
-	return o.Object.SetModTime(ctx, mtime)
+	if !o.f.fpTime {
+		_ = o.f.pruneHash(o.Remote())
+		return o.Object.SetModTime(ctx, mtime)
+	}
+	oldFp := o.fingerprint(ctx)
+	if err := o.Object.SetModTime(ctx, mtime); err != nil {
+		return err
+	}
+	_ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))
+	return nil
 }
 
 // Open opens the file for read.

backend/http/http.go

@@ -63,7 +63,8 @@ The headers are only sent to the host in the configured URL. If the
 server redirects to another host (including a subdomain or a different
 port) the headers are not sent to it, or to any further hop in that
 redirect chain. When headers are set, a redirect from https to http is
-refused as it would send them in cleartext.`,
+refused as it would send them in cleartext, even with
+--allow-https-downgrade.`,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
 		}, {
@@ -100,6 +101,30 @@ that directory listings are much quicker, but rclone won't have the times or
 sizes of any files, and some files that don't exist may be in the listing.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_head_size",
+			Help: `Set the reported file size for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unknown file size.
+Set this option to a non-negative value to report that size for every file,
+including 0. Leave it unset (the default, shown as "off") to keep reporting
+an unknown size.
+
+The configured size is authoritative. If it does not match the actual file
+size, transfers may fail their size check and retry unless --ignore-size is
+used. It can also make multi-thread transfers request incorrect ranges and
+make mounted files unreadable past the reported size.`,
+			Default:  fs.SizeSuffix(-1),
+			Advanced: true,
+		}, {
+			Name: "no_head_time",
+			Help: `Set a default modification time for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unset modification
+time. Set this option to report the specified time for every file instead.
+Changing the configured time later makes every file appear modified to a sync.`,
+			Default:  fs.Time{},
+			Advanced: true,
 		}, {
 			Name:    "no_escape",
 			Help:    "Do not escape URL metacharacters in path names.",
@@ -145,11 +170,13 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 
 // Options defines the configuration for this backend
 type Options struct {
-	Endpoint string          `config:"url"`
-	NoSlash  bool            `config:"no_slash"`
-	NoHead   bool            `config:"no_head"`
-	Headers  fs.CommaSepList `config:"headers"`
-	NoEscape bool            `config:"no_escape"`
+	Endpoint   string          `config:"url"`
+	NoSlash    bool            `config:"no_slash"`
+	NoHead     bool            `config:"no_head"`
+	NoHeadSize fs.SizeSuffix   `config:"no_head_size"`
+	NoHeadTime fs.Time         `config:"no_head_time"`
+	Headers    fs.CommaSepList `config:"headers"`
+	NoEscape   bool            `config:"no_escape"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -289,8 +316,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	}
 
 	client := fshttp.NewClient(ctx)
-	// Without configured headers keep the default policy which
-	// public mirrors that redirect from https to http rely on.
+	// Without configured headers keep the default policy so
+	// --allow-https-downgrade can follow public mirrors that
+	// redirect from https to http.
 	if len(opt.Headers) > 0 {
 		client.CheckRedirect = checkRedirect(opt)
 	}
@@ -322,7 +350,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 // the host specified in the config file.
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
-	opt := new(Options)
+	opt := &Options{
+		NoHeadSize: fs.SizeSuffix(-1),
+	}
 	err := configstruct.Set(m, opt)
 	if err != nil {
 		return nil, err
@@ -717,8 +747,11 @@ func (o *Object) url() string {
 // head sends a HEAD request to update info fields in the Object
 func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
-		o.size = -1
+		o.size = int64(o.fs.opt.NoHeadSize)
 		o.modTime = timeUnset
+		if o.fs.opt.NoHeadTime.IsSet() {
+			o.modTime = time.Time(o.fs.opt.NoHeadTime)
+		}
 		o.contentType = fs.MimeType(ctx, o)
 		return nil
 	}

backend/http/http_internal_test.go

@@ -314,6 +314,54 @@ func TestOpen(t *testing.T) {
 	}
 }
 
+func TestNoHeadMetadataDefaults(t *testing.T) {
+	tests := []struct {
+		name        string
+		size        string
+		modTime     string
+		wantSize    int64
+		wantModTime time.Time
+	}{
+		{
+			name:        "defaults",
+			wantSize:    -1,
+			wantModTime: timeUnset,
+		}, {
+			name:        "zero size",
+			size:        "0B",
+			wantSize:    0,
+			wantModTime: timeUnset,
+		}, {
+			name:        "configured metadata",
+			size:        "100B",
+			modTime:     "2024-08-05T07:36:07Z",
+			wantSize:    100,
+			wantModTime: time.Date(2024, 8, 5, 7, 36, 7, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			m := prepareServer(t)
+			m.Set("no_head", "true")
+			if test.size != "" {
+				m.Set("no_head_size", test.size)
+			}
+			if test.modTime != "" {
+				m.Set("no_head_time", test.modTime)
+			}
+
+			f, err := NewFs(context.Background(), remoteName, "", m)
+			require.NoError(t, err)
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			assert.Equal(t, test.wantSize, o.Size())
+			assert.Equal(t, test.wantModTime, o.ModTime(context.Background()))
+		})
+	}
+}
+
 func TestMimeType(t *testing.T) {
 	f := prepare(t)
 
@@ -640,7 +688,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 
 	// open makes a remote pointing at ts with or without headers
 	// and opens file.txt on it
-	open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
+	open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {
 		configfile.Install()
 		m := configmap.Simple{
 			"type":    "http",
@@ -652,6 +700,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 		}
 		ctx, ci := fs.AddConfig(context.Background())
 		ci.InsecureSkipVerify = true
+		ci.AllowHTTPSDowngrade = allowDowngrade
 		f, err := NewFs(ctx, remoteName, "", m)
 		require.NoError(t, err)
 
@@ -663,13 +712,25 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 
 	t.Run("WithHeaders", func(t *testing.T) {
 		// Headers could leak so the downgrade must be refused
-		_, err := open(t, true)
+		_, err := open(t, true, false)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {
+		// Headers could leak so the downgrade is refused even when allowed
+		_, err := open(t, true, true)
 		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
 	})
 
 	t.Run("WithoutHeaders", func(t *testing.T) {
-		// Nothing to leak so the downgrade is followed
-		fd, err := open(t, false)
+		// The transport refuses downgrades by default
+		_, err := open(t, false, false)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {
+		// Nothing to leak so the downgrade is followed when allowed
+		fd, err := open(t, false, true)
 		require.NoError(t, err)
 		data, err := io.ReadAll(fd)
 		require.NoError(t, err)

backend/huaweidrive/huaweidrive.go

@@ -50,6 +50,10 @@ const (
 	defaultChunkSize            = 8 * fs.Mebi
 )
 
+// jsonContentType is sent with JSON request bodies as the API doesn't
+// decode non-ASCII file names in them as UTF-8 unless told the charset.
+const jsonContentType = "application/json; charset=UTF-8"
+
 // OAuth2 configuration
 var oauthConfig = &oauthutil.Config{
 	Scopes: []string{
@@ -547,6 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File
@@ -1015,6 +1020,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	copyReq := api.CopyFileRequest{
@@ -1199,9 +1205,10 @@ func (f *Fs) updateFile(ctx context.Context, fileID string, parameters url.Value
 	}
 	parameters.Set("fields", "*")
 	opts := rest.Opts{
-		Method:     "PATCH",
-		Path:       "/files/" + fileID,
-		Parameters: parameters,
+		Method:       "PATCH",
+		Path:         "/files/" + fileID,
+		Parameters:   parameters,
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, req, &info)
@@ -1671,6 +1678,7 @@ func (o *Object) uploadEmpty(ctx context.Context, leaf, directoryID string, src
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File
@@ -1846,6 +1854,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 			"fields":     []string{"*"},
 		},
 		ExtraHeaders: map[string]string{
+			"Content-Type":            jsonContentType,
 			"X-Upload-Content-Length": strconv.FormatInt(size, 10),
 		},
 		RootURL: o.fs.uploadURL,
@@ -2145,6 +2154,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File

backend/iclouddrive/iclouddrive.go

@@ -694,6 +694,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
+		CaseInsensitive:         true, // names are matched with EqualFold, see findLeafItem
 		CanHaveEmptyDirectories: true,
 		PartialUploads:          false,
 	}).Fill(ctx, f)

backend/imagekit/client/upload.go

@@ -77,6 +77,8 @@ func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadPara
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading file if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	opts := rest.Opts{
 		Method:      "POST",

backend/internetarchive/internetarchive.go

@@ -634,6 +634,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (_ fs.Objec
 		"x-archive-filemeta-rclone-update-track": updateTracker,
 	}
 
+	mdata, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
+	if err != nil {
+		return nil, err
+	}
+	if mdata != nil {
+		// IA ignores file metadata headers with the COPY directive.
+		headers["x-amz-metadata-directive"] = "REPLACE"
+	}
+	for key, value := range mdata {
+		key = strings.ToLower(key)
+		if key == "mtime" {
+			key = "rclone-mtime"
+		} else if strings.HasPrefix(key, "rclone-") {
+			// Keep the new copy tracker and mapped mtime, not the source internals.
+			continue
+		} else if _, readOnly := roMetadataKey[key]; readOnly {
+			continue
+		}
+		headers["x-archive-filemeta-"+key] = value
+	}
+
 	// make a PUT request at (IAS3)/:item/:path without body
 	var resp *http.Response
 	opts := rest.Opts{

backend/jottacloud/jottacloud.go

@@ -86,8 +86,8 @@ func getServices() []service {
 		{"jottacloud", "Jottacloud", "id.jottacloud.com", "jottacloud", "desktop", []string{"openid", "jotta-default", "offline_access"}}, // Chose client id "desktop" here, will be identified as "Jottacloud for Desktop" in "My logged in devices", but could have used "jottacli" here as well.
 		{"elgiganten_dk", "Elgiganten Cloud (Denmark)", "cloud.elgiganten.dk", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
 		{"elgiganten_se", "Elgiganten Cloud (Sweden)", "cloud.elgiganten.se", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
-		{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "jotta-default", "offline_access"}},
-		{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "jotta-default", "offline_access"}},
+		{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "offline_access"}},
+		{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "offline_access"}},
 		{"gigantti", "Gigantti Cloud (Finland)", "cloud.gigantti.fi", "gigantti", "desktop", []string{"openid", "jotta-default", "offline_access"}},
 		{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
 		{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
@@ -258,6 +258,7 @@ service, for the official service on https://www.jottacloud.com/web/secure.`)
 		}
 		m.Set("configVersion", fmt.Sprint(configVersion))
 		m.Set(configClientID, service.clientID)
+		m.Set(configClientSecret, "")
 		m.Set(configTokenURL, wellKnown.TokenEndpoint)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
 			OAuth2Config: &oauthutil.Config{
@@ -1500,9 +1501,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
 	if err == nil {
-		var createTime time.Time
+		createTime := srcObj.createTime
 		var createTimeMeta bool
-		var modTime time.Time
+		modTime := srcObj.modTime
 		var modTimeMeta bool
 		if meta != nil {
 			createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
@@ -1518,6 +1519,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
 			fs.Debugf(src, "Server-side copied to trashed destination, restoring")
 			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
+			// The restored file keeps the name it was trashed with, which may differ in case
+			if err == nil && info != nil && info.Name != f.opt.Enc.FromStandardName(path.Base(remote)) {
+				fs.Debugf(src, "Renaming restored %q to %q", info.Name, path.Base(remote))
+				info, err = f.copyOrMove(ctx, "mv", f.filePath(remote), remote)
+			}
 		} else if createTimeMeta || modTimeMeta {
 			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
 		}

backend/kdrive/api/types.go

@@ -0,0 +1,235 @@
+// Package api has type definitions for kDrive
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// Time represents unix timestamp date and time information for the
+// kDrive API
+type Time time.Time
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" {
+		return nil
+	}
+
+	timestamp, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	newT := time.Unix(timestamp, 0)
+	*t = Time(newT)
+	return nil
+}
+
+// ResultStatus return error details from kdrive when things go wrong
+//
+// If Status is "success" then everything is OK
+type ResultStatus struct {
+	Status      string `json:"result"`
+	ErrorDetail struct {
+		Result      string `json:"code"`
+		ErrorString string `json:"description"`
+		Context     struct {
+			Values []string `json:"values"`
+		} `json:"context"`
+		Errors []struct {
+			Result      string `json:"code"`
+			ErrorString string `json:"description"`
+		} `json:"errors"`
+	} `json:"error"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e *ResultStatus) Error() string {
+	var details string
+	for i := range e.ErrorDetail.Errors {
+		details += "|" + e.ErrorDetail.Errors[i].Result
+	}
+
+	return fmt.Sprintf("kDrive error: %s (%s: %s %s)", e.Status, e.ErrorDetail.Result, e.ErrorDetail.ErrorString, details)
+}
+
+// IsError returns true if there is an error
+func (e *ResultStatus) IsError() bool {
+	return e.Status != "success"
+}
+
+// Update returns err directly if it was != nil, otherwise it returns
+// an Error or nil if no error was detected
+func (e *ResultStatus) Update(err error) error {
+	if err != nil {
+		return err
+	}
+	if e.IsError() {
+		return e
+	}
+	return nil
+}
+
+// Check ResultStatus satisfies the error interface
+var _ error = (*ResultStatus)(nil)
+
+// Item describes a folder or a file as returned by Get Folder Items and others
+type Item struct {
+	ID             int    `json:"id"`
+	Name           string `json:"name"`
+	Type           string `json:"type"`
+	FullPath       string `json:"path"`
+	Status         string `json:"status"`
+	Hash           string `json:"hash"`
+	Size           int64  `json:"size"`
+	Visibility     string `json:"visibility"`
+	DriveID        int    `json:"drive_id"`
+	Depth          int    `json:"depth"`
+	CreatedBy      int    `json:"created_by"`
+	CreatedAt      Time   `json:"created_at"`
+	AddedAt        int    `json:"added_at"`
+	LastModifiedAt Time   `json:"last_modified_at"`
+	LastModifiedBy int    `json:"last_modified_by"`
+	RevisedAt      int    `json:"revised_at"`
+	UpdatedAt      int    `json:"updated_at"`
+	MimeType       string `json:"mime_type"`
+	ParentID       int    `json:"parent_id"`
+	Color          string `json:"color"`
+}
+
+// ItemResult is returned when an item is queried
+type ItemResult struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// SearchResult is returned when a list of items is requested
+type SearchResult struct {
+	ResultStatus
+	Data       []Item `json:"data"`
+	Cursor     string `json:"cursor"`
+	HasMore    bool   `json:"has_more"`
+	ResponseAt int    `json:"response_at"`
+}
+
+// ModTime returns the modification time of the item
+func (i *Item) ModTime() (t time.Time) {
+	t = time.Time(i.LastModifiedAt)
+	if t.IsZero() {
+		t = time.Time(i.CreatedAt)
+	}
+	return t
+}
+
+// CancelResource is a kdrive resource that can be cancelled after some action
+type CancelResource struct {
+	CancelID   string `json:"cancel_id"`
+	ValidUntil int    `json:"valid_until"`
+}
+
+// CancellableResponse is returned from kdrive when an action can be cancelled afterwards
+type CancellableResponse struct {
+	ResultStatus
+	Data CancelResource `json:"data"`
+}
+
+// CreateDirResult is returned from kdrive after a call to MkDir
+type CreateDirResult struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// FileCopyResponse is returned from kdrive after a call to Copy
+type FileCopyResponse struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// UploadFileResponse is returned from kdrive after a call to Upload
+type UploadFileResponse struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// ChecksumFileResult is returned from kdrive after a call to Hash
+type ChecksumFileResult struct {
+	ResultStatus
+	Data struct {
+		Hash string `json:"hash"`
+	} `json:"data"`
+}
+
+// PubLinkResult is returned from kdrive after a call to public link API
+type PubLinkResult struct {
+	ResultStatus
+	Data struct {
+		URL          string `json:"url"`
+		FileID       int    `json:"file_id"`
+		Right        string `json:"right"`
+		ValidUntil   int    `json:"valid_until"`
+		CreatedBy    int    `json:"created_by"`
+		CreatedAt    int    `json:"created_at"`
+		UpdatedAt    int    `json:"updated_at"`
+		Capabilities struct {
+			CanEdit          bool `json:"can_edit"`
+			CanSeeStats      bool `json:"can_see_stats"`
+			CanSeeInfo       bool `json:"can_see_info"`
+			CanDownload      bool `json:"can_download"`
+			CanComment       bool `json:"can_comment"`
+			CanRequestAccess bool `json:"can_request_access"`
+		} `json:"capabilities"`
+		AccessBlocked bool `json:"access_blocked"`
+	} `json:"data"`
+}
+
+// PubLinkUpdateResult is returned from kdrive after a updating a public link
+type PubLinkUpdateResult struct {
+	ResultStatus
+	Data bool `json:"data"`
+}
+
+// QuotaInfo is return from kdrive after a call get drive info
+type QuotaInfo struct {
+	ResultStatus
+	Data struct {
+		Size     int64 `json:"size"`
+		UsedSize int64 `json:"used_size"`
+	} `json:"data"`
+}
+
+// SessionStartResponse is used to start a multichunk upload session
+type SessionStartResponse struct {
+	ResultStatus
+	Data struct {
+		Token     string `json:"token"`
+		UploadURL string `json:"upload_url"` // URL pour uploader les chunks
+	} `json:"data"`
+}
+
+// ChunkUploadResponse is returned by kdrive after a chunk upload
+type ChunkUploadResponse struct {
+	ResultStatus
+	Data struct {
+		Hash string `json:"hash,omitempty"` // Only present when requested with with:'hash'
+	} `json:"data"`
+}
+
+// SessionFinishResponse is used to end a multichunk upload session
+type SessionFinishResponse struct {
+	ResultStatus
+	Data struct {
+		Token   string `json:"token"`
+		File    Item   `json:"file"`
+		Result  bool   `json:"result"`
+		Message string `json:"message"`
+	} `json:"data"`
+}
+
+// SessionCancelResponse is used to cancel a multichunk upload session
+type SessionCancelResponse struct {
+	ResultStatus
+	Data bool `json:"data"`
+}

backend/kdrive/kdrive_internal_test.go

@@ -0,0 +1,551 @@
+// Unit tests for the kdrive backend
+package kdrive
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/kdrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestCheckChunkSize checks chunk_size validation against the API limits
+func TestCheckChunkSize(t *testing.T) {
+	for _, test := range []struct {
+		in      fs.SizeSuffix
+		wantErr string
+	}{
+		{in: 20 * fs.Mebi, wantErr: ""},                    // default
+		{in: 1000 * 1000, wantErr: ""},                     // API minimum
+		{in: 1000 * 1000 * 1000, wantErr: ""},              // API maximum
+		{in: 0, wantErr: "too small"},                      // panics without validation
+		{in: 999 * 1000, wantErr: "too small"},             // below API minimum
+		{in: 1000*1000*1000 + 1, wantErr: "too large"},     // above API maximum
+		{in: 2 * 1000 * 1000 * 1000, wantErr: "too large"}, // way above API maximum
+	} {
+		t.Run(test.in.String(), func(t *testing.T) {
+			err := checkChunkSize(test.in)
+			if test.wantErr == "" {
+				assert.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), test.wantErr)
+			}
+		})
+	}
+}
+
+// TestOpenChunkWriterClampsCallerChunkSize checks that a caller
+// supplied chunk size hint below the API minimum is raised to it
+func TestOpenChunkWriterClampsCallerChunkSize(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = fmt.Fprintln(w, `{"result":"success","data":{"token":"tok","upload_url":"http://unused"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0)}
+
+	info, _, err := f.OpenChunkWriter(context.Background(), "file.txt", src, &fs.ChunkOption{ChunkSize: 512 * 1024})
+	require.NoError(t, err)
+	assert.Equal(t, int64(minChunkSize), info.ChunkSize, "a chunk size hint below the API minimum must be raised to it")
+}
+
+// TestObjectSetMetaDataResetsStaleHash checks that setMetaData always
+// replaces the hash (a chunked upload finish response carries none)
+func TestObjectSetMetaDataResetsStaleHash(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		staleHash     string
+		staleNested   bool
+		newHash       string
+		wantHash      string
+		wantIsNested  bool
+		wantHashKnown bool
+	}{
+		{
+			name:          "empty hash clears stale plain hash",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "",
+			wantHash:      "",
+			wantIsNested:  false,
+			wantHashKnown: false,
+		},
+		{
+			name:          "empty hash clears stale nested hash",
+			staleHash:     "deadbeef",
+			staleNested:   true,
+			newHash:       "",
+			wantHash:      "",
+			wantIsNested:  false,
+			wantHashKnown: false,
+		},
+		{
+			name:          "plain hash is set",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "xxh3:12345678",
+			wantHash:      "12345678",
+			wantIsNested:  false,
+			wantHashKnown: true,
+		},
+		{
+			name:          "nested hash is set",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "N:xxh3:87654321",
+			wantHash:      "87654321",
+			wantIsNested:  true,
+			wantHashKnown: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o := &Object{
+				remote:      "file.txt",
+				hasMetaData: true,
+				xxh3:        test.staleHash,
+				nestedHash:  test.staleNested,
+			}
+			info := &api.Item{
+				ID:   42,
+				Name: "file.txt",
+				Type: "file",
+				Hash: test.newHash,
+			}
+
+			err := o.setMetaData(info)
+			require.NoError(t, err)
+			assert.Equal(t, test.wantHash, o.xxh3, "hash after setMetaData")
+			assert.Equal(t, test.wantIsNested, o.nestedHash, "nested flag after setMetaData")
+			assert.True(t, o.hasMetaData)
+
+			// Hash() must reflect the new state. Only checked when a
+			// hash is known: with no hash Hash() would query the API.
+			// Nested hashes are not comparable with plain XXH3, so
+			// Hash() returns "" for them.
+			if test.wantHashKnown {
+				got, gotErr := o.Hash(context.Background(), hash.XXH3)
+				require.NoError(t, gotErr)
+				if test.wantIsNested {
+					assert.Equal(t, "", got, "nested hash must not be returned as XXH3")
+				} else {
+					assert.Equal(t, test.wantHash, got)
+				}
+			}
+		})
+	}
+}
+
+// newTestFs creates an Fs wired to a test server with a plain client
+// (no OAuth) and a fast pacer
+func newTestFs(t *testing.T, endpoint string) *Fs {
+	t.Helper()
+	return &Fs{
+		name: "TestKdrive",
+		root: "",
+		opt:  Options{DriveID: "1234", Endpoint: endpoint},
+		srv:  rest.NewClient(&http.Client{}).SetRoot(endpoint),
+		pacer: fs.NewPacer(context.Background(),
+			pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond), pacer.DecayConstant(2))),
+	}
+}
+
+// TestWriteChunkDetectsResultError checks that a 200 with result:error
+// is reported as an error
+func TestWriteChunkDetectsResultError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"chunk_error","description":"chunk upload failed"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	u := &uploadSession{
+		f:           f,
+		token:       "tok",
+		uploadURL:   ts.URL,
+		fileSize:    4,
+		chunkSize:   4,
+		chunkHashes: make(map[int64]string),
+	}
+
+	_, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
+	require.Error(t, err, "result:error with HTTP 200 must not be treated as success")
+	assert.Contains(t, err.Error(), "chunk upload failed")
+}
+
+// TestWriteChunkRetriesAndRewinds checks that transient failures are
+// retried with the chunk re-sent from the start of the reader
+func TestWriteChunkRetriesAndRewinds(t *testing.T) {
+	var mu sync.Mutex
+	var attempts int
+	var bodies []string
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		attempts++
+		bodies = append(bodies, string(body))
+		current := attempts
+		mu.Unlock()
+
+		w.Header().Set("Content-Type", "application/json")
+		if current == 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
+			return
+		}
+		_, _ = fmt.Fprintln(w, `{"result":"success","data":{"chunk_number":1,"status":"OK"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	u := &uploadSession{
+		f:           f,
+		token:       "tok",
+		uploadURL:   ts.URL,
+		fileSize:    4,
+		chunkSize:   4,
+		chunkHashes: make(map[int64]string),
+	}
+
+	written, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), written)
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Len(t, bodies, 2, "chunk upload must be retried once after a 5xx")
+	assert.Equal(t, "test", bodies[0], "first attempt sends the chunk")
+	assert.Equal(t, "test", bodies[1], "retry must re-send the chunk from the start of the reader")
+}
+
+// kdriveMockItem returns a JSON body for a single item result
+func kdriveMockItem(id int, name string) string {
+	return fmt.Sprintf(`{"result":"success","data":{"id":%d,"name":%q,"type":"file","path":%q,"last_modified_at":1700000000}}`, id, name, "/"+name)
+}
+
+// mockObjectInfo is a minimal fs.ObjectInfo for testing
+type mockObjectInfo struct {
+	remote  string
+	size    int64
+	modTime time.Time
+	hash    string
+}
+
+func (m *mockObjectInfo) Fs() fs.Info                         { return nil }
+func (m *mockObjectInfo) String() string                      { return m.remote }
+func (m *mockObjectInfo) Remote() string                      { return m.remote }
+func (m *mockObjectInfo) Size() int64                         { return m.size }
+func (m *mockObjectInfo) ModTime(_ context.Context) time.Time { return m.modTime }
+func (m *mockObjectInfo) Storable() bool                      { return true }
+func (m *mockObjectInfo) MimeType(_ context.Context) string   { return "application/octet-stream" }
+
+func (m *mockObjectInfo) Hash(_ context.Context, t hash.Type) (string, error) {
+	if m.hash == "" || t != hash.XXH3 {
+		return "", hash.ErrUnsupported
+	}
+	return m.hash, nil
+}
+
+// TestUpdateDirectNoRetryOnServerError checks that a failed single
+// part upload is not retried here: the input must not be buffered, the
+// higher layers retry the whole transfer
+func TestUpdateDirectNoRetryOnServerError(t *testing.T) {
+	var attempts int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&attempts, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	o := &Object{fs: f, remote: "file.txt"}
+	src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: "12345678"}
+
+	err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
+	require.Error(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&attempts), "single part uploads must not be retried here")
+}
+
+// TestUpdateDirect checks the upload parameters and metadata of a
+// successful single part upload, and that total_chunk_hash is only
+// sent when the source hash is known
+func TestUpdateDirect(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		srcHash       string
+		wantTotalHash string
+	}{
+		{name: "hash known", srcHash: "12345678", wantTotalHash: "xxh3:12345678"},
+		{name: "hash unknown", srcHash: "", wantTotalHash: ""},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var gotTotalHash, gotBody string
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				gotTotalHash = r.URL.Query().Get("total_chunk_hash")
+				body, _ := io.ReadAll(r.Body)
+				gotBody = string(body)
+				_, _ = fmt.Fprintln(w, `{"result":"success","data":{"id":55,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:99998888","last_modified_at":1700000000}}`)
+			}))
+			defer ts.Close()
+
+			f := newTestFs(t, ts.URL)
+			o := &Object{fs: f, remote: "file.txt"}
+			src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: test.srcHash}
+
+			err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
+			require.NoError(t, err)
+			assert.Equal(t, "test", gotBody, "the content must be streamed as-is")
+			assert.Equal(t, test.wantTotalHash, gotTotalHash, "total_chunk_hash")
+			assert.Equal(t, "55", o.id)
+			gotHash, hashErr := o.Hash(context.Background(), hash.XXH3)
+			require.NoError(t, hashErr)
+			assert.Equal(t, "99998888", gotHash)
+		})
+	}
+}
+
+// TestNewFsRequiresDriveID checks that NewFs fails fast when drive_id
+// is missing
+func TestNewFsRequiresDriveID(t *testing.T) {
+	m := configmap.Simple{"chunk_size": "20Mi"}
+	_, err := NewFs(context.Background(), "kdrive", "", m)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "drive_id")
+}
+
+// TestUploadSessionStringMasksToken checks that the upload session
+// String does not leak the session token in debug logs
+func TestUploadSessionStringMasksToken(t *testing.T) {
+	u := &uploadSession{token: "very-secret-uuid"}
+	assert.NotContains(t, u.String(), "very-secret-uuid")
+}
+
+// TestListNonRecursiveSingleRoundTrip checks that a non recursive
+// listing needs a single API request: without depth=unlimited the API
+// returns direct children only, so neither the parent item lookup nor
+// a child filter is needed
+func TestListNonRecursiveSingleRoundTrip(t *testing.T) {
+	var requests int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&requests, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/files"):
+			// Direct children only, as returned without depth=unlimited
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":[`+
+				`{"id":10,"name":"sub","type":"dir","path":"/sub"},`+
+				`{"id":11,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:12345678","last_modified_at":1700000000}],"has_more":false}`)
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+
+	entries, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&requests), "non recursive listing must make a single API request")
+
+	names := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		names = append(names, entry.Remote())
+	}
+	assert.ElementsMatch(t, []string{"sub", "file.txt"}, names)
+}
+
+// TestMoveConflictReturnsError checks that moving onto an occupied
+// destination reports the API conflict instead of resolving it: rclone
+// removes the destination itself before calling Move
+func TestMoveConflictReturnsError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"conflict_error","description":"destination already exists"}}`)
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	_, err := f.Move(context.Background(), srcObj, "file2")
+	require.Error(t, err, "a move onto an occupied destination must fail")
+	assert.Contains(t, err.Error(), "conflict", "the API conflict must be reported")
+}
+
+// TestMoveDetectsForeignObjectAtDestination checks that Move does not
+// return a file re-created by another writer at the destination
+func TestMoveDetectsForeignObjectAtDestination(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			// The move API reports success...
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
+			// ...but the file at the destination is not the moved
+			// file: another writer re-created it
+			_, _ = fmt.Fprintln(w, kdriveMockItem(999, "file2"))
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	_, err := f.Move(context.Background(), srcObj, "file2")
+	require.Error(t, err, "Move must not report success when the destination is not the moved file")
+}
+
+// TestMoveSuccess checks the nominal move: the object returned is the
+// moved file (same ID as the source).
+func TestMoveSuccess(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
+			// After the move, the moved file is at the destination
+			_, _ = fmt.Fprintln(w, kdriveMockItem(111, "file2"))
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	dstObj, err := f.Move(context.Background(), srcObj, "file2")
+	require.NoError(t, err)
+	moved, ok := dstObj.(*Object)
+	require.True(t, ok)
+	assert.Equal(t, "111", moved.id, "the returned object must be the moved file")
+}
+
+// TestFormatModTime checks that pre-1970 times format as negative
+// unix timestamps instead of wrapping through a uint64 cast
+func TestFormatModTime(t *testing.T) {
+	for _, test := range []struct {
+		in   time.Time
+		want string
+	}{
+		{in: time.Unix(1700000000, 0), want: "1700000000"},
+		{in: time.Unix(0, 0), want: "0"},
+		{in: time.Unix(-1577923200, 0), want: "-1577923200"}, // Sat Jan 2 1965
+	} {
+		t.Run(test.want, func(t *testing.T) {
+			assert.Equal(t, test.want, formatModTime(test.in))
+		})
+	}
+}
+
+// TestObjectSetMetaDataIsDir checks that directories are rejected
+func TestObjectSetMetaDataIsDir(t *testing.T) {
+	o := &Object{remote: "dir"}
+	info := &api.Item{ID: 42, Name: "dir", Type: "dir"}
+	err := o.setMetaData(info)
+	assert.Equal(t, fs.ErrorIsDir, err)
+}
+
+// TestShouldRetryLockError checks that transient locks held by server
+// side copies are retried while locks held by applications and other
+// errors are not
+func TestShouldRetryLockError(t *testing.T) {
+	for _, test := range []struct {
+		name      string
+		body      string
+		status    int
+		wantRetry bool
+	}{
+		{
+			name:      "lock held by internal copy",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy","context":{"values":["internal_copy"]}}}`,
+			status:    http.StatusForbidden,
+			wantRetry: true,
+		},
+		{
+			name:      "lock held by an application",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by onlyoffice","context":{"values":["onlyoffice"]}}}`,
+			status:    http.StatusForbidden,
+			wantRetry: false,
+		},
+		{
+			name:      "lock without context",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy"}}`,
+			status:    http.StatusForbidden,
+			wantRetry: false,
+		},
+		{
+			name:      "other api error",
+			body:      `{"result":"error","error":{"code":"object_not_found","description":"File not found"}}`,
+			status:    http.StatusNotFound,
+			wantRetry: false,
+		},
+		{
+			name:      "server error",
+			status:    http.StatusServiceUnavailable,
+			wantRetry: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var apiErr error
+			if test.body != "" {
+				e := new(api.ResultStatus)
+				require.NoError(t, json.Unmarshal([]byte(test.body), e))
+				apiErr = e
+			}
+			resp := &http.Response{StatusCode: test.status, Header: http.Header{}}
+			got, gotErr := shouldRetry(context.Background(), resp, apiErr)
+			assert.Equal(t, test.wantRetry, got)
+			assert.Equal(t, apiErr, gotErr)
+		})
+	}
+}

backend/kdrive/kdrive_test.go

@@ -0,0 +1,25 @@
+// Test kdrive filesystem interface
+package kdrive
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestKdrive:",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+			MaxChunkSize: maxChunkSize,
+		},
+	})
+}
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setChunkSize(cs)
+}

backend/kdrive/khash/khash.go

@@ -0,0 +1,67 @@
+// Package khash provides the kDrive specific hash handling.
+//
+// kDrive uses XXH3-64 for file hashing. Files uploaded in chunks (by
+// rclone, or by the kDrive apps which each use their own chunk size)
+// get a "nested" hash: the XXH3 of the concatenated lowercase hex
+// strings of the individual chunk hashes. Such a nested hash can't be
+// compared with a plain XXH3 of the file contents.
+//
+// Hash formats returned by the kDrive API:
+//   - "xxh3:HASH" for a plain XXH3 of the file contents
+//   - "N:xxh3:HASH" for the nested hash of a chunked upload
+package khash
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/zeebo/xxh3"
+)
+
+// ParseHash parses a kDrive hash string from the API and returns the hex hash value
+// along with whether it is a nested (chunked) hash.
+func ParseHash(hashStr string) (hexHash string, isNested bool, err error) {
+	if hashStr == "" {
+		return "", false, nil
+	}
+
+	// Check for nested hash format: "N:xxh3:HASH"
+	if strings.HasPrefix(hashStr, "N:xxh3:") {
+		hashStr = strings.TrimPrefix(hashStr, "N:xxh3:")
+		isNested = true
+	} else {
+		hashStr = strings.TrimPrefix(hashStr, "xxh3:")
+	}
+
+	// Validate hex string
+	if _, decodeErr := hex.DecodeString(hashStr); decodeErr != nil {
+		return "", false, fmt.Errorf("invalid hash format: %w", decodeErr)
+	}
+
+	return hashStr, isNested, nil
+}
+
+// NestedChunkHash computes the kDrive nested hash from the hashes of the
+// file chunks, in order.
+//
+// kDrive stores the nested hash of a chunked file as the XXH3 of the
+// concatenated lowercase hex strings of the individual chunk hashes.
+// Each chunk hash may be passed with or without the "xxh3:" prefix.
+func NestedChunkHash(chunkHashes []string) (string, error) {
+	if len(chunkHashes) == 0 {
+		return "", errors.New("no chunk hashes to nest")
+	}
+	hexHashes := make([]string, len(chunkHashes))
+	for i, chunkHash := range chunkHashes {
+		hexHash, _, err := ParseHash(chunkHash)
+		if err != nil {
+			return "", fmt.Errorf("invalid chunk hash %q: %w", chunkHash, err)
+		}
+		hexHashes[i] = hexHash
+	}
+	hasher := xxh3.New()
+	_, _ = hasher.Write([]byte(strings.Join(hexHashes, "")))
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}

backend/kdrive/khash/khash_test.go

@@ -0,0 +1,150 @@
+package khash
+
+import "testing"
+
+func TestParseHash(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        string
+		expectedHash string
+		isNested     bool
+		expectError  bool
+	}{
+		{
+			name:         "simple hash with prefix",
+			input:        "xxh3:1db797b96febd334",
+			expectedHash: "1db797b96febd334",
+			isNested:     false,
+		},
+		{
+			name:         "nested hash",
+			input:        "N:xxh3:877abf3579f0a5c0",
+			expectedHash: "877abf3579f0a5c0",
+			isNested:     true,
+		},
+		{
+			name:         "plain hex",
+			input:        "1db797b96febd334",
+			expectedHash: "1db797b96febd334",
+			isNested:     false,
+		},
+		{
+			name:         "empty string",
+			input:        "",
+			expectedHash: "",
+			isNested:     false,
+		},
+		{
+			name:        "invalid hex",
+			input:       "xxh3:notahexvalue",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hash, isNested, err := ParseHash(tt.input)
+
+			if tt.expectError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if hash != tt.expectedHash {
+				t.Errorf("Hash: got %q, expected %q", hash, tt.expectedHash)
+			}
+			if isNested != tt.isNested {
+				t.Errorf("IsNested: got %t, expected %t", isNested, tt.isNested)
+			}
+		})
+	}
+}
+
+func TestNestedChunkHash(t *testing.T) {
+	// Golden values: the nested hash is the XXH3 of the concatenated
+	// lowercase hex strings of the chunk hashes, in order
+	chunk1 := "xxh3:d447b1ea40e6988b" // XXH3 of "hello world"
+	chunk2 := "2d06800538d394c2"      // XXH3 of ""
+
+	tests := []struct {
+		name        string
+		chunkHashes []string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:        "two chunks in order",
+			chunkHashes: []string{chunk1, chunk2},
+			expected:    "6d51edccc9c05a8b",
+		},
+		{
+			name:        "order matters",
+			chunkHashes: []string{chunk2, chunk1},
+			expected:    "596c127c7bcc8820",
+		},
+		{
+			name:        "single chunk",
+			chunkHashes: []string{chunk1},
+			expected:    "b73081dff8fe443c",
+		},
+		{
+			name:        "mixed prefixes",
+			chunkHashes: []string{chunk1, "xxh3:" + chunk2, chunk1},
+			expected:    "ca0b13abcdcd1661",
+		},
+		{
+			name:        "no chunks",
+			chunkHashes: []string{},
+			expectError: true,
+		},
+		{
+			name:        "invalid chunk hash",
+			chunkHashes: []string{chunk1, "notahexvalue"},
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nested, err := NestedChunkHash(tt.chunkHashes)
+
+			if tt.expectError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if nested != tt.expected {
+				t.Errorf("NestedChunkHash: got %q, expected %q", nested, tt.expected)
+			}
+		})
+	}
+}
+
+func TestNestedChunkHashAcceptsPrefixedHashes(t *testing.T) {
+	plain := []string{"d447b1ea40e6988b", "2d06800538d394c2"}
+	prefixed := []string{"xxh3:d447b1ea40e6988b", "N:xxh3:2d06800538d394c2"}
+
+	gotPlain, err := NestedChunkHash(plain)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	gotPrefixed, err := NestedChunkHash(prefixed)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if gotPlain != gotPrefixed {
+		t.Errorf("Prefixed hashes should give the same nested hash: %q != %q", gotPlain, gotPrefixed)
+	}
+}

backend/kdrive/multipart.go

@@ -0,0 +1,383 @@
+package kdrive
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/backend/kdrive/api"
+	"github.com/rclone/rclone/backend/kdrive/khash"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/chunksize"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/zeebo/xxh3"
+	"golang.org/x/text/unicode/norm"
+)
+
+const (
+	// maxUploadParts is the maximum number of chunks the kDrive API accepts per file
+	maxUploadParts = 10000
+
+	// maxChunkSize is the largest chunk size the kDrive API accepts
+	maxChunkSize = 1 * 1000 * 1000 * 1000
+
+	// minChunkSize is the smallest chunk size the kDrive API accepts
+	minChunkSize = 1 * 1000 * 1000
+)
+
+// checkChunkSize checks the user supplied chunk_size against the
+// limits the kDrive API accepts
+func checkChunkSize(chunkSize fs.SizeSuffix) error {
+	if chunkSize < minChunkSize {
+		return fmt.Errorf("chunk_size %s is too small: the kDrive API requires chunks of at least 1 MB", chunkSize)
+	}
+	if chunkSize > maxChunkSize {
+		return fmt.Errorf("chunk_size %s is too large: the kDrive API accepts chunks of at most 1 GB (1000000000 bytes)", chunkSize)
+	}
+	return nil
+}
+
+// setChunkSize sets the chunk size, returning the old value
+func (f *Fs) setChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkChunkSize(cs)
+	if err == nil {
+		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	}
+	return
+}
+
+// formatModTime formats the modification time of an upload as the
+// unix timestamp string the kDrive API expects
+func formatModTime(modTime time.Time) string {
+	return fmt.Sprintf("%d", modTime.Unix())
+}
+
+// uploadSession implements fs.ChunkWriter for kdrive multipart uploads
+type uploadSession struct {
+	f         *Fs
+	token     string
+	uploadURL string
+	fileInfo  *api.Item
+	fileSize  int64 // total size of the file being uploaded
+	chunkSize int64 // size of the chunks of this session
+
+	mu          sync.Mutex
+	chunkHashes map[int64]string // chunk number -> hash of the chunk contents
+}
+
+// OpenChunkWriter returns the chunk size and a ChunkWriter
+//
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+	fileSize := src.Size()
+	if fileSize < 0 {
+		return info, nil, errors.New("kdrive can't upload files with unknown size")
+	}
+
+	dir, leaf := path.Split(remote)
+	dir = strings.TrimSuffix(dir, "/")
+
+	// Create parent directories if they don't exist
+	parentID, err := f.dirCache.FindDir(ctx, dir, true)
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to find parent directory: %w", err)
+	}
+
+	// Calculate the chunk size, honouring any fs.ChunkOption hint
+	chunkSize := f.opt.ChunkSize
+	for _, opt := range options {
+		if chunkOpt, ok := opt.(*fs.ChunkOption); ok && chunkOpt.ChunkSize > 0 {
+			chunkSize = fs.SizeSuffix(chunkOpt.ChunkSize)
+		}
+	}
+	chunkSize = chunksize.Calculator(src, fileSize, maxUploadParts, chunkSize)
+	if chunkSize < minChunkSize {
+		chunkSize = minChunkSize
+	}
+	if chunkSize > maxChunkSize {
+		chunkSize = maxChunkSize
+	}
+
+	// Calculate the total number of chunks
+	totalChunks := max((fileSize+int64(chunkSize)-1)/int64(chunkSize), 1)
+
+	sessionReq := struct {
+		Conflict       string `json:"conflict"`
+		DirectoryID    string `json:"directory_id"`
+		FileName       string `json:"file_name"`
+		LastModifiedAt string `json:"last_modified_at"`
+		TotalChunks    int64  `json:"total_chunks"`
+		TotalSize      int64  `json:"total_size"`
+	}{
+		Conflict:       "version",
+		DirectoryID:    parentID,
+		FileName:       f.opt.Enc.FromStandardName(norm.NFC.String(leaf)),
+		LastModifiedAt: formatModTime(src.ModTime(ctx)),
+		TotalChunks:    totalChunks,
+		TotalSize:      fileSize,
+	}
+
+	// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   fmt.Sprintf("/3/drive/%s/upload/session/start", f.opt.DriveID),
+	}
+	var sessionResp api.SessionStartResponse
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = f.srv.CallJSON(ctx, &opts, &sessionReq, &sessionResp)
+		callErr = sessionResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to start upload session: %w", err)
+	}
+
+	info = fs.ChunkWriterInfo{
+		ChunkSize:   int64(chunkSize),
+		Concurrency: f.opt.UploadConcurrency,
+	}
+
+	writer = &uploadSession{
+		f:           f,
+		token:       sessionResp.Data.Token,
+		uploadURL:   sessionResp.Data.UploadURL,
+		fileSize:    fileSize,
+		chunkSize:   int64(chunkSize),
+		chunkHashes: make(map[int64]string),
+	}
+
+	fs.Debugf(f, "open chunk writer: started upload session for %d chunks of %v", totalChunks, chunkSize)
+	return info, writer, nil
+}
+
+// WriteChunk uploads a single chunk
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
+func (u *uploadSession) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
+	if chunkNumber < 0 {
+		return -1, fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
+	}
+
+	// Calculate the size of this chunk from the session parameters so
+	// the chunk can be streamed to the API in a single pass
+	chunkLen := u.chunkSize
+	if remaining := u.fileSize - int64(chunkNumber)*u.chunkSize; remaining < chunkLen {
+		chunkLen = remaining
+	}
+	if chunkLen <= 0 {
+		return 0, nil
+	}
+
+	sourceChunkNumber := chunkNumber + 1 // kDrive API uses 1-based numbering
+
+	// The chunk_hash parameter is optional, so the chunk is streamed
+	// in a single pass, hashing the contents on the way with a
+	// TeeReader. The server returns the hash of the chunk it received,
+	// which is checked below. Retries seek back to the chunk start
+	// and hash with a fresh hasher.
+	var chunkHash string
+	err = u.f.pacer.Call(func() (bool, error) {
+		if _, seekErr := reader.Seek(0, io.SeekStart); seekErr != nil {
+			return false, fmt.Errorf("failed to rewind chunk %d: %w", sourceChunkNumber, seekErr)
+		}
+		chunkHasher := xxh3.New()
+
+		// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
+		chunkOpts := rest.Opts{
+			Method:  "POST",
+			RootURL: u.uploadURL,
+			Path:    fmt.Sprintf("/3/drive/%s/upload/session/%s/chunk", u.f.opt.DriveID, u.token),
+			Parameters: url.Values{
+				"chunk_number": {fmt.Sprintf("%d", sourceChunkNumber)},
+				"chunk_size":   {fmt.Sprintf("%d", chunkLen)},
+				"with":         {"hash"},
+			},
+			Body:          io.TeeReader(reader, chunkHasher),
+			ContentLength: &chunkLen,
+		}
+
+		var chunkResp api.ChunkUploadResponse
+		resp, callErr := u.f.srv.CallJSON(ctx, &chunkOpts, nil, &chunkResp)
+		callErr = chunkResp.ResultStatus.Update(callErr)
+		if callErr != nil {
+			return shouldRetry(ctx, resp, callErr)
+		}
+
+		chunkHash = hex.EncodeToString(chunkHasher.Sum(nil))
+
+		// The server hashed the chunk it received, so compare with what we
+		// sent to detect any corruption immediately
+		if chunkResp.Data.Hash != "" {
+			if serverHash, _, parseErr := khash.ParseHash(chunkResp.Data.Hash); parseErr == nil {
+				if !strings.EqualFold(chunkHash, serverHash) {
+					return false, fmt.Errorf("chunk %d hash mismatch: client=%s, server=%s", sourceChunkNumber, chunkHash, serverHash)
+				}
+			} else {
+				fs.Debugf(u, "Failed to parse server chunk hash %q: %v", chunkResp.Data.Hash, parseErr)
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return -1, err
+	}
+
+	u.mu.Lock()
+	u.chunkHashes[int64(chunkNumber)] = chunkHash
+	u.mu.Unlock()
+
+	fs.Debugf(u, "uploaded chunk %d (size: %d, hash: %s)", sourceChunkNumber, chunkLen, chunkHash)
+	return chunkLen, nil
+}
+
+// Close finalizes the upload session, verifies the hash of the uploaded
+// file and returns the created file info
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
+func (u *uploadSession) Close(ctx context.Context) error {
+	// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   fmt.Sprintf("/3/drive/%s/upload/session/%s/finish", u.f.opt.DriveID, u.token),
+	}
+	var resp *http.Response
+	var finishResp api.SessionFinishResponse
+	err := u.f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &finishResp)
+		callErr = finishResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to finish upload session: %w", err)
+	}
+
+	u.fileInfo = &finishResp.Data.File
+	fs.Debugf(u, "multipart upload completed: file id %d", finishResp.Data.File.ID)
+
+	return u.verifyHash(ctx)
+}
+
+// verifyHash checks the hash of the uploaded file against the hashes of
+// the chunks we uploaded, deleting the file if it is corrupted
+func (u *uploadSession) verifyHash(ctx context.Context) (err error) {
+	remoteHash := u.fileInfo.Hash
+	if remoteHash == "" {
+		// Hash might be missing in the finish response, try to fetch it
+		obj := &Object{
+			fs: u.f,
+			id: strconv.Itoa(u.fileInfo.ID),
+		}
+		var hashErr error
+		if remoteHash, hashErr = obj.retrieveHash(ctx); hashErr != nil {
+			// skip verification
+			fs.Debugf(u, "Failed to retrieve hash for verification: %v", hashErr)
+			return nil
+		}
+	}
+	parsedRemoteHash, isNested, err := khash.ParseHash(remoteHash)
+	if err != nil {
+		fs.Debugf(u, "Failed to parse server hash %q: %v", remoteHash, err)
+		return nil
+	}
+
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	// Rebuild the hash the server should have stored from the chunk
+	// hashes we uploaded, in order
+	chunkCount := int64(len(u.chunkHashes))
+	if chunkCount == 0 {
+		return nil
+	}
+	chunkHashes := make([]string, 0, chunkCount)
+	for i := int64(0); i < chunkCount; i++ {
+		chunkHash, ok := u.chunkHashes[i]
+		if !ok {
+			fs.Debugf(u, "Chunk %d hash missing, skipping hash verification", i)
+			return nil
+		}
+		chunkHashes = append(chunkHashes, chunkHash)
+	}
+
+	var localHash string
+	if isNested {
+		if localHash, err = khash.NestedChunkHash(chunkHashes); err != nil {
+			fs.Debugf(u, "Failed to compute nested hash: %v", err)
+			return nil
+		}
+	} else if chunkCount == 1 {
+		localHash, _, _ = khash.ParseHash(chunkHashes[0])
+	} else {
+		// The server returned a simple hash for a multi chunk upload,
+		// nothing to compare it against
+		fs.Debugf(u, "Server returned a simple hash for a %d chunk upload, skipping hash verification", chunkCount)
+		return nil
+	}
+
+	if !strings.EqualFold(localHash, parsedRemoteHash) {
+		err = fmt.Errorf(
+			"multipart upload hash mismatch: local=%s, remote=%s",
+			localHash, parsedRemoteHash,
+		)
+
+		// Remove the corrupted file
+		obj := &Object{
+			fs: u.f,
+			id: strconv.Itoa(u.fileInfo.ID),
+		}
+		if delErr := obj.Remove(ctx); delErr != nil {
+			fs.Errorf(nil, "Failed to remove corrupted file after hash mismatch: %v", delErr)
+		} else {
+			fs.Debugf(nil, "Removed corrupted file after hash mismatch")
+		}
+
+		return err
+	}
+	fs.Debugf(u, "Multipart upload hash verified: %s", localHash)
+
+	return nil
+}
+
+// Abort the upload session
+// @see  https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
+func (u *uploadSession) Abort(ctx context.Context) error {
+	// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
+	opts := rest.Opts{
+		Method: "DELETE",
+		Path:   fmt.Sprintf("/2/drive/%s/upload/session/%s", u.f.opt.DriveID, u.token),
+	}
+	var resp *http.Response
+	var cancelResp api.SessionCancelResponse
+	err := u.f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &cancelResp)
+		callErr = cancelResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to cancel upload session: %w", err)
+	}
+
+	fs.Debugf(u, "upload session cancelled")
+	return nil
+}
+
+// String implements fmt.Stringer
+func (u *uploadSession) String() string {
+	return "kdrive upload session"
+}
+
+// Verify interface compliance
+var _ fs.ChunkWriter = (*uploadSession)(nil)

backend/local/clone_darwin.go

@@ -6,11 +6,13 @@ package local
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"runtime"
 
 	"github.com/go-darwin/apfs"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/random"
 )
 
 // Copy src to this remote using server-side copy operations.
@@ -63,7 +65,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = Clone(srcPath, dstPath)
+	err = cloneOver(srcPath, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -79,6 +81,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
+// cloneOver clones src to dst, replacing dst if it already exists.
+//
+// COPYFILE_CLONE implies COPYFILE_EXCL so Clone fails if dst exists.
+// Cloning to a temporary name in the same directory and renaming it
+// over dst avoids this and leaves dst untouched if the clone fails.
+func cloneOver(src, dst string) error {
+	if _, err := os.Lstat(dst); os.IsNotExist(err) {
+		return Clone(src, dst)
+	}
+	tmp := filepath.Join(filepath.Dir(dst), "."+random.String(16)+".rclone-clone")
+	err := Clone(src, tmp)
+	if err == nil {
+		err = os.Rename(tmp, dst)
+	}
+	if err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("failed to clone %q to %q: %w", src, dst, err)
+	}
+	return nil
+}
+
 // Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)
 // note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical
 // https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html

backend/local/local.go

@@ -87,8 +87,10 @@ Metadata is supported on files and directories.
 When restoring metadata with ` + "`--metadata`" + ` rclone applies the
 "mode", "uid" and "gid" from the source. These come from the source
 remote which may not be trusted, so restoring metadata as root from an
-untrusted source can change file ownership and is not recommended. The
-setuid, setgid and sticky bits are not restored by default - see the
+untrusted source can change file ownership and is not recommended. A
+user other than root keeps its own uid and sets the gid only to a group
+it is a member of, as rsync does. The setuid, setgid and sticky bits are
+not restored by default - see the
 ` + "`--local-metadata-restore-special-bits`" + ` flag.
 `,
 		},
@@ -252,6 +254,13 @@ deduplicated by default (i.e. having two identical files does not consume more
 storage than having just one.)  However, for use cases where data redundancy is
 preferable, --local-no-clone can be used to disable cloning and force "deep" copies.
 
+Cloning also preserves macOS-specific metadata such as extended attributes,
+which a normal copy does not, so this metadata is lost when cloning is disabled.
+
+If the destination file already exists, cloning replaces it with a new file,
+even with --inplace, so hard links to it are broken. Use --local-no-clone to
+update existing files in place.
+
 Currently, cloning is only supported when using APFS on macOS (support for other
 platforms may be added in the future.)`,
 				Default:  false,
@@ -951,13 +960,13 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
 	if err != nil {
 		return nil, err
 	}
-	fi, err := f.lstat(localPath)
+	fi, err := f.lstatRoot(localPath)
 	if errors.Is(err, os.ErrNotExist) {
 		err := f.Mkdir(ctx, dir)
 		if err != nil {
 			return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
 		}
-		fi, err = f.lstat(localPath)
+		fi, err = f.lstatRoot(localPath)
 		if err != nil {
 			return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
 		}
@@ -989,20 +998,34 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
 // Rmdir removes the directory
 //
 // If it isn't empty it will return an error
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	localPath, err := f.localPath(dir)
 	if err != nil {
 		return err
 	}
-	if fi, err := os.Stat(localPath); err != nil {
+	// When translating symlinks go through os.Root so a symlink planted at
+	// any parent can't redirect the removal out of the destination, and
+	// don't follow the leaf as a symlink is a file in this mode. The root
+	// itself is the trusted destination and is removed directly.
+	stat, removeFn, chmod := os.Stat, os.Remove, os.Chmod
+	if f.opt.TranslateSymlinks && localPath != f.root {
+		var root *os.Root
+		root, localPath, err = f.osRoot(localPath)
+		if err != nil {
+			return err
+		}
+		defer fs.CheckClose(root, &err)
+		stat, removeFn, chmod = root.Lstat, root.Remove, root.Chmod
+	}
+	if fi, err := stat(localPath); err != nil {
 		return err
 	} else if !fi.IsDir() {
 		return fs.ErrorIsFile
 	}
-	err = os.Remove(localPath)
+	err = removeFn(localPath)
 	if runtime.GOOS == "windows" && errors.Is(err, iofs.ErrPermission) { // https://github.com/golang/go/issues/26295
-		if os.Chmod(localPath, 0o600) == nil {
-			err = os.Remove(localPath)
+		if chmod(localPath, 0o600) == nil {
+			err = removeFn(localPath)
 		}
 	}
 	return err
@@ -1179,9 +1202,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	// Create parent of destination
+	// Create parent of destination. The root itself is the trusted
+	// destination so its parent, which lies outside it, is created directly.
 	dstParentPath := filepath.Dir(dstPath)
-	err = file.MkdirAll(dstParentPath, 0777)
+	if dstPath == f.root {
+		err = file.MkdirAll(dstParentPath, 0777)
+	} else {
+		err = f.mkdirAll(dstParentPath)
+	}
 	if err != nil {
 		return err
 	}
@@ -1598,10 +1626,26 @@ func (f *Fs) mkdirAll(localPath string) (err error) {
 	return root.MkdirAll(rel, 0777)
 }
 
-// openFile opens localPath for writing. When translating symlinks it goes
-// through os.Root so a symlink planted at the path, or at any parent, is never
-// followed out of the destination. The returned file is independent of the
-// root, which is closed before returning.
+// lstatRoot returns the info for localPath without following a symlink at
+// the leaf. When translating symlinks it goes through os.Root so a symlink
+// planted at any parent can't make a path outside the destination look like
+// it exists, which would let its metadata be written there.
+func (f *Fs) lstatRoot(localPath string) (fi os.FileInfo, err error) {
+	if !f.opt.TranslateSymlinks {
+		return f.lstat(localPath)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return nil, err
+	}
+	defer fs.CheckClose(root, &err)
+	return root.Lstat(rel)
+}
+
+// openFile opens localPath with the given flags and perm. When translating
+// symlinks it goes through os.Root so a symlink planted at the path, or at any
+// parent, is never followed out of the destination. The returned file is
+// independent of the root, which is closed before returning.
 func (f *Fs) openFile(localPath string, flags int, perm os.FileMode) (fi *os.File, err error) {
 	if !f.opt.TranslateSymlinks {
 		return file.OpenFile(localPath, flags, perm)
@@ -1670,6 +1714,21 @@ func (f *Fs) chtimes(localPath string, atime, mtime time.Time) (err error) {
 	return root.Chtimes(rel, atime, mtime)
 }
 
+// remove removes the file, symlink or empty directory at localPath. When
+// translating symlinks it goes through os.Root so a symlink planted at any
+// parent can't redirect the removal out of the destination.
+func (f *Fs) remove(localPath string) (err error) {
+	if !f.opt.TranslateSymlinks {
+		return remove(localPath, os.Remove)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(root, &err)
+	return remove(rel, root.Remove)
+}
+
 // mkdirAll makes all the directories needed to store the object
 func (o *Object) mkdirAll() error {
 	return o.fs.mkdirAll(filepath.Dir(o.path))
@@ -1794,7 +1853,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
-		if removeErr := os.Remove(o.path); removeErr != nil {
+		if removeErr := o.fs.remove(o.path); removeErr != nil {
 			fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
 		}
 		return err
@@ -1942,7 +2001,7 @@ func (o *Object) lstat() error {
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	o.clearHashCache()
-	return remove(o.path)
+	return o.fs.remove(o.path)
 }
 
 // Metadata returns metadata for an object

backend/local/local_internal_test.go

@@ -99,6 +99,31 @@ func TestVerifyCopy(t *testing.T) {
 	assert.Error(t, err)
 }
 
+// Test server-side copy over an existing file
+func TestCopyOverExisting(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	doCopy := r.Flocal.Features().Copy
+	if doCopy == nil {
+		t.Skip("local backend has no server-side copy on this platform")
+	}
+	t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	t2 := fstest.Time("2002-02-03T04:05:06.499999999Z")
+	file1 := r.WriteFile("a.txt", "hello1", t1)
+	file2 := r.WriteFile("b.txt", "HELLO2 longer", t2)
+
+	src, err := r.Flocal.NewObject(ctx, file1.Path)
+	require.NoError(t, err)
+	dst, err := doCopy(ctx, src, file2.Path)
+	require.NoError(t, err)
+	assert.Equal(t, file2.Path, dst.Remote())
+
+	file2.Size = file1.Size
+	file2.Hashes = file1.Hashes
+	file2.ModTime = file1.ModTime
+	r.CheckLocalItems(t, file1, file2)
+}
+
 // skipIfNoSymlinks skips the test if this process can't create symlinks.
 //
 // Windows grants the privilege only to an elevated process or one running with
@@ -535,6 +560,115 @@ func TestDirBTimeThroughPlantedSymlinkBlocked(t *testing.T) {
 	require.Equal(t, before, readBTime(), "btime escaped through planted symlink to %q", evilDir)
 }
 
+// TestMkdirMetadataThroughPlantedSymlinkBlocked checks MkdirMetadata can't
+// be fooled into treating a directory outside the root, reached through a
+// planted symlink, as the directory to apply metadata to.
+func TestMkdirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	require.NoError(t, os.Mkdir(filepath.Join(evilDir, "sub"), 0700))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: an existing in-tree directory is found as normal.
+	require.NoError(t, f.Mkdir(ctx, "sub"))
+	_, err := f.MkdirMetadata(ctx, "sub", nil)
+	require.NoError(t, err)
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// "pwn/sub" exists outside the root but must not be found through the
+	// planted symlink, and must not be created there either.
+	_, err = f.MkdirMetadata(ctx, "pwn/sub", nil)
+	require.Error(t, err, "finding a directory through a planted symlink should be refused")
+	_, err = f.MkdirMetadata(ctx, "pwn/new", nil)
+	require.Error(t, err, "making a directory through a planted symlink should be refused")
+	_, err = os.Lstat(filepath.Join(evilDir, "new"))
+	require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evilDir)
+}
+
+// TestRemoveThroughPlantedSymlinkBlocked checks Remove and Rmdir can't
+// delete anything outside the root through a planted symlink.
+func TestRemoveThroughPlantedSymlinkBlocked(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		// os.Root.Symlink makes a file symlink to a directory outside
+		// the root, which Windows can't traverse, so the planted link
+		// can't reach the outside files.
+		t.Skip("planted directory symlinks can't be traversed on Windows")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilFile := filepath.Join(evil, "secret.txt")
+	require.NoError(t, os.WriteFile(evilFile, []byte("keep me"), 0600))
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: in-tree removals work as normal.
+	require.NoError(t, putFile(ctx, f, "file.txt", "data"))
+	o, err := f.NewObject(ctx, "file.txt")
+	require.NoError(t, err)
+	require.NoError(t, o.Remove(ctx))
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	require.NoError(t, f.Rmdir(ctx, "dir"))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evil))
+
+	// Removing the outside file and directory named through the planted
+	// symlink must be refused.
+	o, err = f.NewObject(ctx, "pwn/secret.txt")
+	require.NoError(t, err)
+	err = o.Remove(ctx)
+	require.Error(t, err, "removing through a planted symlink should be refused")
+	_, err = os.Lstat(evilFile)
+	require.NoError(t, err, "a file was removed outside the destination at %q", evilFile)
+
+	err = f.Rmdir(ctx, "pwn/secret.d")
+	require.Error(t, err, "removing a directory through a planted symlink should be refused")
+	_, err = os.Lstat(evilDir)
+	require.NoError(t, err, "a directory was removed outside the destination at %q", evilDir)
+}
+
+// TestDirMoveThroughPlantedSymlinkBlocked checks DirMove can't move a
+// directory out of the root through a planted symlink.
+func TestDirMoveThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: an in-tree directory move works as normal.
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	require.NoError(t, f.DirMove(ctx, f, "dir", "sub/moved"))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evil))
+
+	err := f.DirMove(ctx, f, "sub/moved", "pwn/moved")
+	require.Error(t, err, "moving a directory through a planted symlink should be refused")
+	_, err = os.Lstat(filepath.Join(evil, "moved"))
+	require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evil)
+	_, err = os.Lstat(filepath.Join(f.root, "sub", "moved"))
+	require.NoError(t, err, "the directory should still be in the destination")
+}
+
 // TestEncodingEscapeBlocked checks that a name from a malicious source can't
 // be decoded into path syntax which writes outside the destination.
 func TestEncodingEscapeBlocked(t *testing.T) {

backend/local/metadata.go

@@ -5,7 +5,9 @@ import (
 	"math"
 	"os"
 	"runtime"
+	"slices"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -121,13 +123,15 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	uid, hasUID := o.parseMetadataInt(m, "uid", 10)
 	gid, hasGID := o.parseMetadataInt(m, "gid", 10)
 	if hasUID {
-		// FIXME should read UID and GID of current user and only attempt to set it if different
 		if !hasGID {
 			gid = uid
 		}
 		if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
 			fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
+		} else if setUID, setGID := ownerToSet(uid, gid); setUID < 0 && setGID < 0 {
+			fs.Debugf(o, "Not setting ownership %d.%d: only root can", uid, gid)
 		} else {
+			uid, gid = setUID, setGID
 			if o.translatedLink {
 				err = os.Lchown(o.path, uid, gid)
 			} else {
@@ -169,3 +173,27 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	// FIXME not parsing rdev yet
 	return outErr
 }
+
+// processGroups are the supplementary groups of this process
+var processGroups = sync.OnceValue(func() []int {
+	groups, _ := os.Getgroups()
+	return groups
+})
+
+// ownerToSet returns the uid and gid to pass to chown for the wanted
+// ownership: unchanged for root, else -1 (leave it as it is) for a uid
+// other than our own and for a group we are not a member of, which
+// only root may set.
+func ownerToSet(uid, gid int) (setUID, setGID int) {
+	euid := os.Geteuid()
+	if euid <= 0 {
+		return uid, gid
+	}
+	if uid != euid {
+		uid = -1
+	}
+	if gid != os.Getegid() && !slices.Contains(processGroups(), gid) {
+		gid = -1
+	}
+	return uid, gid
+}

backend/local/metadata_owner_unix_test.go

@@ -0,0 +1,50 @@
+//go:build !windows && !plan9 && !js
+
+package local
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetadataOwnershipAsNonRoot(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("needs a user other than root")
+	}
+	uid, gid := os.Geteuid(), os.Getegid()
+	other := uid + 1
+	notMember := gid + 1
+	for slices.Contains(processGroups(), notMember) {
+		notMember++
+	}
+
+	// only what a non-root user may set: its own uid, its groups
+	setUID, setGID := ownerToSet(other, notMember)
+	assert.Equal(t, [2]int{-1, -1}, [2]int{setUID, setGID})
+	setUID, setGID = ownerToSet(other, gid)
+	assert.Equal(t, [2]int{-1, gid}, [2]int{setUID, setGID})
+	setUID, setGID = ownerToSet(uid, notMember)
+	assert.Equal(t, [2]int{uid, -1}, [2]int{setUID, setGID})
+
+	// a file of another user (e.g. root:root) is copied with our ownership
+	// instead of failing with "operation not permitted"
+	r := fstest.NewRun(t)
+	r.WriteFile("owned.txt", "payload", time.Now())
+	f := r.Flocal.(*Fs)
+	obj, err := f.NewObject(context.Background(), "owned.txt")
+	require.NoError(t, err)
+	require.NoError(t, obj.(*Object).writeMetadataToFile(fs.Metadata{"uid": "0", "gid": "0"}))
+	fi, err := os.Stat(filepath.Join(f.root, "owned.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, uint32(uid), fi.Sys().(*syscall.Stat_t).Uid)
+}

backend/local/remove_other.go

@@ -2,9 +2,7 @@
 
 package local
 
-import "os"
-
-// Removes name, retrying on a sharing violation
-func remove(name string) error {
-	return os.Remove(name)
+// remove removes name with removeFn, retrying on a sharing violation
+func remove(name string, removeFn func(string) error) error {
+	return removeFn(name)
 }

backend/local/remove_test.go

@@ -38,7 +38,7 @@ func TestRemove(t *testing.T) {
 		require.NoError(t, fd.Close())
 	})
 	// delete the open file
-	err = remove(name)
+	err = remove(name, os.Remove)
 	require.NoError(t, err)
 	// check it no longer exists
 	assert.False(t, exists())

backend/local/remove_windows.go

@@ -10,12 +10,12 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-// Removes name, retrying on a sharing violation
-func remove(name string) (err error) {
+// remove removes name with removeFn, retrying on a sharing violation
+func remove(name string, removeFn func(string) error) (err error) {
 	const maxTries = 10
 	var sleepTime = 1 * time.Millisecond
 	for i := range maxTries {
-		err = os.Remove(name)
+		err = removeFn(name)
 		if err == nil {
 			break
 		}

backend/local/xattr.go

@@ -4,6 +4,7 @@ package local
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"syscall"
 
@@ -95,6 +96,11 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 	if !xattrSupported || o.fs.xattrSupported.Load() == 0 {
 		return nil
 	}
+	// When translating symlinks (-l/--links) write the xattrs through a
+	// descriptor opened under os.Root so a symlink planted in the destination
+	// can't redirect the write outside it (CWE-59). A translated link is
+	// itself the leaf, so it keeps the non-following path based write.
+	var fd *os.File
 	for k, value := range metadata {
 		k = strings.ToLower(k)
 		if _, found := systemMetadataInfo[k]; found {
@@ -102,9 +108,27 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 		}
 		k = xattrPrefix + k
 		v := []byte(value)
-		if o.fs.opt.FollowSymlinks {
+		switch {
+		case o.fs.opt.TranslateSymlinks && !o.translatedLink:
+			if fd == nil {
+				fd, err = o.fs.openFile(o.path, os.O_RDONLY, 0)
+				if os.IsPermission(err) {
+					// A write only file can't be opened for reading but its
+					// owner may still set xattrs on it, so try opening it for
+					// writing instead (which can't succeed for a directory).
+					if wfd, werr := o.fs.openFile(o.path, os.O_WRONLY, 0); werr == nil {
+						fd, err = wfd, nil
+					}
+				}
+				if err != nil {
+					return fmt.Errorf("failed to open for xattr: %w", err)
+				}
+				defer fs.CheckClose(fd, &err)
+			}
+			err = xattr.FSet(fd, k, v)
+		case o.fs.opt.FollowSymlinks:
 			err = xattr.Set(o.path, k, v)
-		} else {
+		default:
 			err = xattr.LSet(o.path, k, v)
 		}
 		if err != nil {

backend/local/xattr_internal_test.go

@@ -0,0 +1,96 @@
+//go:build !openbsd && !plan9
+
+package local
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/xattr"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestXattrThroughPlantedSymlinkBlocked checks user metadata (xattrs)
+// can't be written outside the root through a planted symlink with
+// --links, while in-tree writes still work.
+func TestXattrThroughPlantedSymlinkBlocked(t *testing.T) {
+	if !xattrSupported {
+		t.Skip("xattrs not supported on this OS")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	metadata := fs.Metadata{
+		"pwned": "attacker-controlled-value",
+	}
+
+	// Control: an in-tree directory gets the xattr as normal.
+	_, err := f.MkdirMetadata(ctx, "sub", metadata)
+	require.NoError(t, err)
+	if f.xattrSupported.Load() == 0 {
+		t.Skip("xattrs not supported on this filesystem")
+	}
+	v, err := xattr.LGet(filepath.Join(f.root, "sub"), "user.pwned")
+	require.NoError(t, err)
+	assert.Equal(t, "attacker-controlled-value", string(v))
+
+	// A directory outside the destination, with a pre-existing child,
+	// whose xattrs the attacker targets.
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	evilSub := filepath.Join(evilDir, "sub")
+	require.NoError(t, os.Mkdir(evilSub, 0700))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// The source now presents "pwn/sub" as a directory with attacker-chosen
+	// user metadata. Applying it must not reach through the planted symlink.
+	_, err = f.MkdirMetadata(ctx, "pwn/sub", metadata)
+	require.Error(t, err, "applying metadata through a planted symlink should be refused")
+
+	_, err = xattr.LGet(evilSub, "user.pwned")
+	require.Error(t, err, "xattr escaped through planted symlink to %q", evilSub)
+}
+
+// TestXattrWriteOnlyFile checks user metadata can still be set on a
+// write-only file with --links, where the confined open can't read it.
+func TestXattrWriteOnlyFile(t *testing.T) {
+	if !xattrSupported {
+		t.Skip("xattrs not supported on this OS")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	require.NoError(t, putFile(ctx, f, "wo.txt", "write only"))
+	path := filepath.Join(f.root, "wo.txt")
+	require.NoError(t, os.Chmod(path, 0200))
+
+	obj, err := f.NewObject(ctx, "wo.txt")
+	require.NoError(t, err)
+	o := obj.(*Object)
+	err = o.setXattr(fs.Metadata{"potato": "chips"})
+	require.NoError(t, err)
+	if f.xattrSupported.Load() == 0 {
+		t.Skip("xattrs not supported on this filesystem")
+	}
+	// Reading xattrs needs read permission
+	require.NoError(t, os.Chmod(path, 0600))
+	v, err := xattr.LGet(path, "user.potato")
+	require.NoError(t, err)
+	assert.Equal(t, "chips", string(v))
+}

backend/netstorage/netstorage.go

@@ -208,6 +208,24 @@ func (d Du) GetName() xml.Name {
 
 //------------------------------------------------------------------------------
 
+// apiHeaders are the NetStorage API request headers which carry the
+// account name, the request signature and the signed action. They must
+// not be forwarded when a redirect leaves the configured host.
+//
+// The action header is sent uncanonicalised, hence the "*" prefix.
+var apiHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "*X-Akamai-ACS-Action"}
+
+// newClient makes the http client used for all NetStorage requests.
+//
+// The signer adds the API headers to every request, so the client
+// strips them on cross host redirects and refuses an HTTPS to HTTP
+// downgrade.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(apiHeaders...)
+	return client
+}
+
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 //
@@ -240,7 +258,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
-	client := fshttp.NewClient(ctx)
+	client := newClient(ctx)
 
 	f := &Fs{
 		name:        name,

backend/netstorage/netstorage_internal_test.go

@@ -4,12 +4,17 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -59,3 +64,142 @@ func TestListResumeCannotChangeHost(t *testing.T) {
 	assert.Equal(t, 1, listRequests, "list request sent to the configured host")
 	assert.Equal(t, 0, requests, "request sent to the other host")
 }
+
+// apiTestHeaders is a deliberately literal copy of apiHeaders, without
+// the "*" prefix, so that a header dropped from the production list
+// shows up as a leak in the cross host tests rather than silently
+// losing coverage.
+var apiTestHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "X-Akamai-ACS-Action"}
+
+// statXML is a minimal stat response saying the root is a directory
+const statXML = `<?xml version="1.0" encoding="UTF-8"?>
+<stat directory="/dir"><file type="dir" name="dir" mtime="1"/></stat>`
+
+// newTestFs makes an Fs via NewFs pointing at serverURL.
+//
+// NewFs makes a stat request for the root and treats any failure as
+// the root not existing, so callers check what the servers received
+// or make a further request rather than relying on its error.
+func newTestFs(ctx context.Context, t *testing.T, serverURL string) *Fs {
+	u, err := url.Parse(serverURL)
+	require.NoError(t, err)
+	m := configmap.Simple{
+		"type":     "netstorage",
+		"protocol": u.Scheme,
+		"host":     u.Host + "/dir",
+		"account":  "test-account",
+		"secret":   obscure.MustObscure("test-secret"),
+	}
+	f, err := NewFs(ctx, "netstorage-test", "", m)
+	require.NoError(t, err)
+	return f.(*Fs)
+}
+
+// assertAPIHeaders checks all the API headers are present (want) or
+// absent (!want) in r.
+func assertAPIHeaders(t *testing.T, r *http.Request, want bool) {
+	for _, header := range apiTestHeaders {
+		if want {
+			assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
+		} else {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+	}
+}
+
+// TestRedirectStripsAPIHeadersOnCrossHost checks the API headers are
+// sent to the configured host but not to another host it redirects to.
+func TestRedirectStripsAPIHeadersOnCrossHost(t *testing.T) {
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAPIHeaders(t, r, false)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer other.Close()
+
+	var originHits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAPIHeaders(t, r, true)
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	newTestFs(context.Background(), t, origin.URL)
+	assert.Equal(t, int32(1), originHits.Load())
+	assert.Equal(t, int32(1), otherHits.Load())
+}
+
+// TestUploadStripsAPIHeadersOnCrossHost checks an upload, which
+// carries the API headers on the PUT as well as in the trailers, does
+// not send them to another host the configured host redirects to.
+func TestUploadStripsAPIHeadersOnCrossHost(t *testing.T) {
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAPIHeaders(t, r, false)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer other.Close()
+
+	var originHits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAPIHeaders(t, r, true)
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusMovedPermanently)
+	}))
+	defer origin.Close()
+
+	ctx := context.Background()
+	f := newTestFs(ctx, t, origin.URL)
+	o := &Object{fs: f, remote: "file"}
+	src := object.NewStaticObjectInfo("file", time.Now(), 4, true, nil, f)
+	err := o.netStorageUploadRequest(ctx, strings.NewReader("data"), src)
+	require.NoError(t, err)
+	// NewFs stat, PUT and the stat after the upload
+	assert.Equal(t, int32(3), originHits.Load())
+	assert.Equal(t, int32(3), otherHits.Load())
+}
+
+// TestRedirectKeepsAPIHeadersOnSameHost checks the API headers are
+// still sent after a redirect which stays on the configured host.
+func TestRedirectKeepsAPIHeadersOnSameHost(t *testing.T) {
+	var hits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		assertAPIHeaders(t, r, true)
+		if !strings.HasSuffix(r.URL.Path, "/moved") {
+			http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
+			return
+		}
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer origin.Close()
+
+	newTestFs(context.Background(), t, origin.URL)
+	assert.Equal(t, int32(2), hits.Load())
+}
+
+// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
+// plaintext HTTP is not followed.
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+	var targetHits atomic.Int32
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		targetHits.Add(1)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer target.Close()
+
+	origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.InsecureSkipVerify = true
+	f := newTestFs(ctx, t, origin.URL)
+	_, err := f.netStorageStatRequest(ctx, f.url(""), true)
+	require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	assert.Equal(t, int32(0), targetHits.Load())
+}

backend/onedrive/onedrive.go

@@ -41,6 +41,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -844,19 +845,33 @@ type Options struct {
 
 // Fs represents a remote OneDrive
 type Fs struct {
-	name         string             // name of this remote
-	root         string             // the path we are working on
-	opt          Options            // parsed options
-	ci           *fs.ConfigInfo     // global config
-	features     *fs.Features       // optional features
-	srv          *rest.Client       // the connection to the OneDrive server
-	unAuth       *rest.Client       // no authentication connection to the OneDrive server
-	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
-	tokenRenewer *oauthutil.Renew   // renew the token on expiry
-	driveID      string             // ID to use for querying Microsoft Graph
-	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
-	hashType     hash.Type          // type of the hash we are using
+	name         string                 // name of this remote
+	root         string                 // the path we are working on
+	opt          Options                // parsed options
+	ci           *fs.ConfigInfo         // global config
+	features     *fs.Features           // optional features
+	srv          *rest.Client           // the connection to the OneDrive server
+	unAuth       *rest.Client           // no authentication connection to the OneDrive server
+	dirCache     *dircache.DirCache     // Map of directory path to directory id
+	pacer        *fs.Pacer              // pacer for API calls
+	tokenRenewer *oauthutil.Renew       // renew the token on expiry
+	tokenSource  *oauthutil.TokenSource // source of the API token
+	sharePoint   *sharePointClients     // authenticated clients for SharePoint hosts
+	driveID      string                 // ID to use for querying Microsoft Graph
+	driveType    string                 // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
+	hashType     hash.Type              // type of the hash we are using
+}
+
+// sharePointClients holds the clients authenticated for SharePoint hosts
+type sharePointClients struct {
+	mu    sync.Mutex
+	hosts map[string]sharePointAuth
+}
+
+// sharePointAuth is a client authenticated for a SharePoint host
+type sharePointAuth struct {
+	client      *rest.Client
+	tokenSource oauth2.TokenSource
 }
 
 // Object describes a OneDrive object
@@ -1076,6 +1091,154 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
+// errPreauthRetired explains an authentication failure on a URL which
+// SharePoint used to return pre-authenticated.
+var errPreauthRetired = errors.New("SharePoint refused to authenticate a request - Microsoft is retiring pre-authenticated (tempauth) URLs, see https://rclone.org/onedrive/#pre-authenticated-urls")
+
+// sharePointDomains are the domains of SharePoint Online in the
+// different Microsoft clouds
+var sharePointDomains = []string{".sharepoint.com", ".sharepoint.us", ".sharepoint-mil.us", ".sharepoint.cn", ".sharepoint.de"}
+
+// stripTempauth removes the tempauth parameter from the URLs the API
+// returns, which simulates the retirement of pre-authenticated URLs
+// in tests.
+var stripTempauth = false
+
+// returnedURL processes a URL returned by the API before use
+func returnedURL(rawURL string) string {
+	if !stripTempauth {
+		return rawURL
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	q := u.Query()
+	q.Del("tempauth")
+	u.RawQuery = q.Encode()
+	return u.String()
+}
+
+// urlAuth says how a URL returned by the API is authenticated
+type urlAuth int
+
+const (
+	urlAuthNone       urlAuth = iota // call without authentication
+	urlAuthAPI                       // call with the API token
+	urlAuthSharePoint                // call with a token for the SharePoint host
+)
+
+// apiHost returns the host the API calls made with f.srv go to.
+func (f *Fs) apiHost() string {
+	endpoint := graphAPIEndpoint[f.opt.Region]
+	if f.opt.TenantURL != "" {
+		endpoint = f.opt.TenantURL
+	}
+	u, err := url.Parse(endpoint)
+	if err != nil || u.Scheme != "https" {
+		return ""
+	}
+	return u.Host
+}
+
+// urlAuthFor returns how to authenticate a URL returned by the API,
+// such as an upload session or an async job monitor.
+//
+// Pre-authenticated URLs fail if they are sent an Authorization
+// header and tokens must not be sent to other hosts, so only HTTPS
+// URLs without tempauth on the API host or a SharePoint host are
+// authenticated.
+func (f *Fs) urlAuthFor(rawURL string) urlAuth {
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Scheme != "https" || u.Query().Has("tempauth") {
+		return urlAuthNone
+	}
+	if apiHost := f.apiHost(); apiHost != "" && strings.EqualFold(u.Host, apiHost) {
+		return urlAuthAPI
+	}
+	if isSharePointHost(u.Hostname()) {
+		return urlAuthSharePoint
+	}
+	return urlAuthNone
+}
+
+// isSharePointHost returns true if host is in a SharePoint Online domain.
+func isSharePointHost(host string) bool {
+	host = strings.ToLower(host)
+	for _, domain := range sharePointDomains {
+		if strings.HasSuffix(host, domain) {
+			return true
+		}
+	}
+	return false
+}
+
+// checkSharePointRedirect stops the SharePoint token being sent to a
+// host which isn't SharePoint, as the token is added to every request
+// the client makes.
+func checkSharePointRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if req.URL.Scheme != "https" || !isSharePointHost(req.URL.Hostname()) {
+		return fmt.Errorf("refusing to send a SharePoint token on a redirect to %s://%s", req.URL.Scheme, req.URL.Host)
+	}
+	return nil
+}
+
+// clientForURL returns the client to call a URL returned by the API with.
+func (f *Fs) clientForURL(ctx context.Context, rawURL string) (*rest.Client, error) {
+	switch f.urlAuthFor(rawURL) {
+	case urlAuthAPI:
+		return f.srv, nil
+	case urlAuthSharePoint:
+		u, _ := url.Parse(rawURL)
+		return f.sharePointClient(ctx, u.Host)
+	}
+	return f.unAuth, nil
+}
+
+// sharePointClient returns a client authenticated for the SharePoint
+// host, which upload session and async job monitor URLs need once
+// they are no longer pre-authenticated.
+//
+// The token is got from the existing credentials, which Entra ID
+// allows for SharePoint as the user has consented to the Graph file
+// scopes.
+func (f *Fs) sharePointClient(ctx context.Context, host string) (*rest.Client, error) {
+	if f.tokenSource == nil || f.sharePoint == nil {
+		return nil, fmt.Errorf("%w: no credentials to get a SharePoint token with", errPreauthRetired)
+	}
+	host = strings.ToLower(host)
+	f.sharePoint.mu.Lock()
+	defer f.sharePoint.mu.Unlock()
+	sp, ok := f.sharePoint.hosts[host]
+	if !ok {
+		tokenSource := f.tokenSource.ScopedTokenSource([]string{"https://" + host + "/.default"})
+		client := oauth2.NewClient(oauthutil.Context(ctx, fshttp.NewClient(ctx)), tokenSource)
+		client.CheckRedirect = checkSharePointRedirect
+		sp = sharePointAuth{client: rest.NewClient(client), tokenSource: tokenSource}
+		f.sharePoint.hosts[host] = sp
+		fs.Debugf(f, "Using a SharePoint token for %s", host)
+	}
+	// Get the token now so failures are reported before the client is used
+	if _, err := sp.tokenSource.Token(); err != nil {
+		return nil, fmt.Errorf("%w: failed to get a SharePoint token: %w", errPreauthRetired, err)
+	}
+	return sp.client, nil
+}
+
+// checkPreauthRetired adds an explanation to a 401 error from a URL
+// returned by the API which needs a SharePoint token, as these were
+// pre-authenticated until Microsoft retired them.
+func (f *Fs) checkPreauthRetired(rawURL string, resp *http.Response, err error) error {
+	if err == nil || resp == nil || resp.StatusCode != http.StatusUnauthorized ||
+		f.driveType == driveTypePersonal || errors.Is(err, errPreauthRetired) || f.urlAuthFor(rawURL) != urlAuthSharePoint {
+		return err
+	}
+	return fmt.Errorf("%w: %w", errPreauthRetired, err)
+}
+
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
 	const minChunkSize = fs.SizeSuffixBase
 	if cs%chunkSizeMultiple != 0 {
@@ -1144,16 +1307,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:      name,
-		root:      root,
-		opt:       *opt,
-		ci:        ci,
-		driveID:   opt.DriveID,
-		driveType: opt.DriveType,
-		srv:       rest.NewClient(oAuthClient).SetRoot(rootURL),
-		unAuth:    rest.NewClient(client).SetRoot(rootURL),
-		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		hashType:  QuickXorHashType,
+		name:        name,
+		root:        root,
+		opt:         *opt,
+		ci:          ci,
+		driveID:     opt.DriveID,
+		driveType:   opt.DriveType,
+		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
+		tokenSource: ts,
+		sharePoint:  &sharePointClients{hosts: make(map[string]sharePointAuth)},
+		unAuth:      rest.NewClient(client).SetRoot(rootURL),
+		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		hashType:    QuickXorHashType,
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:          true,
@@ -1725,11 +1890,25 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 		var resp *http.Response
 		var err error
 		var body []byte
+		var redirected bool
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = http.Get(location)
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
+			if err != nil {
+				return false, err
+			}
+			client, err := f.clientForURL(ctx, location)
+			if err != nil {
+				return false, err
+			}
+			resp, err = client.Do(req)
 			if err != nil {
 				return fserrors.ShouldRetry(err), err
 			}
+			redirected = resp.Request.URL.String() != req.URL.String()
+			if resp.StatusCode == http.StatusUnauthorized {
+				_ = resp.Body.Close()
+				return false, f.checkPreauthRetired(location, resp, fmt.Errorf("async job monitor: HTTP error %s", resp.Status))
+			}
 			body, err = rest.ReadBody(resp)
 			return fserrors.ShouldRetry(err), err
 		})
@@ -1742,6 +1921,11 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 		if err != nil {
 			return fmt.Errorf("async status result not JSON: %q: %w", body, err)
 		}
+		// An authenticated monitor redirects to the new item when the
+		// job has finished instead of returning its status
+		if status.Status == "" && redirected {
+			status.Status = "completed"
+		}
 
 		switch status.Status {
 		case "failed":
@@ -1841,7 +2025,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	// read location header
-	location := resp.Header.Get("Location")
+	location := returnedURL(resp.Header.Get("Location"))
 	if location == "" {
 		return nil, errors.New("didn't receive location header in copy response")
 	}
@@ -1852,6 +2036,10 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		fs.Debugf(src, "Server-side copy failed - file not shared between drives")
 		return nil, fs.ErrorCantCopy
 	}
+	if errors.Is(err, errPreauthRetired) {
+		fs.Debugf(src, "Server-side copy can't be monitored - copying another way: %v", err)
+		return nil, fs.ErrorCantCopy
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -2526,16 +2714,23 @@ func (o *Object) openContentPrefer(ctx context.Context, options ...fs.OpenOption
 	return o.openWithRedirect(ctx, &opts)
 }
 
-// openWithRedirect downloads via /content style endpoints that 302 to a preauthenticated URL.
+// openWithRedirect downloads via /content style endpoints that 302 to
+// a download URL, which is pre-authenticated or needs a SharePoint
+// token.
 func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.ReadCloser, err error) {
 	var resp *http.Response
-	// Make a note of the redirect target as we need to call it without Auth
+	// Make a note of the redirect target as it needs different auth
 	var redirectReq *http.Request
 	opts.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 		if len(via) >= 10 {
 			return errors.New("stopped after 10 redirects")
 		}
-		// Preauthenticated download URLs must not carry the Graph Authorization header.
+		if stripTempauth {
+			if u, err := url.Parse(returnedURL(req.URL.String())); err == nil {
+				req.URL = u
+			}
+		}
+		// Download URLs must not carry the Graph Authorization header.
 		req.Header.Del("Authorization")
 		// Keep Prefer on the redirect when forcing infected download; some SharePoint
 		// endpoints honor it only on the final download request.
@@ -2574,17 +2769,24 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
 		return nil, err
 	}
 	if redirectReq != nil {
+		redirectURL := redirectReq.URL.String()
+		var client *rest.Client
+		client, err = o.fs.clientForURL(ctx, redirectURL)
+		if err != nil {
+			return nil, err
+		}
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.unAuth.Do(redirectReq)
+			resp, err = client.Do(redirectReq)
 			if err != nil {
 				return shouldRetry(ctx, resp, err)
 			}
-			// unAuth.Do does not check status; treat non-2xx as failure so a malware
+			// Do does not check status; treat non-2xx as failure so a malware
 			// JSON body is not written out as file content.
 			if resp.StatusCode < 200 || resp.StatusCode > 299 {
 				body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
 				_ = resp.Body.Close()
 				err = fmt.Errorf("HTTP error %d (%s) %s", resp.StatusCode, resp.Status, strings.TrimSpace(string(body)))
+				err = o.fs.checkPreauthRetired(redirectURL, resp, err)
 				if strings.Contains(string(body), "malwareDetected") || resp.Header.Get("X-Virus-Infected") != "" {
 					err = malwareDownloadError(o.fs.opt.AVOverride, err)
 				}
@@ -2606,7 +2808,19 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
 		// Overwrite size with actual size since size readings from Onedrive is unreliable.
 		o.size = resp.ContentLength
 	}
-	return resp.Body, err
+	return o.checkRange(resp, opts.Options)
+}
+
+// checkRange returns the body of a download, checking it is the range
+// requested in options.
+func (o *Object) checkRange(resp *http.Response, options []fs.OpenOption) (in io.ReadCloser, err error) {
+	// The size isn't passed as it is unreliable on OneDrive
+	err = rest.CheckContentRange(resp, options, -1)
+	if err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
+	return resp.Body, nil
 }
 
 // malwareDownloadError formats an error when the server blocks a malware-flagged file.
@@ -2646,12 +2860,16 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
 	}
 	var info api.UploadFragmentResponse
 	var resp *http.Response
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return 0, err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
+		resp, err = client.CallJSON(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return 0, err
+		return 0, o.fs.checkPreauthRetired(url, resp, err)
 	}
 	if len(info.NextExpectedRanges) != 1 {
 		return 0, fmt.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
@@ -2675,6 +2893,10 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 	var resp *http.Response
 	var body []byte
 	skip := int64(0)
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return nil, err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		toSend := chunkSize - skip
 		opts := rest.Opts{
@@ -2686,7 +2908,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			Options:       options,
 		}
 		_, _ = chunk.Seek(skip, io.SeekStart)
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = client.Call(ctx, &opts)
 		if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
 			fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
 			pos, posErr := o.getPosition(ctx, url)
@@ -2725,7 +2947,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 		}
 		return false, nil
 	})
-	return info, err
+	return info, o.fs.checkPreauthRetired(url, resp, err)
 }
 
 // cancelUploadSession cancels an upload session
@@ -2736,11 +2958,15 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
 		NoResponse: true,
 	}
 	var resp *http.Response
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = client.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
-	return
+	return o.fs.checkPreauthRetired(url, resp, err)
 }
 
 // uploadMultipart uploads a file using multipart upload
@@ -2758,7 +2984,20 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 	if err != nil {
 		return nil, err
 	}
-	uploadURL := session.UploadURL
+	uploadURL := returnedURL(session.UploadURL)
+
+	// Check a session which needs SharePoint authentication can get it
+	// before reading any data, so it can be uploaded in a single part
+	// instead if it can't.
+	if o.fs.urlAuthFor(uploadURL) == urlAuthSharePoint {
+		if _, err = o.getPosition(ctx, uploadURL); err != nil {
+			if !errors.Is(err, errPreauthRetired) || size >= int64(maxSinglePartSize) {
+				return nil, err
+			}
+			fs.Debugf(o, "Uploading in a single part as the upload session can't be used: %v", err)
+			return o.uploadSinglepart(ctx, in, src, options...)
+		}
+	}
 
 	// Cancel the session if something went wrong
 	defer atexit.OnError(&err, func() {

backend/onedrive/onedrive_internal_test.go

@@ -3,6 +3,7 @@ package onedrive
 import (
 	"context"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -15,6 +16,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/backend/onedrive/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -43,6 +45,10 @@ const (
 	// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1
 )
 
+func init() {
+	flag.BoolVar(&stripTempauth, "strip-tempauth", false, "Remove tempauth from URLs the API returns to simulate the retirement of pre-authenticated URLs")
+}
+
 // TestMain drives the tests
 func TestMain(m *testing.M) {
 	fstest.TestMain(m)
@@ -770,3 +776,96 @@ func TestListDirectorySize(t *testing.T) {
 	assert.Equal(t, int64(2646204760), dir.Size())
 	assert.Equal(t, int64(3), dir.Items())
 }
+
+// TestWaitForJobUsesConfiguredClient checks that the async job monitor is
+// polled with the configured HTTP client and request context rather than
+// http.DefaultClient, so proxy, TLS and timeout options and cancellation apply.
+func TestWaitForJobUsesConfiguredClient(t *testing.T) {
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.UserAgent = "rclone-waitforjob-test"
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "rclone-waitforjob-test", r.UserAgent())
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(api.AsyncOperationStatus{Status: "deleteFailed"})
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(ctx))
+
+	err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "deleteFailed")
+}
+
+// TestWaitForJobCancel checks that cancelling the context stops a poll of
+// the async job monitor which is not answering.
+func TestWaitForJobCancel(t *testing.T) {
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-release:
+		}
+	}))
+	t.Cleanup(server.Close)
+	t.Cleanup(func() { close(release) })
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"}) }()
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, context.DeadlineExceeded)
+	case <-time.After(5 * time.Second):
+		t.Fatal("waitForJob did not return after the context was cancelled")
+	}
+}
+
+// TestWaitForJobUnauthorized checks that a monitor URL which refuses
+// the request fails straight away instead of polling until the timeout.
+func TestWaitForJobUnauthorized(t *testing.T) {
+	var polls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		polls.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusUnauthorized)
+		_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "unauthenticated", "message": "no token"}})
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+	f.driveType = driveTypeBusiness
+
+	err := f.waitForJob(context.Background(), server.URL+"/monitor", &Object{remote: "file.txt"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "401")
+	assert.Equal(t, int32(1), polls.Load())
+}
+
+// TestWaitForJobRedirectCompleted checks that a monitor which
+// redirects to the new item, as it does when authenticated, is taken
+// to mean the job has finished.
+func TestWaitForJobRedirectCompleted(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/monitor":
+			http.Redirect(w, r, "/item", http.StatusSeeOther)
+		case "/item":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"id":"newid","name":"file.txt","size":5,"file":{}}`)
+		default:
+			t.Errorf("unexpected request %s", r.URL)
+		}
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt", hasMetaData: true})
+	require.NoError(t, err)
+}

backend/onedrive/upload_internal_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,11 +14,14 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/oauth2"
 )
 
 // fragmentServer is a fake OneDrive upload session which records the
@@ -66,6 +70,8 @@ func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		w.WriteHeader(http.StatusAccepted)
 		_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)
+	case r.Method == "DELETE" && r.URL.Path == "/upload":
+		w.WriteHeader(http.StatusNoContent)
 	default:
 		s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
 		w.WriteHeader(http.StatusNotFound)
@@ -118,3 +124,327 @@ func TestUploadMultipartSkip(t *testing.T) {
 	assert.Equal(t, 2, s.fragments)
 	assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")
 }
+
+// TestURLAuthFor checks that tokens are only sent to HTTPS URLs on
+// the API host or a SharePoint host which aren't pre-authenticated.
+func TestURLAuthFor(t *testing.T) {
+	const tenant = "https://contoso-my.sharepoint.com/_api"
+	for _, tc := range []struct {
+		name      string
+		region    string
+		tenantURL string
+		url       string
+		want      urlAuth
+	}{
+		{"graph", "global", "", "https://graph.microsoft.com/v1.0/drives/x/items/y/uploadSession", urlAuthAPI},
+		{"graph case", "global", "", "https://Graph.Microsoft.com/v1.0/monitor/abc", urlAuthAPI},
+		{"graph tempauth", "global", "", "https://graph.microsoft.com/v1.0/upload?tempauth=abc", urlAuthNone},
+		{"graph http", "global", "", "http://graph.microsoft.com/v1.0/upload", urlAuthNone},
+		{"graph other region", "global", "", "https://graph.microsoft.us/v1.0/upload", urlAuthNone},
+		{"graph us", "us", "", "https://graph.microsoft.us/v1.0/upload", urlAuthAPI},
+		{"graph lookalike", "global", "", "https://graph.microsoft.com.example.com/upload", urlAuthNone},
+		{"sharepoint tempauth", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x&tempauth=abc", urlAuthNone},
+		{"sharepoint", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x", urlAuthSharePoint},
+		{"sharepoint case", "global", "", "https://Contoso.SharePoint.com/_api/v2.0/monitor/abc", urlAuthSharePoint},
+		{"sharepoint us", "us", "", "https://contoso.sharepoint.us/_api/v2.0/upload", urlAuthSharePoint},
+		{"sharepoint cn", "cn", "", "https://contoso.sharepoint.cn/_api/v2.0/upload", urlAuthSharePoint},
+		{"sharepoint http", "global", "", "http://contoso-my.sharepoint.com/_api/v2.0/upload", urlAuthNone},
+		{"sharepoint lookalike", "global", "", "https://contoso.sharepoint.com.example.com/upload", urlAuthNone},
+		{"sharepoint suffix only", "global", "", "https://evilsharepoint.com/upload", urlAuthNone},
+		{"personal", "global", "", "https://api.onedrive.com/rup/abc", urlAuthNone},
+		{"personal upload", "global", "", "https://my.microsoftpersonalcontent.com/personal/abc/_api/v2.0/drive/items/x/uploadSession", urlAuthNone},
+		{"unparseable", "global", "", "https://graph.microsoft.com/%zz", urlAuthNone},
+		{"tenant", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/monitor/abc", urlAuthAPI},
+		{"tenant tempauth", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/upload?tempauth=abc", urlAuthNone},
+		{"tenant graph", "global", tenant, "https://graph.microsoft.com/v1.0/upload", urlAuthNone},
+		{"tenant other sharepoint", "global", tenant, "https://contoso.sharepoint.com/_api/v2.0/upload", urlAuthSharePoint},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &Fs{}
+			f.opt.Region = tc.region
+			f.opt.TenantURL = tc.tenantURL
+			assert.Equal(t, tc.want, f.urlAuthFor(tc.url))
+		})
+	}
+}
+
+// TestCheckSharePointRedirect checks the SharePoint token is only
+// sent on redirects to SharePoint over HTTPS.
+func TestCheckSharePointRedirect(t *testing.T) {
+	for _, tc := range []struct {
+		url string
+		ok  bool
+	}{
+		{"https://contoso-my.sharepoint.com/download.aspx", true},
+		{"https://205807-ipv4v6.gr.global.aa-rt.sharepoint.com/download.aspx?tempauth=x", true},
+		{"https://contoso.sharepoint.us/download.aspx", true},
+		{"http://contoso-my.sharepoint.com/download.aspx", false},
+		{"https://contoso.sharepoint.com.example.com/download.aspx", false},
+		{"https://graph.microsoft.com/v1.0/me", false},
+		{"https://example.com/", false},
+	} {
+		req, err := http.NewRequest("GET", tc.url, nil)
+		require.NoError(t, err)
+		err = checkSharePointRedirect(req, nil)
+		if tc.ok {
+			assert.NoError(t, err, tc.url)
+		} else {
+			assert.Error(t, err, tc.url)
+		}
+	}
+	req, err := http.NewRequest("GET", "https://contoso-my.sharepoint.com/", nil)
+	require.NoError(t, err)
+	assert.Error(t, checkSharePointRedirect(req, make([]*http.Request, 10)), "too many redirects")
+}
+
+// TestCheckPreauthRetired checks which 401 errors get the explanation.
+func TestCheckPreauthRetired(t *testing.T) {
+	const sharePointURL = "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x"
+	errBase := errors.New("HTTP error 401")
+	for _, tc := range []struct {
+		name      string
+		driveType string
+		url       string
+		status    int
+		want      bool
+	}{
+		{"sharepoint 401", driveTypeBusiness, sharePointURL, http.StatusUnauthorized, true},
+		{"library 401", driveTypeSharepoint, sharePointURL, http.StatusUnauthorized, true},
+		{"personal 401", driveTypePersonal, sharePointURL, http.StatusUnauthorized, false},
+		{"sharepoint 403", driveTypeBusiness, sharePointURL, http.StatusForbidden, false},
+		{"tempauth 401", driveTypeBusiness, sharePointURL + "&tempauth=abc", http.StatusUnauthorized, false},
+		{"graph 401", driveTypeBusiness, "https://graph.microsoft.com/v1.0/upload", http.StatusUnauthorized, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &Fs{driveType: tc.driveType}
+			f.opt.Region = "global"
+			err := f.checkPreauthRetired(tc.url, &http.Response{StatusCode: tc.status}, errBase)
+			assert.ErrorIs(t, err, errBase)
+			assert.Equal(t, tc.want, errors.Is(err, errPreauthRetired), "err = %v", err)
+		})
+	}
+	f := &Fs{driveType: driveTypeBusiness}
+	assert.NoError(t, f.checkPreauthRetired(sharePointURL, &http.Response{StatusCode: http.StatusUnauthorized}, nil))
+	assert.ErrorIs(t, f.checkPreauthRetired(sharePointURL, nil, errBase), errBase)
+}
+
+// TestSharePointClient checks that the client for a SharePoint host
+// is authenticated with a token for that host got from the existing
+// refresh token, which is fetched once and not stored.
+func TestSharePointClient(t *testing.T) {
+	var scopes []string
+	var auths []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/token":
+			_ = r.ParseForm()
+			assert.Equal(t, "refresh_token", r.PostForm.Get("grant_type"))
+			assert.Equal(t, "refresh", r.PostForm.Get("refresh_token"))
+			scopes = append(scopes, r.PostForm.Get("scope"))
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"access_token":"sharepoint_token","token_type":"Bearer","expires_in":3600}`)
+		case "/upload":
+			auths = append(auths, r.Header.Get("Authorization"))
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"nextExpectedRanges":["0-"]}`)
+		case "/redirect":
+			http.Redirect(w, r, "/upload", http.StatusFound)
+		default:
+			t.Errorf("unexpected request %s %s", r.Method, r.URL)
+		}
+	}))
+	t.Cleanup(server.Close)
+
+	storedToken := `{"access_token":"graph_token","token_type":"Bearer","refresh_token":"refresh","expiry":"2099-01-01T00:00:00Z"}`
+	m := configmap.Simple{"token": storedToken}
+	ctx := context.Background()
+	_, ts, err := oauthutil.NewClientWithBaseClient(ctx, "remote", m, &oauthutil.Config{
+		ClientID:     "client_id",
+		ClientSecret: "client_secret",
+		TokenURL:     server.URL + "/token",
+	}, server.Client())
+	require.NoError(t, err)
+	f := &Fs{tokenSource: ts, sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)}}
+
+	for range 2 {
+		client, err := f.sharePointClient(ctx, "Contoso-my.sharepoint.com")
+		require.NoError(t, err)
+		_, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/upload"})
+		require.NoError(t, err)
+	}
+	assert.Equal(t, []string{"https://contoso-my.sharepoint.com/.default"}, scopes, "token should be fetched once for the host")
+	assert.Equal(t, []string{"Bearer sharepoint_token", "Bearer sharepoint_token"}, auths)
+	assert.Equal(t, storedToken, m["token"], "the stored token must not change")
+
+	// The token isn't sent on a redirect to a host which isn't SharePoint
+	client, err := f.sharePointClient(ctx, "contoso-my.sharepoint.com")
+	require.NoError(t, err)
+	_, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/redirect"})
+	assert.ErrorContains(t, err, "refusing to send a SharePoint token")
+	assert.Len(t, auths, 2)
+
+	// Without credentials there is no client and the error explains why
+	_, err = (&Fs{}).sharePointClient(ctx, "contoso-my.sharepoint.com")
+	assert.ErrorIs(t, err, errPreauthRetired)
+}
+
+// TestUploadSessionFallback checks that an upload session which needs
+// SharePoint authentication which can't be got is replaced by a single
+// part upload before any data is read.
+func TestUploadSessionFallback(t *testing.T) {
+	content := bytes.Repeat([]byte("onedrive"), 1024)
+	var requests []string
+	var uploaded []byte
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests = append(requests, r.Method+" "+r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":
+			_, _ = fmt.Fprint(w, `{"uploadUrl":"https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x","nextExpectedRanges":["0-"]}`)
+		case r.Method == "PUT" && r.URL.Path == "/root/content":
+			var err error
+			uploaded, err = io.ReadAll(r.Body)
+			require.NoError(t, err)
+			_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
+		case r.Method == "PATCH":
+			_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
+		default:
+			t.Errorf("unexpected request %s %s", r.Method, r.URL)
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(server.Close)
+	client := rest.NewClient(server.Client()).SetRoot(server.URL)
+	f := &Fs{
+		ci:        fs.GetConfig(context.Background()),
+		srv:       client,
+		unAuth:    client,
+		driveType: driveTypeBusiness,
+		pacer:     fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+	}
+	f.opt.Region = "global"
+	f.opt.ChunkSize = fs.SizeSuffix(len(content) / 2)
+	o := &Object{fs: f}
+
+	src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)
+	_, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)
+	require.NoError(t, err)
+	assert.Equal(t, content, uploaded)
+	assert.Equal(t, []string{"POST /root/createUploadSession", "PUT /root/content"}, requests[:2])
+}
+
+// redirectTransport sends every request to the test server, so URLs
+// on other hosts can be tested.
+type redirectTransport struct {
+	server *httptest.Server
+}
+
+func (t redirectTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.URL.Scheme = "http"
+	req.URL.Host = t.server.Listener.Addr().String()
+	return t.server.Client().Transport.RoundTrip(req)
+}
+
+// newDownloadTest returns an Object which downloads content from a
+// server which handles the /content call with contentHandler and
+// serves download.aspx with Range support, recording the
+// Authorization header of each download.
+func newDownloadTest(t *testing.T, content []byte, contentHandler http.HandlerFunc) (o *Object, auths *[]string) {
+	auths = new([]string)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/items/fake/content":
+			contentHandler(w, r)
+		case "/download.aspx":
+			*auths = append(*auths, r.Header.Get("Authorization"))
+			http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(content))
+		default:
+			t.Errorf("unexpected request %s %s", r.Method, r.URL)
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(server.Close)
+	transport := redirectTransport{server: server}
+	sharePointToken := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "sharepoint_token"})
+	sharePointHTTP := &http.Client{Transport: &oauth2.Transport{Source: sharePointToken, Base: transport}}
+	f := &Fs{
+		ci:          fs.GetConfig(context.Background()),
+		srv:         rest.NewClient(server.Client()).SetRoot(server.URL),
+		unAuth:      rest.NewClient(&http.Client{Transport: transport}),
+		driveType:   driveTypeBusiness,
+		tokenSource: &oauthutil.TokenSource{},
+		sharePoint: &sharePointClients{hosts: map[string]sharePointAuth{
+			"contoso-my.sharepoint.com": {client: rest.NewClient(sharePointHTTP), tokenSource: sharePointToken},
+		}},
+		pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+	}
+	f.opt.Region = "global"
+	return &Object{fs: f, id: "fake", size: int64(len(content))}, auths
+}
+
+// TestOpenRedirectAuth checks that downloads are redirected to a
+// pre-authenticated URL without authentication and to a SharePoint
+// URL with a SharePoint token.
+func TestOpenRedirectAuth(t *testing.T) {
+	content := []byte("0123456789abcdefghij")
+	for _, test := range []struct {
+		name     string
+		location string
+		wantAuth string
+	}{
+		{"tempauth", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x&tempauth=v1.secret", ""},
+		{"sharepoint", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", "Bearer sharepoint_token"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
+				http.Redirect(w, r, test.location, http.StatusFound)
+			})
+			in, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
+			require.NoError(t, err)
+			got, err := io.ReadAll(in)
+			require.NoError(t, err)
+			require.NoError(t, in.Close())
+			assert.Equal(t, "56789", string(got))
+			assert.Equal(t, []string{test.wantAuth}, *auths)
+		})
+	}
+
+	// Without credentials the error explains why
+	o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", http.StatusFound)
+	})
+	o.fs.tokenSource = nil
+	_, err := o.Open(context.Background())
+	assert.ErrorIs(t, err, errPreauthRetired)
+	assert.Empty(t, *auths)
+}
+
+// TestOpenRangeCheck checks that a download which doesn't return the
+// requested range is an error rather than wrong data.
+func TestOpenRangeCheck(t *testing.T) {
+	content := []byte("0123456789abcdefghij")
+	for _, test := range []struct {
+		name    string
+		handler http.HandlerFunc
+		wantErr error
+	}{
+		{"ignored", func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write(content)
+		}, fs.ErrorRangeIgnored},
+		{"wrong range", func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Range", "bytes 0-4/20")
+			w.WriteHeader(http.StatusPartialContent)
+			_, _ = w.Write(content[:5])
+		}, nil},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o, _ := newDownloadTest(t, content, test.handler)
+			_, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
+			require.Error(t, err)
+			if test.wantErr != nil {
+				assert.ErrorIs(t, err, test.wantErr)
+			}
+		})
+	}
+}

backend/oracleobjectstorage/client.go

@@ -126,6 +126,10 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 		if ociError.GetCode() == "RequestTimeout" {
 			return true, err
 		}
+		// Parallel part uploads can collide on the same object
+		if ociError.GetHTTPStatusCode() == http.StatusConflict && ociError.GetCode() == "ConcurrentObjectUpdate" {
+			return true, err
+		}
 	}
 	// Ok, not an oci error, check for generic failure conditions
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err

backend/oracleobjectstorage/client_test.go

@@ -102,3 +102,34 @@ func TestGzipEncoding(t *testing.T) {
 		assert.Equal(t, "", md5sum)
 	})
 }
+
+// serviceError is a minimal common.ServiceError for shouldRetry
+type serviceError struct {
+	status int
+	code   string
+}
+
+func (e serviceError) Error() string           { return e.code }
+func (e serviceError) GetHTTPStatusCode() int  { return e.status }
+func (e serviceError) GetMessage() string      { return e.code }
+func (e serviceError) GetCode() string         { return e.code }
+func (e serviceError) GetOpcRequestID() string { return "" }
+
+func TestShouldRetryServiceError(t *testing.T) {
+	ctx := context.Background()
+	for _, test := range []struct {
+		status int
+		code   string
+		want   bool
+	}{
+		{http.StatusConflict, "ConcurrentObjectUpdate", true},
+		{http.StatusConflict, "BucketAlreadyExists", false},
+		{http.StatusRequestTimeout, "RequestTimeout", true},
+		{http.StatusNotFound, "ObjectNotFound", false},
+	} {
+		err := serviceError{status: test.status, code: test.code}
+		retry, gotErr := shouldRetry(ctx, nil, err)
+		assert.Equal(t, test.want, retry, test.code)
+		assert.Equal(t, error(err), gotErr)
+	}
+}

backend/pcloud/pcloud.go

@@ -1537,6 +1537,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if err != nil {
 			return fmt.Errorf("failed to make multipart upload for 0 length file: %w", err)
 		}
+		// Stop the form writer reading in if the request fails before sending it all
+		defer func() { _ = formReader.Close() }()
 
 		contentLength := overhead + size
 

backend/pikpak/helper.go

@@ -184,7 +184,7 @@ func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api
 }
 
 // getTask gets api.Task from API for the ID passed
-func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api.Task, err error) {
+func (f *Fs) getTask(ctx context.Context, ID string) (info *api.Task, err error) {
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   "/drive/v1/tasks/" + ID,
@@ -192,27 +192,48 @@ func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
-		if checkPhase {
-			if err == nil && info.Phase != api.PhaseTypeComplete {
-				// could be pending right after the task is created
-				return true, fmt.Errorf("%s (%s) is still in %s", info.Name, info.Type, info.Phase)
-			}
-		}
 		return f.shouldRetry(ctx, resp, err)
 	})
 	return
 }
 
-// waitTask waits for async tasks to be completed
+// waitTask waits up to taskTimeout for the async task ID to complete
 func (f *Fs) waitTask(ctx context.Context, ID string) (err error) {
-	time.Sleep(taskWaitTime)
-	if info, err := f.getTask(ctx, ID, true); err != nil {
-		if info == nil {
-			return fmt.Errorf("can't verify the task is completed: %q", ID)
+	start := time.Now()
+	sleep := taskWaitTime
+	for {
+		if err := sleepCtx(ctx, sleep); err != nil {
+			return err
 		}
-		return fmt.Errorf("can't verify the task is completed: %#v", info)
+		info, err := f.getTask(ctx, ID)
+		if err != nil {
+			return fmt.Errorf("can't verify the task %q is completed: %w", ID, err)
+		}
+		switch info.Phase {
+		case api.PhaseTypeComplete:
+			fs.Debugf(f, "task %q (%s) completed after %v", ID, info.Type, time.Since(start))
+			return nil
+		case api.PhaseTypeError:
+			return fmt.Errorf("task failed: %#v", info)
+		}
+		// Tasks can stay pending for tens of seconds when the server is busy
+		if time.Since(start) > taskTimeout {
+			return fmt.Errorf("can't verify the task is completed after %v: %#v", time.Since(start), info)
+		}
+		sleep = min(2*sleep, taskMaxSleep)
+	}
+}
+
+// sleepCtx sleeps for d or until ctx is cancelled
+func sleepCtx(ctx context.Context, d time.Duration) error {
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
 	}
-	return
 }
 
 // deleteTask remove a task having the specified ID

backend/pikpak/pikpak.go

@@ -72,6 +72,8 @@ const (
 	minSleep         = 100 * time.Millisecond
 	maxSleep         = 2 * time.Second
 	taskWaitTime     = 500 * time.Millisecond
+	taskMaxSleep     = 5 * time.Second
+	taskTimeout      = 5 * time.Minute
 	decayConstant    = 2 // bigger for slower decay, exponential
 	rootURL          = "https://api-drive.mypikpak.com"
 
@@ -1388,6 +1390,8 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
 	if err != nil {
 		return fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading in if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	contentLength := overhead + size
 	opts := rest.Opts{
@@ -1946,6 +1950,12 @@ func (o *Object) httpResponse(ctx context.Context, url, method string, options [
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
+		// client.Do does not fail a completed non-2xx response, so the
+		// HTML error page would be returned as the file. errorHandler
+		// closes the body; shouldRetry then retries retryErrorCodes.
+		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
+			err = errorHandler(res)
+		}
 		return o.fs.shouldRetry(ctx, res, err)
 	})
 	if err != nil {

backend/pikpak/pikpak_internal_test.go

@@ -0,0 +1,185 @@
+package pikpak
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync/atomic"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// closeTracker counts Close calls on a response body.
+type closeTracker struct {
+	io.ReadCloser
+	closed *atomic.Int32
+}
+
+func (b closeTracker) Close() error {
+	b.closed.Add(1)
+	return b.ReadCloser.Close()
+}
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+// newDownloadTestObject returns an object whose downloads are served by srv.
+// closed counts response bodies closed through the client.
+func newDownloadTestObject(t *testing.T, srv *httptest.Server, closed *atomic.Int32) *Object {
+	t.Helper()
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.LowLevelRetries = 3
+	client := *srv.Client()
+	base := client.Transport
+	if base == nil {
+		base = http.DefaultTransport
+	}
+	client.Transport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		res, err := base.RoundTrip(req)
+		if err == nil && res != nil && res.Body != nil {
+			res.Body = closeTracker{ReadCloser: res.Body, closed: closed}
+		}
+		return res, err
+	})
+	return &Object{
+		fs: &Fs{
+			client: &client,
+			pacer:  fs.NewPacer(ctx, &pacer.ZeroDelayCalculator{}),
+		},
+		size: 1,
+	}
+}
+
+// TestOpenHTTPStatus checks that a non-2xx download response is an error
+// (body closed, retriable statuses retried by the pacer) and that 200 and
+// 206 responses are still returned as file content.
+func TestOpenHTTPStatus(t *testing.T) {
+	const htmlBody = "<html><head><title>502 Bad Gateway</title></head><body>502</body></html>"
+	fileBody := []byte("pikpak-file-bytes-0123456789")
+
+	t.Run("502", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			w.Header().Set("Content-Type", "text/html")
+			w.WriteHeader(http.StatusBadGateway)
+			_, _ = io.WriteString(w, htmlBody)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		in, err := o.open(context.Background(), srv.URL)
+		if err == nil {
+			got, readErr := io.ReadAll(in)
+			require.NoError(t, readErr)
+			require.NoError(t, in.Close())
+			t.Fatalf("open returned the error page as file data (%d bytes): %q", len(got), got)
+		}
+		assert.Nil(t, in)
+		assert.ErrorContains(t, err, "502")
+		assert.NotContains(t, err.Error(), htmlBody)
+		assert.Equal(t, int32(3), hits.Load(), "502 should be retried by the pacer")
+		assert.Equal(t, hits.Load(), closed.Load(), "error response bodies should be closed")
+	})
+
+	t.Run("404", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			http.Error(w, "missing", http.StatusNotFound)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		in, err := o.open(context.Background(), srv.URL)
+		require.Error(t, err)
+		assert.Nil(t, in)
+		assert.Equal(t, int32(1), hits.Load(), "404 is not a retry status")
+		assert.Equal(t, hits.Load(), closed.Load())
+	})
+
+	t.Run("502 then 200", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if hits.Add(1) == 1 {
+				w.Header().Set("Content-Type", "text/html")
+				w.WriteHeader(http.StatusBadGateway)
+				_, _ = io.WriteString(w, htmlBody)
+				return
+			}
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write(fileBody)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		in, err := o.open(context.Background(), srv.URL)
+		require.NoError(t, err)
+		got, err := io.ReadAll(in)
+		require.NoError(t, err)
+		require.NoError(t, in.Close())
+		assert.Equal(t, fileBody, got)
+		assert.Equal(t, int32(2), hits.Load())
+		assert.Equal(t, int32(2), closed.Load())
+	})
+
+	t.Run("200", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			if r.Header.Get("Range") != "" {
+				t.Errorf("unexpected Range header %q", r.Header.Get("Range"))
+			}
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write(fileBody)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		o.size = int64(len(fileBody))
+		in, err := o.open(context.Background(), srv.URL)
+		require.NoError(t, err)
+		got, err := io.ReadAll(in)
+		require.NoError(t, err)
+		assert.Equal(t, int32(0), closed.Load(), "successful body stays open for the caller")
+		require.NoError(t, in.Close())
+		assert.Equal(t, fileBody, got)
+		assert.Equal(t, int32(1), hits.Load())
+		assert.Equal(t, int32(1), closed.Load())
+	})
+
+	t.Run("206", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			assert.Equal(t, "bytes=0-3", r.Header.Get("Range"))
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.Header().Set("Content-Range", "bytes 0-3/"+strconv.Itoa(len(fileBody)))
+			w.WriteHeader(http.StatusPartialContent)
+			_, _ = w.Write(fileBody[:4])
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		o.size = int64(len(fileBody))
+		in, err := o.open(context.Background(), srv.URL, &fs.RangeOption{Start: 0, End: 3})
+		require.NoError(t, err)
+		got, err := io.ReadAll(in)
+		require.NoError(t, err)
+		require.NoError(t, in.Close())
+		assert.Equal(t, fileBody[:4], got)
+		assert.Equal(t, int32(1), hits.Load())
+		assert.Equal(t, int32(1), closed.Load())
+	})
+}

backend/protondrive/account.go

@@ -0,0 +1,362 @@
+package protondrive
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"sync"
+
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/rclone/rclone/backend/protondrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// unlockMu is held while unlocking account keys. Unlocking is memory
+// hungry, so unlocks for different sessions run one at a time.
+var unlockMu sync.Mutex
+
+// address is one of the user's addresses with its unlocked keys.
+type address struct {
+	ID           string
+	Email        string
+	keyRing      *crypto.KeyRing // unlocked address keys in the server's order, primary first
+	primaryKeyID string          // ID of the first key of keyRing
+}
+
+// account is the signed in user's unlocked keys and addresses.
+type account struct {
+	userKR    *crypto.KeyRing // unlocked user keys
+	addresses []*address      // the primary address is first
+
+	mu      sync.Mutex
+	foreign map[string]*publicKeyLookup // public keys of other users by emailKey
+}
+
+// publicKeyLookup is a fetch of another user's public keys, done when
+// done is closed.
+type publicKeyLookup struct {
+	done    chan struct{}
+	keyRing *crypto.KeyRing
+	err     error
+}
+
+// loadAccount fetches the user and addresses of the session and unlocks
+// their keys with keyPassword.
+//
+// User keys which don't unlock are skipped, but an error is returned
+// if none unlocks or if the primary address has no key which unlocks.
+// Other addresses without a key which unlocks are left out.
+func loadAccount(ctx context.Context, s *session, keyPassword []byte) (*account, error) {
+	var user api.UserResponse
+	if _, err := s.call(ctx, &rest.Opts{Method: "GET", Path: "/core/v4/users"}, nil, &user); err != nil {
+		return nil, fmt.Errorf("load account: get user: %w", err)
+	}
+	var addrs api.AddressesResponse
+	// The SDK fetches only the first page of 50 addresses.
+	opts := &rest.Opts{Method: "GET", Path: "/core/v4/addresses", Parameters: url.Values{"Page": {"0"}, "PageSize": {"50"}}}
+	if _, err := s.call(ctx, opts, nil, &addrs); err != nil {
+		return nil, fmt.Errorf("load account: get addresses: %w", err)
+	}
+	if len(addrs.Addresses) == 0 {
+		return nil, errors.New("load account: the account has no addresses")
+	}
+
+	unlockMu.Lock()
+	defer unlockMu.Unlock()
+	if s.unlockHook != nil {
+		s.unlockHook()
+	}
+	a := &account{foreign: make(map[string]*publicKeyLookup)}
+	var err error
+	a.userKR, err = unlockUserKeys(user.User.Keys, keyPassword)
+	if err != nil {
+		return nil, fmt.Errorf("load account: unlock user keys: %w", err)
+	}
+	var verifyTime int64
+	if t := s.serverTime(); !t.IsZero() {
+		verifyTime = t.Unix()
+	}
+	for i, addr := range addrs.Addresses {
+		kr, primaryKeyID, err := unlockAddressKeys(addr.Keys, keyPassword, a.userKR, verifyTime)
+		if err != nil {
+			if i == 0 {
+				a.close()
+				return nil, fmt.Errorf("load account: unlock keys of primary address: %w", err)
+			}
+			fs.Debugf(nil, "protondrive: skipping address %s: %v", addr.ID, err)
+			continue
+		}
+		a.addresses = append(a.addresses, &address{ID: addr.ID, Email: addr.Email, keyRing: kr, primaryKeyID: primaryKeyID})
+	}
+	return a, nil
+}
+
+// unlockUserKeys returns a key ring of the active user keys which
+// unlock with keyPassword.
+func unlockUserKeys(keys []api.UserKey, keyPassword []byte) (*crypto.KeyRing, error) {
+	kr, err := crypto.NewKeyRing(nil)
+	if err != nil {
+		return nil, err
+	}
+	var lastErr error
+	for _, k := range keys {
+		if k.Active == 0 {
+			continue
+		}
+		unlocked, err := unlockKey(k.PrivateKey, keyPassword)
+		if err == nil {
+			if err = kr.AddKey(unlocked); err != nil {
+				unlocked.ClearPrivateParams()
+			}
+		}
+		if err != nil {
+			fs.Debugf(nil, "protondrive: can't unlock user key %s: %v", k.ID, err)
+			lastErr = err
+		}
+	}
+	if kr.CountEntities() == 0 {
+		if lastErr == nil {
+			return nil, errors.New("no active user key")
+		}
+		return nil, fmt.Errorf("no user key unlocks, is the password right?: %w", lastErr)
+	}
+	return kr, nil
+}
+
+// unlockAddressKeys returns a key ring of the active address keys
+// which unlock, in the order given, and the ID of the first of them.
+//
+// A key with a Token is locked with the passphrase in it, encrypted to
+// and signed by the user keys in userKR; the signature is checked as
+// at verifyTime (Unix seconds, or now if zero). A key without a Token
+// is locked with keyPassword.
+func unlockAddressKeys(keys []api.AddressKey, keyPassword []byte, userKR *crypto.KeyRing, verifyTime int64) (kr *crypto.KeyRing, firstID string, err error) {
+	kr, err = crypto.NewKeyRing(nil)
+	if err != nil {
+		return nil, "", err
+	}
+	var lastErr error
+	for _, k := range keys {
+		if k.Active == 0 {
+			continue
+		}
+		unlocked, err := unlockAddressKey(k, keyPassword, userKR, verifyTime)
+		if err == nil {
+			if err = kr.AddKey(unlocked); err != nil {
+				unlocked.ClearPrivateParams()
+			}
+		}
+		if err != nil {
+			fs.Debugf(nil, "protondrive: can't unlock address key %s: %v", k.ID, err)
+			lastErr = err
+			continue
+		}
+		if firstID == "" {
+			firstID = k.ID
+		}
+	}
+	if kr.CountEntities() == 0 {
+		if lastErr == nil {
+			return nil, "", errors.New("no active address key")
+		}
+		return nil, "", fmt.Errorf("no address key unlocks: %w", lastErr)
+	}
+	return kr, firstID, nil
+}
+
+// unlockAddressKey unlocks one address key as unlockAddressKeys does.
+func unlockAddressKey(k api.AddressKey, keyPassword []byte, userKR *crypto.KeyRing, verifyTime int64) (*crypto.Key, error) {
+	if k.Token == "" {
+		return unlockKey(k.PrivateKey, keyPassword)
+	}
+	dh, err := crypto.PGP().Decryption().DecryptionKeys(userKR).New()
+	if err != nil {
+		return nil, err
+	}
+	res, err := dh.Decrypt([]byte(k.Token), crypto.Armor)
+	if err != nil {
+		return nil, fmt.Errorf("decrypt token: %w", err)
+	}
+	passphrase := res.Bytes()
+	defer clear(passphrase)
+	vb := crypto.PGP().Verify().VerificationKeys(userKR)
+	if verifyTime != 0 {
+		vb = vb.VerifyTime(verifyTime)
+	}
+	vh, err := vb.New()
+	if err != nil {
+		return nil, err
+	}
+	// As the SDK does, refuse a token without a valid signature.
+	verified, err := vh.VerifyDetached(passphrase, []byte(k.Signature), crypto.Armor)
+	if err == nil {
+		err = verified.SignatureError()
+	}
+	if err != nil {
+		return nil, fmt.Errorf("verify token: %w", err)
+	}
+	return unlockKey(k.PrivateKey, passphrase)
+}
+
+// unlockKey returns the armored private key unlocked with passphrase.
+func unlockKey(armored string, passphrase []byte) (*crypto.Key, error) {
+	locked, err := crypto.NewKeyFromArmored(armored)
+	if err != nil {
+		return nil, err
+	}
+	return locked.Unlock(passphrase)
+}
+
+// signingKeyRing returns a key ring holding only the first key of the
+// address, which the SDK calls the primary address key.
+func (a *address) signingKeyRing() (*crypto.KeyRing, error) {
+	return crypto.NewKeyRing(a.keyRing.GetKeys()[0])
+}
+
+// primary returns the primary address.
+func (a *account) primary() *address {
+	return a.addresses[0]
+}
+
+// byID returns the address with the ID id, or nil.
+func (a *account) byID(id string) *address {
+	for _, addr := range a.addresses {
+		if addr.ID == id {
+			return addr
+		}
+	}
+	return nil
+}
+
+// emailKey returns email with ASCII letters lowercased, which is how
+// emails are compared: Proton treats addresses case-insensitively, but
+// Unicode case folding would let a non-ASCII letter, such as the Kelvin
+// sign, stand for an ASCII one.
+func emailKey(email string) string {
+	b := []byte(email)
+	for i, c := range b {
+		if 'A' <= c && c <= 'Z' {
+			b[i] = c + 'a' - 'A'
+		}
+	}
+	return string(b)
+}
+
+// byEmail returns the address email, compared as emailKey does, or
+// nil.
+func (a *account) byEmail(email string) *address {
+	key := emailKey(email)
+	for _, addr := range a.addresses {
+		if emailKey(addr.Email) == key {
+			return addr
+		}
+	}
+	return nil
+}
+
+// owns reports whether email is one of the account's addresses whose
+// keys it holds, compared as emailKey does.
+func (a *account) owns(email string) bool {
+	return a.byEmail(email) != nil
+}
+
+// apiCaller makes an API call with the semantics of session.call,
+// without returning the response; client.apiCall is one which paces
+// and retries.
+type apiCaller func(ctx context.Context, opts *rest.Opts, request, response any) error
+
+// publicKeys returns the public keys of the address email, which the
+// caller must not modify.
+//
+// The keys of the user's own addresses, in any ASCII case, come from
+// the account, never the server. Other users' keys are fetched once
+// per emailKey and kept for the life of the account; an email without
+// a Proton address has an empty key ring.
+func (a *account) publicKeys(ctx context.Context, call apiCaller, email string) (*crypto.KeyRing, error) {
+	if addr := a.byEmail(email); addr != nil {
+		return publicKeyRing(addr.keyRing)
+	}
+	key := emailKey(email)
+	a.mu.Lock()
+	l := a.foreign[key]
+	if l == nil {
+		l = &publicKeyLookup{done: make(chan struct{})}
+		a.foreign[key] = l
+		a.mu.Unlock()
+		l.keyRing, l.err = fetchPublicKeys(ctx, call, email)
+		if l.err != nil {
+			// Let the next caller try again.
+			a.mu.Lock()
+			delete(a.foreign, key)
+			a.mu.Unlock()
+		}
+		close(l.done)
+	} else {
+		a.mu.Unlock()
+	}
+	select {
+	case <-l.done:
+		return l.keyRing, l.err
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+}
+
+// publicKeyRing returns the public parts of the keys in kr.
+func publicKeyRing(kr *crypto.KeyRing) (*crypto.KeyRing, error) {
+	pub, err := crypto.NewKeyRing(nil)
+	if err != nil {
+		return nil, err
+	}
+	for _, key := range kr.GetKeys() {
+		pk, err := key.ToPublic()
+		if err != nil {
+			return nil, err
+		}
+		if err := pub.AddKey(pk); err != nil {
+			return nil, err
+		}
+	}
+	return pub, nil
+}
+
+// fetchPublicKeys fetches the public keys of the Proton address email
+// with call.
+func fetchPublicKeys(ctx context.Context, call apiCaller, email string) (*crypto.KeyRing, error) {
+	kr, err := crypto.NewKeyRing(nil)
+	if err != nil {
+		return nil, err
+	}
+	var out api.PublicKeysResponse
+	opts := &rest.Opts{Method: "GET", Path: "/core/v4/keys/all", Parameters: url.Values{"Email": {email}, "InternalOnly": {"1"}}}
+	err = call(ctx, opts, nil, &out)
+	var apiErr *api.Error
+	if errors.As(err, &apiErr) && (apiErr.Code == api.CodeAddressMissing || apiErr.Code == api.CodeDomainExternal) {
+		return kr, nil
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get public keys of %s: %w", email, err)
+	}
+	for _, k := range out.Address.Keys {
+		key, err := crypto.NewKeyFromArmored(k.PublicKey)
+		if err == nil {
+			err = kr.AddKey(key)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("public key of %s: %w", email, err)
+		}
+	}
+	return kr, nil
+}
+
+// close clears the private key material of the account.
+func (a *account) close() {
+	if a.userKR != nil {
+		a.userKR.ClearPrivateParams()
+	}
+	for _, addr := range a.addresses {
+		addr.keyRing.ClearPrivateParams()
+	}
+}

backend/protondrive/account_internal_test.go

@@ -0,0 +1,300 @@
+package protondrive
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newAccountSession returns a session for a new session of u.
+func newAccountSession(srv *protontest.Server, u *protontest.User) *session {
+	tokens := srv.NewSession(u)
+	store := &memoryStore{creds: credentials{
+		UID:          tokens.UID,
+		AccessToken:  tokens.AccessToken,
+		RefreshToken: tokens.RefreshToken,
+		KeyPassword:  u.KeyPassword(),
+	}}
+	return newSession(srv.Client(), srv.URL(), testAppVersion, store)
+}
+
+// sessionCaller returns an apiCaller making unpaced calls with s.
+func sessionCaller(s *session) apiCaller {
+	return func(ctx context.Context, opts *rest.Opts, request, response any) error {
+		_, err := s.call(ctx, opts, request, response)
+		return err
+	}
+}
+
+// decryptWith decrypts the armored message with kr.
+func decryptWith(t *testing.T, kr *crypto.KeyRing, armored string) string {
+	t.Helper()
+	dh, err := crypto.PGP().Decryption().DecryptionKeys(kr).New()
+	require.NoError(t, err)
+	res, err := dh.Decrypt([]byte(armored), crypto.Armor)
+	require.NoError(t, err)
+	return string(res.Bytes())
+}
+
+func TestLoadAccount(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	u.AddAddress("alias@proton.test")
+	s := newAccountSession(srv, u)
+
+	a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
+	require.NoError(t, err)
+	defer a.close()
+
+	require.NotNil(t, a.userKR)
+	require.Len(t, a.addresses, 2)
+	primary := a.primary()
+	require.NotNil(t, primary)
+	assert.Equal(t, u.Email(), primary.Email)
+	assert.NotEmpty(t, primary.ID)
+	assert.Same(t, primary, a.byID(primary.ID))
+	assert.Same(t, primary, a.byEmail(u.Email()))
+	assert.Nil(t, a.byID("unknown"))
+	assert.Nil(t, a.byEmail("unknown@proton.test"))
+
+	for _, email := range []string{u.Email(), "alias@proton.test"} {
+		addr := a.byEmail(email)
+		require.NotNil(t, addr, email)
+		assert.Equal(t, "secret for "+email, decryptWith(t, addr.keyRing, u.EncryptToAddress(email, []byte("secret for "+email))))
+	}
+	// The address keys are distinct.
+	dh, err := crypto.PGP().Decryption().DecryptionKeys(primary.keyRing).New()
+	require.NoError(t, err)
+	_, err = dh.Decrypt([]byte(u.EncryptToAddress("alias@proton.test", []byte("x"))), crypto.Armor)
+	assert.Error(t, err, "primary address key decrypted a message for the alias")
+}
+
+func TestLoadAccountTwoPassword(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddTwoPasswordUser("user", "login", "mailbox")
+	want, err := computeKeyPassword([]byte("mailbox"), u.KeySalt)
+	require.NoError(t, err)
+	assert.Equal(t, string(want), u.KeyPassword())
+
+	a, err := loadAccount(context.Background(), newAccountSession(srv, u), want)
+	require.NoError(t, err)
+	a.close()
+}
+
+func TestLoadAccountWrongKeyPassword(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	s := newAccountSession(srv, u)
+
+	for _, keyPassword := range [][]byte{[]byte("wrong"), nil} {
+		a, err := loadAccount(context.Background(), s, keyPassword)
+		require.Error(t, err)
+		assert.Nil(t, a)
+		assert.Contains(t, err.Error(), "unlock user keys")
+	}
+}
+
+func TestLoadAccountRotatedUserKey(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	u.AddUserKey()
+
+	// The old user key no longer unlocks with the key password, which
+	// must not stop the account loading.
+	a, err := loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
+	require.NoError(t, err)
+	defer a.close()
+	assert.Equal(t, "msg", decryptWith(t, a.primary().keyRing, u.EncryptToAddress(u.Email(), []byte("msg"))))
+}
+
+func TestLoadAccountBadAddressKeySignature(t *testing.T) {
+	t.Parallel()
+	for _, test := range []struct {
+		name  string
+		empty bool
+	}{
+		{"impostor", false},
+		{"empty", true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+			srv := protontest.New(t)
+
+			// On the primary address the account doesn't load.
+			u := srv.AddUser("user", "pass")
+			srv.CorruptAddressKeySignature(u, u.Email(), test.empty)
+			a, err := loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
+			require.Error(t, err)
+			assert.Nil(t, a)
+			assert.Contains(t, err.Error(), "verify token")
+
+			// A secondary address is left out.
+			u = srv.AddUser("user2", "pass")
+			u.AddAddress("alias@proton.test")
+			srv.CorruptAddressKeySignature(u, "alias@proton.test", test.empty)
+			a, err = loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
+			require.NoError(t, err)
+			defer a.close()
+			assert.Nil(t, a.byEmail("alias@proton.test"))
+			assert.Len(t, a.addresses, 1)
+			assert.Equal(t, u.Email(), a.primary().Email)
+		})
+	}
+}
+
+func TestLoadAccountSerialised(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	s := newAccountSession(srv, u)
+
+	var active, peak, calls atomic.Int32
+	s.unlockHook = func() {
+		n := active.Add(1)
+		for {
+			p := peak.Load()
+			if n <= p || peak.CompareAndSwap(p, n) {
+				break
+			}
+		}
+		calls.Add(1)
+		time.Sleep(5 * time.Millisecond)
+		active.Add(-1)
+	}
+
+	var wg sync.WaitGroup
+	for range 16 {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
+			if assert.NoError(t, err) {
+				a.close()
+			}
+		}()
+	}
+	wg.Wait()
+	assert.Equal(t, int32(16), calls.Load())
+	assert.Equal(t, int32(1), peak.Load(), "unlocks overlapped")
+}
+
+func TestAccountClose(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	a, err := loadAccount(context.Background(), newAccountSession(srv, u), []byte(u.KeyPassword()))
+	require.NoError(t, err)
+	a.close()
+
+	for _, kr := range []*crypto.KeyRing{a.userKR, a.primary().keyRing} {
+		for _, key := range kr.GetKeys() {
+			assert.False(t, key.IsPrivate(), "private key material left after close")
+		}
+	}
+}
+
+func TestPublicKeysCached(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	other := srv.AddUser("other", "pass")
+	s := newAccountSession(srv, u)
+	a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
+	require.NoError(t, err)
+	defer a.close()
+	otherAccount, err := loadAccount(context.Background(), newAccountSession(srv, other), []byte(other.KeyPassword()))
+	require.NoError(t, err)
+	defer otherAccount.close()
+
+	const route = "GET /core/v4/keys/all"
+	srv.ResetRequests()
+	for range 5 {
+		kr, err := a.publicKeys(context.Background(), sessionCaller(s), other.Email())
+		require.NoError(t, err)
+		require.Equal(t, 1, kr.CountEntities())
+		assert.False(t, kr.GetKeys()[0].IsPrivate())
+		// A message encrypted to the key decrypts with other's address key.
+		eh, err := crypto.PGP().Encryption().Recipients(kr).New()
+		require.NoError(t, err)
+		msg, err := eh.Encrypt([]byte("hello"))
+		require.NoError(t, err)
+		armored, err := msg.ArmorBytes()
+		require.NoError(t, err)
+		assert.Equal(t, "hello", decryptWith(t, otherAccount.primary().keyRing, string(armored)))
+	}
+	assert.Equal(t, 1, srv.Requests()[route])
+
+	srv.ResetRequests()
+	for range 5 {
+		kr, err := a.publicKeys(context.Background(), sessionCaller(s), u.Email())
+		require.NoError(t, err)
+		require.Equal(t, 1, kr.CountEntities())
+		assert.False(t, kr.GetKeys()[0].IsPrivate())
+	}
+	assert.Zero(t, srv.Requests()[route])
+
+	// An email without a Proton account has no keys, and that is cached too.
+	for range 3 {
+		kr, err := a.publicKeys(context.Background(), sessionCaller(s), "nobody@example.com")
+		require.NoError(t, err)
+		assert.Zero(t, kr.CountEntities())
+	}
+	assert.Equal(t, 1, srv.Requests()[route])
+
+	// Case variants share a lookup, and an own address in any case
+	// never reaches the server.
+	srv.ResetRequests()
+	for _, email := range []string{"OTHER@proton.test", "Other@Proton.Test", strings.ToUpper(u.Email())} {
+		_, err := a.publicKeys(context.Background(), sessionCaller(s), email)
+		require.NoError(t, err)
+	}
+	assert.Zero(t, srv.Requests()[route])
+}
+
+func TestPublicKeysErrorNotCached(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	other := srv.AddUser("other", "pass")
+	s := newAccountSession(srv, u)
+	a, err := loadAccount(context.Background(), s, []byte(u.KeyPassword()))
+	require.NoError(t, err)
+	defer a.close()
+
+	srv.Inject(protontest.Fault{Pattern: "/core/v4/keys/all", Times: 1, Status: 500})
+	_, err = a.publicKeys(context.Background(), sessionCaller(s), other.Email())
+	require.Error(t, err)
+	kr, err := a.publicKeys(context.Background(), sessionCaller(s), other.Email())
+	require.NoError(t, err)
+	assert.Equal(t, 1, kr.CountEntities())
+}
+
+func TestSameEmail(t *testing.T) {
+	for _, tc := range []struct {
+		a, b string
+		same bool
+	}{
+		{"user@proton.me", "user@proton.me", true},
+		{"User@Proton.Me", "uSER@pROTON.mE", true},
+		{"user@proton.me", "user@proton.mee", false},
+		// Unicode case folding would match these to ASCII letters.
+		{"kim@proton.me", "\u212aim@proton.me", false},
+		{"sam@proton.me", "\u017fam@proton.me", false},
+		{"élan@proton.me", "Élan@proton.me", false},
+	} {
+		assert.Equal(t, tc.same, emailKey(tc.a) == emailKey(tc.b), "%q %q", tc.a, tc.b)
+	}
+}

backend/protondrive/api/types.go

@@ -0,0 +1,661 @@
+// Package api has the type definitions for the Proton Drive API.
+package api
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// Response codes returned in the Code field of every API response.
+//
+// The names follow ErrorCode in the Proton Drive SDK
+// (client/js/src/internal/apiService/errorCodes.ts) unless noted.
+const (
+	CodeOK                                             = 1000
+	CodeOKMany                                         = 1001
+	CodeOKAsync                                        = 1002
+	CodeInvalidRequirements                            = 2000
+	CodeInvalidValue                                   = 2001
+	CodeNotEnoughPermissions                           = 2011
+	CodeNotEnoughPermissionsToGrantPermissions         = 2026
+	CodeAlreadyExists                                  = 2500
+	CodeNotExists                                      = 2501
+	CodeInsufficientQuota                              = 200001
+	CodeInsufficientSpace                              = 200002
+	CodeMaxFileSizeForFreeUser                         = 200003
+	CodeMaxPublicEditModeForFreeUser                   = 200004
+	CodeInsufficientVolumeQuota                        = 200100
+	CodeInsufficientDeviceQuota                        = 200101
+	CodeAlreadyMemberOfShareInVolumeWithAnotherAddress = 200201
+	CodeTooManyChildren                                = 200300
+	CodeNestingTooDeep                                 = 200301
+	CodeInsufficientInvitationQuota                    = 200600
+	CodeInsufficientShareQuota                         = 200601
+	CodeInsufficientShareJoinedQuota                   = 200602
+	CodeInsufficientBookmarksQuota                     = 200800
+
+	// CodeFeatureBlocked is returned when a feature is disabled,
+	// rate-limited or blocked because of abuse (driveTypes.ts).
+	CodeFeatureBlocked = 2028
+	// CodeAppVersionMissing is returned when the request has no app
+	// version (go-proton-api AppVersionMissingCode).
+	CodeAppVersionMissing = 5001
+	// CodeAppVersionBad is returned when the app version is no longer
+	// accepted: "You need to update this app" (coreTypes.ts,
+	// go-proton-api AppVersionBadCode).
+	CodeAppVersionBad = 5003
+	// CodeInvalidSection is returned when a request header such as
+	// x-pm-appversion is malformed.
+	CodeInvalidSection = 2064
+	// CodePasswordWrong is returned when the login credentials are
+	// incorrect (go-proton-api PasswordWrong).
+	CodePasswordWrong = 8002
+	// CodeHumanVerification is returned when a CAPTCHA must be solved.
+	CodeHumanVerification = 9001
+	// CodeRefreshTokenInvalid is returned when the refresh token is
+	// rejected and the session must be established again.
+	CodeRefreshTokenInvalid = 10013
+	// CodeAddressMissing is returned by GET /core/v4/keys/all when no
+	// Proton address has the email (account accountApi.ts).
+	CodeAddressMissing = 33102
+	// CodeDomainExternal is returned by GET /core/v4/keys/all when the
+	// email's domain is not hosted by Proton (account accountApi.ts).
+	CodeDomainExternal = 33103
+)
+
+// Error is the body of an unsuccessful API response.
+type Error struct {
+	Status  int             `json:"-"`                 // HTTP status of the response
+	Code    int             `json:"Code"`              // Proton response code
+	Message string          `json:"Error"`             // human readable message
+	Details json.RawMessage `json:"Details,omitempty"` // request specific details
+}
+
+// Error returns the message with its code and HTTP status.
+func (e *Error) Error() string {
+	return fmt.Sprintf("%s (Code=%d, Status=%d)", e.Message, e.Code, e.Status)
+}
+
+// AuthRefreshRequest is the body of POST /auth/v4/refresh.
+type AuthRefreshRequest struct {
+	ResponseType string `json:"ResponseType"` // always "token"
+	GrantType    string `json:"GrantType"`    // always "refresh_token"
+	RefreshToken string `json:"RefreshToken"`
+}
+
+// AuthRefreshResponse is the response to POST /auth/v4/refresh.
+type AuthRefreshResponse struct {
+	Code         int    `json:"Code"`
+	UID          string `json:"UID"`
+	AccessToken  string `json:"AccessToken"`
+	RefreshToken string `json:"RefreshToken"`
+}
+
+// AuthInfoRequest is the body of POST /core/v4/auth/info.
+type AuthInfoRequest struct {
+	Intent   string `json:"Intent"` // always "Proton"
+	Username string `json:"Username"`
+}
+
+// AuthInfoResponse is the response to POST /core/v4/auth/info: the
+// server's SRP challenge.
+type AuthInfoResponse struct {
+	Code            int    `json:"Code"`
+	Version         int    `json:"Version"`         // SRP password hashing version
+	Modulus         string `json:"Modulus"`         // signed base64 SRP modulus
+	ServerEphemeral string `json:"ServerEphemeral"` // base64
+	Salt            string `json:"Salt"`            // base64 SRP salt
+	SRPSession      string `json:"SRPSession"`
+	Username        string `json:"Username"`
+}
+
+// AuthRequest is the body of POST /core/v4/auth.
+type AuthRequest struct {
+	Username          string            `json:"Username"`
+	ClientEphemeral   string            `json:"ClientEphemeral"` // base64
+	ClientProof       string            `json:"ClientProof"`     // base64
+	SRPSession        string            `json:"SRPSession"`
+	PersistentCookies int               `json:"PersistentCookies"`
+	Payload           map[string]string `json:"Payload"`
+}
+
+// Values of TwoFAInfo.Enabled, which may be combined.
+const (
+	TwoFATOTP  = 1
+	TwoFAFIDO2 = 2
+)
+
+// Values of AuthResponse.PasswordMode.
+const (
+	PasswordModeOne = 1 // the login password also unlocks the keys
+	PasswordModeTwo = 2 // a separate mailbox password unlocks the keys
+)
+
+// TwoFAInfo describes the second factors an account requires.
+type TwoFAInfo struct {
+	Enabled int `json:"Enabled"` // bitmask of TwoFATOTP and TwoFAFIDO2
+}
+
+// AuthResponse is the response to POST /core/v4/auth.
+type AuthResponse struct {
+	Code         int       `json:"Code"`
+	UID          string    `json:"UID"`
+	UserID       string    `json:"UserID"`
+	AccessToken  string    `json:"AccessToken"`
+	RefreshToken string    `json:"RefreshToken"`
+	TokenType    string    `json:"TokenType"`
+	ServerProof  string    `json:"ServerProof"` // base64
+	Scopes       []string  `json:"Scopes"`
+	PasswordMode int       `json:"PasswordMode"`
+	TwoFA        TwoFAInfo `json:"2FA"`
+}
+
+// Auth2FARequest is the body of POST /core/v4/auth/2fa.
+type Auth2FARequest struct {
+	TwoFactorCode string `json:"TwoFactorCode"`
+}
+
+// KeySalt is the salt of one user key's password.
+type KeySalt struct {
+	ID      string `json:"ID"`      // user key ID
+	KeySalt string `json:"KeySalt"` // base64
+}
+
+// KeySaltsResponse is the response to GET /core/v4/keys/salts.
+type KeySaltsResponse struct {
+	Code     int       `json:"Code"`
+	KeySalts []KeySalt `json:"KeySalts"`
+}
+
+// ForkInitResponse is the response to GET /auth/v4/sessions/forks.
+type ForkInitResponse struct {
+	Code     int    `json:"Code"`
+	Selector string `json:"Selector"`
+	UserCode string `json:"UserCode"`
+}
+
+// ForkStatusResponse is the response to GET
+// /auth/v4/sessions/forks/{selector} once the user has signed in.
+type ForkStatusResponse struct {
+	Code         int    `json:"Code"`
+	Payload      string `json:"Payload"` // base64 AES-256-GCM encrypted key password
+	UID          string `json:"UID"`
+	AccessToken  string `json:"AccessToken"`
+	RefreshToken string `json:"RefreshToken"`
+}
+
+// UserKey is one of the user's keys.
+type UserKey struct {
+	ID         string `json:"ID"`
+	Version    int    `json:"Version"`
+	PrivateKey string `json:"PrivateKey"` // armored, locked with the key password
+	Primary    int    `json:"Primary"`    // 1 for the primary key
+	Active     int    `json:"Active"`     // 0 for a key which must not be unlocked
+}
+
+// User is the account of the signed in user.
+type User struct {
+	ID    string    `json:"ID"`
+	Name  string    `json:"Name"`
+	Email string    `json:"Email"`
+	Keys  []UserKey `json:"Keys"`
+
+	UsedSpace int64 `json:"UsedSpace"` // bytes used by all products
+	MaxSpace  int64 `json:"MaxSpace"`  // bytes of storage all products share
+}
+
+// UserResponse is the response to GET /core/v4/users.
+type UserResponse struct {
+	Code int  `json:"Code"`
+	User User `json:"User"`
+}
+
+// AddressKey is one of an address's keys.
+type AddressKey struct {
+	ID         string `json:"ID"`
+	Version    int    `json:"Version"`
+	PrivateKey string `json:"PrivateKey"` // armored, locked with the passphrase in Token
+	// Token is the armored key passphrase encrypted to the user key,
+	// or empty for a legacy key locked with the key password.
+	Token     string `json:"Token"`
+	Signature string `json:"Signature"` // armored detached signature of the passphrase by the user key
+	Primary   int    `json:"Primary"`   // 1 for the primary key
+	Active    int    `json:"Active"`    // 0 for a key which must not be unlocked
+	Flags     int    `json:"Flags"`     // bitmap of key usages
+}
+
+// Address is one of the user's email addresses.
+type Address struct {
+	ID       string       `json:"ID"`
+	Email    string       `json:"Email"`
+	Status   int          `json:"Status"`   // 0 disabled, 1 enabled, 2 deleting
+	Type     int          `json:"Type"`     // 1 original, 2 alias, 3 custom, 4 premium, 5 external
+	Priority int          `json:"Priority"` // order of the addresses, lowest first
+	Keys     []AddressKey `json:"Keys"`
+}
+
+// AddressesResponse is the response to GET /core/v4/addresses.
+type AddressesResponse struct {
+	Code      int       `json:"Code"`
+	Addresses []Address `json:"Addresses"`
+}
+
+// PublicKey is a public key of an address.
+type PublicKey struct {
+	PublicKey string `json:"PublicKey"` // armored
+	Primary   int    `json:"Primary"`   // 1 for the primary key
+	Flags     int    `json:"Flags"`     // bitmap of key usages
+	Source    int    `json:"Source"`    // 0 for internal keys
+}
+
+// PublicKeysResponse is the response to GET /core/v4/keys/all.
+type PublicKeysResponse struct {
+	Code    int `json:"Code"`
+	Address struct {
+		Keys []PublicKey `json:"Keys"`
+	} `json:"Address"`
+}
+
+// Link types, as Link.Type encodes them.
+const (
+	LinkTypeFolder = 1
+	LinkTypeFile   = 2
+	LinkTypeAlbum  = 3
+)
+
+// Link states, as Link.State encodes them.
+const (
+	LinkStateDraft   = 0
+	LinkStateActive  = 1
+	LinkStateTrashed = 2
+)
+
+// Volume is the summary of a volume in PrimaryRootShareResponseDto.
+type Volume struct {
+	VolumeID  string `json:"VolumeID"`
+	UsedSpace int64  `json:"UsedSpace"`
+}
+
+// Share is a share with its key, as ShareDto.
+type Share struct {
+	ShareID             string `json:"ShareID"`
+	CreatorEmail        string `json:"CreatorEmail"`
+	Key                 string `json:"Key"`                 // armored share key, locked with the passphrase
+	Passphrase          string `json:"Passphrase"`          // armored, encrypted to the address key
+	PassphraseSignature string `json:"PassphraseSignature"` // armored detached signature by the creator
+	AddressID           string `json:"AddressID"`           // address whose key decrypts the passphrase
+}
+
+// Link is the metadata every node has, as LinkDto. Strings the API
+// sends as null are empty.
+type Link struct {
+	LinkID                  string `json:"LinkID"`
+	Type                    int    `json:"Type"`         // LinkTypeFolder, LinkTypeFile or LinkTypeAlbum
+	ParentLinkID            string `json:"ParentLinkID"` // empty for a root
+	State                   int    `json:"State"`        // LinkStateDraft, LinkStateActive or LinkStateTrashed
+	CreateTime              int64  `json:"CreateTime"`   // Unix seconds
+	ModifyTime              int64  `json:"ModifyTime"`   // Unix seconds
+	TrashTime               int64  `json:"TrashTime"`    // Unix seconds, 0 unless trashed
+	Name                    string `json:"Name"`         // armored, encrypted to the parent key
+	NameHash                string `json:"NameHash"`     // lower-case hex lookup hash; empty for a root
+	NodeKey                 string `json:"NodeKey"`      // armored, locked with the node passphrase
+	NodePassphrase          string `json:"NodePassphrase"`
+	NodePassphraseSignature string `json:"NodePassphraseSignature"`
+	SignatureEmail          string `json:"SignatureEmail"`     // signer of the key and attributes; empty if anonymous
+	NameSignatureEmail      string `json:"NameSignatureEmail"` // signer of the name; empty if anonymous
+	ThirdParty              bool   `json:"ThirdParty"`
+	Sdk                     bool   `json:"Sdk"`
+}
+
+// Folder is the folder part of a node's metadata, as FolderDto.
+type Folder struct {
+	NodeHashKey string `json:"NodeHashKey"` // armored, encrypted to the node key
+	XAttr       string `json:"XAttr"`       // armored extended attributes; may be empty
+	IsImported  bool   `json:"IsImported"`
+}
+
+// ActiveRevision is the summary of a file's active revision, as
+// ActiveRevisionDto.
+type ActiveRevision struct {
+	RevisionID        string `json:"RevisionID"`
+	CreateTime        int64  `json:"CreateTime"` // Unix seconds
+	EncryptedSize     int64  `json:"EncryptedSize"`
+	ChecksumVerified  bool   `json:"ChecksumVerified"`
+	ManifestSignature string `json:"ManifestSignature"` // armored
+	XAttr             string `json:"XAttr"`             // armored extended attributes; may be empty
+	SignatureEmail    string `json:"SignatureEmail"`    // signer of the revision; empty if anonymous
+	IsImported        bool   `json:"IsImported"`
+	ThirdParty        bool   `json:"ThirdParty"`
+	Sdk               bool   `json:"Sdk"`
+}
+
+// File is the file part of a node's metadata, as FileDto.
+type File struct {
+	ActiveRevision            *ActiveRevision `json:"ActiveRevision"` // nil for a draft
+	TotalEncryptedSize        int64           `json:"TotalEncryptedSize"`
+	ContentKeyPacket          string          `json:"ContentKeyPacket"` // base64
+	MediaType                 string          `json:"MediaType"`
+	ContentKeyPacketSignature string          `json:"ContentKeyPacketSignature"` // armored; may be empty
+}
+
+// LinkDetails is a node's metadata, as FileDetailsDto and
+// FolderDetailsDto.
+type LinkDetails struct {
+	Link   Link    `json:"Link"`
+	Folder *Folder `json:"Folder"` // set for a folder
+	File   *File   `json:"File"`   // set for a file
+}
+
+// MyFilesResponse is the response to GET /drive/v2/shares/my-files, as
+// PrimaryRootShareResponseDto.
+type MyFilesResponse struct {
+	Code   int         `json:"Code"`
+	Volume Volume      `json:"Volume"`
+	Share  Share       `json:"Share"`
+	Link   LinkDetails `json:"Link"` // the root folder
+}
+
+// CreateVolumeRequest is the body of POST /drive/volumes, as
+// CreateVolumeRequestDto without the deprecated names.
+type CreateVolumeRequest struct {
+	AddressID                 string `json:"AddressID"`
+	AddressKeyID              string `json:"AddressKeyID"` // the primary key of AddressID
+	ShareKey                  string `json:"ShareKey"`
+	SharePassphrase           string `json:"SharePassphrase"`
+	SharePassphraseSignature  string `json:"SharePassphraseSignature"`
+	FolderName                string `json:"FolderName"`
+	FolderKey                 string `json:"FolderKey"`
+	FolderPassphrase          string `json:"FolderPassphrase"`
+	FolderPassphraseSignature string `json:"FolderPassphraseSignature"`
+	FolderHashKey             string `json:"FolderHashKey"`
+}
+
+// ShareReference identifies the main share of a volume and its root,
+// as ShareReferenceResponseDto.
+type ShareReference struct {
+	ShareID string `json:"ShareID"`
+	LinkID  string `json:"LinkID"`
+}
+
+// VolumeDetails is a volume, as VolumeResponseDto.
+type VolumeDetails struct {
+	ID        string         `json:"ID"`
+	VolumeID  string         `json:"VolumeID"`
+	UsedSpace int64          `json:"UsedSpace"`
+	State     int            `json:"State"`
+	Share     ShareReference `json:"Share"`
+}
+
+// VolumeResponse is the response to POST /drive/volumes and GET
+// /drive/volumes/{volumeID}, as GetVolumeResponseDto.
+type VolumeResponse struct {
+	Code   int           `json:"Code"`
+	Volume VolumeDetails `json:"Volume"`
+}
+
+// LinksRequest is the body of POST /drive/v2/volumes/{volumeID}/links
+// and /drive/v2/volumes/{volumeID}/trash_multiple, as LinkIDsRequestDto.
+type LinksRequest struct {
+	LinkIDs []string `json:"LinkIDs"`
+}
+
+// LinksResponse is the response to POST
+// /drive/v2/volumes/{volumeID}/links, as LoadLinkDetailsResponseDto.
+// Unknown IDs are left out.
+type LinksResponse struct {
+	Code  int           `json:"Code"`
+	Links []LinkDetails `json:"Links"`
+}
+
+// ChildrenResponse is the response to GET
+// /drive/v2/volumes/{volumeID}/folders/{linkID}/children, as
+// ListChildrenResponseDto.
+type ChildrenResponse struct {
+	Code     int      `json:"Code"`
+	LinkIDs  []string `json:"LinkIDs"`
+	AnchorID string   `json:"AnchorID"` // pass as AnchorID for the next page
+	More     bool     `json:"More"`
+}
+
+// CreateFolderRequest is the body of POST
+// /drive/v2/volumes/{volumeID}/folders, as CreateFolderDto with the
+// SignatureEmail the SDK sends in place of SignatureAddress.
+type CreateFolderRequest struct {
+	ParentLinkID            string `json:"ParentLinkID"`
+	NodeKey                 string `json:"NodeKey"`
+	NodeHashKey             string `json:"NodeHashKey"`
+	NodePassphrase          string `json:"NodePassphrase"`
+	NodePassphraseSignature string `json:"NodePassphraseSignature"`
+	SignatureEmail          string `json:"SignatureEmail"`
+	Name                    string `json:"Name"`
+	Hash                    string `json:"Hash"`
+	XAttr                   string `json:"XAttr,omitempty"`
+}
+
+// CreateFolderResponse is the response to POST
+// /drive/v2/volumes/{volumeID}/folders, as CreateFolderResponseDto.
+type CreateFolderResponse struct {
+	Code   int `json:"Code"`
+	Folder struct {
+		ID string `json:"ID"`
+	} `json:"Folder"`
+}
+
+// RenameRequest is the body of PUT
+// /drive/v2/volumes/{volumeID}/links/{linkID}/rename, as
+// RenameLinkRequestDto.
+type RenameRequest struct {
+	Name               string `json:"Name"` // reusing the session key of the current name
+	Hash               string `json:"Hash"`
+	NameSignatureEmail string `json:"NameSignatureEmail"`
+	OriginalHash       string `json:"OriginalHash"` // current name hash, which must match
+}
+
+// MoveRequest is the body of PUT
+// /drive/v2/volumes/{volumeID}/links/{linkID}/move, as
+// MoveLinkV2RequestDto.
+//
+// NodePassphraseSignature and SignatureEmail are given if and only if
+// the node's key is anonymous (has no SignatureEmail).
+type MoveRequest struct {
+	ParentLinkID            string `json:"ParentLinkID"`
+	NodePassphrase          string `json:"NodePassphrase"` // reusing the session key of the current passphrase
+	NodePassphraseSignature string `json:"NodePassphraseSignature,omitempty"`
+	SignatureEmail          string `json:"SignatureEmail,omitempty"`
+	Name                    string `json:"Name"` // reusing the session key of the current name
+	NameSignatureEmail      string `json:"NameSignatureEmail"`
+	Hash                    string `json:"Hash"`
+	OriginalHash            string `json:"OriginalHash"` // current name hash, which must match
+}
+
+// CopyRequest is the body of POST
+// /drive/volumes/{volumeID}/links/{linkID}/copy, as CopyLinkRequestDto.
+// The server copies everything else of the node.
+//
+// NodePassphraseSignature and SignatureEmail are given if and only if
+// the node's key is anonymous (has no SignatureEmail).
+type CopyRequest struct {
+	Name                    string `json:"Name"`           // reusing the session key of the current name
+	NodePassphrase          string `json:"NodePassphrase"` // reusing the session key of the current passphrase
+	Hash                    string `json:"Hash"`
+	TargetVolumeID          string `json:"TargetVolumeID"`
+	TargetParentLinkID      string `json:"TargetParentLinkID"`
+	NameSignatureEmail      string `json:"NameSignatureEmail"`
+	NodePassphraseSignature string `json:"NodePassphraseSignature,omitempty"`
+	SignatureEmail          string `json:"SignatureEmail,omitempty"`
+}
+
+// CopyResponse is the response to POST
+// /drive/volumes/{volumeID}/links/{linkID}/copy, as CopyLinkResponseDto.
+type CopyResponse struct {
+	Code   int    `json:"Code"`
+	LinkID string `json:"LinkID"` // the new node
+}
+
+// LinkResult is the outcome for one link of a request on many.
+type LinkResult struct {
+	LinkID   string `json:"LinkID"`
+	Response struct {
+		Code  int    `json:"Code"`
+		Error string `json:"Error"`
+	} `json:"Response"`
+}
+
+// MultiLinkResponse is the response to a request on many links, such
+// as POST /drive/v2/volumes/{volumeID}/trash_multiple, as
+// MultiResponsesPerLinkFactory.
+type MultiLinkResponse struct {
+	Code      int          `json:"Code"`
+	Responses []LinkResult `json:"Responses"`
+}
+
+// RevisionResponse is the response to GET
+// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID},
+// as GetRevisionResponseDto.
+type RevisionResponse struct {
+	Code     int      `json:"Code"`
+	Revision Revision `json:"Revision"`
+}
+
+// Revision is a revision of a file with a page of its blocks, as
+// DetailedRevisionResponseDto.
+type Revision struct {
+	ID                string      `json:"ID"`
+	ManifestSignature string      `json:"ManifestSignature"` // armored
+	Size              int64       `json:"Size"`              // encrypted size
+	State             int         `json:"State"`
+	XAttr             string      `json:"XAttr"` // armored extended attributes; may be empty
+	Blocks            []Block     `json:"Blocks"`
+	Thumbnails        []Thumbnail `json:"Thumbnails"`
+	SignatureEmail    string      `json:"SignatureEmail"` // signer of the manifest; empty if anonymous
+	CreateTime        int64       `json:"CreateTime"`     // Unix seconds
+}
+
+// Block is a block of a revision with the URL to download it, as
+// BlockResponseDto.
+type Block struct {
+	Index   int    `json:"Index"`   // from 1
+	Hash    string `json:"Hash"`    // base64 SHA-256 of the encrypted block
+	Token   string `json:"Token"`   // send as pm-storage-token
+	BareURL string `json:"BareURL"` // storage URL of the block
+}
+
+// Thumbnail is a thumbnail of a revision, as ThumbnailResponseDto.
+type Thumbnail struct {
+	ThumbnailID string `json:"ThumbnailID"`
+	Type        int    `json:"Type"`
+	Hash        string `json:"Hash"` // base64 SHA-256 of the encrypted thumbnail
+	Size        int64  `json:"Size"`
+}
+
+// ConflictDetails are the Details of a CodeAlreadyExists error from
+// creating a node, naming the node which has the name.
+type ConflictDetails struct {
+	ConflictLinkID          string `json:"ConflictLinkID"`
+	ConflictRevisionID      string `json:"ConflictRevisionID"`      // active revision; empty for a draft or folder
+	ConflictDraftRevisionID string `json:"ConflictDraftRevisionID"` // draft revision; empty if none
+	ConflictDraftClientUID  string `json:"ConflictDraftClientUID"`  // ClientUID of the draft revision's creator
+}
+
+// CreateFileRequest is the body of POST
+// /drive/v2/volumes/{volumeID}/files, as CreateFileDto: a draft file.
+type CreateFileRequest struct {
+	ParentLinkID              string  `json:"ParentLinkID"`
+	Name                      string  `json:"Name"`
+	Hash                      string  `json:"Hash"`
+	MIMEType                  string  `json:"MIMEType"`
+	ClientUID                 *string `json:"ClientUID"`          // identifies the creator's drafts; null if none
+	IntendedUploadSize        *int64  `json:"IntendedUploadSize"` // rounded size for an early quota check; null if unknown
+	NodeKey                   string  `json:"NodeKey"`
+	NodePassphrase            string  `json:"NodePassphrase"`
+	NodePassphraseSignature   string  `json:"NodePassphraseSignature"`
+	ContentKeyPacket          string  `json:"ContentKeyPacket"`          // base64
+	ContentKeyPacketSignature string  `json:"ContentKeyPacketSignature"` // armored, by the node key
+	SignatureAddress          string  `json:"SignatureAddress"`          // signer of the name and passphrase
+}
+
+// CreateFileResponse is the response to POST
+// /drive/v2/volumes/{volumeID}/files, as CreateDraftFileResponseDto.
+type CreateFileResponse struct {
+	Code int `json:"Code"`
+	File struct {
+		ID         string `json:"ID"`
+		RevisionID string `json:"RevisionID"` // the draft revision
+	} `json:"File"`
+}
+
+// CreateRevisionRequest is the body of POST
+// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions, as
+// CreateRevisionRequestDto: a draft revision of an active file.
+type CreateRevisionRequest struct {
+	CurrentRevisionID  string  `json:"CurrentRevisionID"` // the active revision, which must match
+	ClientUID          *string `json:"ClientUID"`
+	IntendedUploadSize *int64  `json:"IntendedUploadSize"`
+}
+
+// CreateRevisionResponse is the response to POST
+// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions.
+type CreateRevisionResponse struct {
+	Code     int `json:"Code"`
+	Revision struct {
+		ID string `json:"ID"`
+	} `json:"Revision"`
+}
+
+// VerificationResponse is the response to GET
+// /drive/v2/volumes/{volumeID}/links/{linkID}/revisions/{revisionID}/verification,
+// as VerificationData.
+type VerificationResponse struct {
+	Code             int    `json:"Code"`
+	VerificationCode string `json:"VerificationCode"` // base64
+	ContentKeyPacket string `json:"ContentKeyPacket"` // base64
+}
+
+// BlockUploadRequest is the body of POST /drive/blocks, as
+// RequestUploadInput without its deprecated fields.
+type BlockUploadRequest struct {
+	AddressID     string            `json:"AddressID"` // address signing the blocks
+	VolumeID      string            `json:"VolumeID"`
+	LinkID        string            `json:"LinkID"`
+	RevisionID    string            `json:"RevisionID"`
+	BlockList     []BlockUploadInfo `json:"BlockList"`
+	ThumbnailList []struct{}        `json:"ThumbnailList"`
+}
+
+// BlockUploadInfo is a block to upload, as RequestUploadBlockInput
+// without its deprecated Size and Hash, which the SDK doesn't send.
+type BlockUploadInfo struct {
+	Index        int           `json:"Index"`        // from 1, consecutive
+	EncSignature string        `json:"EncSignature"` // armored signature of the plaintext, encrypted
+	Verifier     BlockVerifier `json:"Verifier"`
+}
+
+// BlockVerifier proves a block was encrypted with the right key.
+type BlockVerifier struct {
+	Token string `json:"Token"` // base64 verification code XOR the start of the encrypted block
+}
+
+// BlockUploadResponse is the response to POST /drive/blocks, as
+// RequestUploadResponse.
+type BlockUploadResponse struct {
+	Code        int          `json:"Code"`
+	UploadLinks []UploadLink `json:"UploadLinks"`
+}
+
+// UploadLink is the storage URL to upload a block to, as BlockURL.
+type UploadLink struct {
+	Index   int    `json:"Index"`
+	BareURL string `json:"BareURL"`
+	Token   string `json:"Token"` // send as pm-storage-token
+}
+
+// CommitRevisionRequest is the body of PUT
+// /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID},
+// as CommitRevisionDto without its deprecated fields.
+type CommitRevisionRequest struct {
+	ManifestSignature string    `json:"ManifestSignature"` // armored, of the block hashes in order
+	SignatureAddress  string    `json:"SignatureAddress"`  // signer of the manifest, blocks and attributes
+	XAttr             string    `json:"XAttr"`             // armored extended attributes
+	ChecksumVerified  bool      `json:"ChecksumVerified"`  // whether the SHA1 matched the source's
+	Photo             *struct{} `json:"Photo"`             // always null outside the photo volume
+}

backend/protondrive/budget_test.go

@@ -0,0 +1,333 @@
+package protondrive_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fs/operations"
+	fssync "github.com/rclone/rclone/fs/sync"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Routes counted by the request budgets.
+const (
+	linksRoute         = "POST /drive/v2/volumes/{volumeID}/links"
+	revisionDraftRoute = "POST /drive/v2/volumes/{volumeID}/files/{linkID}/revisions"
+	verifierRoute      = "GET /drive/v2/volumes/{volumeID}/links/{linkID}/revisions/{revisionID}/verification"
+	blockURLsRoute     = "POST /drive/blocks"
+	blockPutRoute      = "POST /storage/upload/{token}"
+	commitRoute        = "PUT /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}"
+	createFolderRoute  = "POST /drive/v2/volumes/{volumeID}/folders"
+)
+
+// TestMain runs the tests, then fails the run if any of them, through
+// any fake server, requested a removed share-scoped route.
+//
+// The config file is RCLONE_CONFIG's, as on rclone's command line,
+// whichever tests run: only fstest.Initialise would apply it, so the
+// default config, which may hold a real Proton remote, would be read.
+func TestMain(m *testing.M) {
+	if path, ok := os.LookupEnv("RCLONE_CONFIG"); ok {
+		if err := config.SetConfigPath(path); err != nil {
+			fmt.Fprintf(os.Stderr, "FAIL: RCLONE_CONFIG: %v\n", err)
+			os.Exit(1)
+		}
+	}
+	code := m.Run()
+	if legacy := protontest.LegacyRequests(); len(legacy) > 0 {
+		fmt.Fprintf(os.Stderr, "FAIL: TestBudgetNoLegacyRoutes: removed share-scoped routes were requested: %v\n", legacy)
+		code = 1
+	}
+	os.Exit(code)
+}
+
+// totalRequests returns the sum of the counts in requests.
+func totalRequests(requests map[string]int) int {
+	n := 0
+	for _, count := range requests {
+		n += count
+	}
+	return n
+}
+
+// ceilDiv returns n/d rounded up.
+func ceilDiv(n, d int) int {
+	return (n + d - 1) / d
+}
+
+// childPages returns the number of child ID pages the fake returns for
+// a folder of n children with page size pageSize: at least one.
+func childPages(n, pageSize int) int {
+	return max(1, ceilDiv(n, pageSize))
+}
+
+// seedFiles seeds n small files into folder parentID.
+func seedFiles(u *protontest.User, parentID string, n int) {
+	for i := range n {
+		u.SeedFile(parentID, fmt.Sprintf("file%03d.txt", i), []byte(fmt.Sprintf("file %d", i)), time.Now())
+	}
+}
+
+// budgetSizes are the folder sizes the listing budgets are checked at:
+// around the metadata batch of 100 and the child page of 150.
+var budgetSizes = []int{0, 1, 100, 101, 250}
+
+// budgetPageSize is the child ID page size the listing budgets use.
+const budgetPageSize = 150
+
+// seedBudgetFolders seeds a folder "nN" of N files for each of sizes.
+func seedBudgetFolders(srv *protontest.Server, u *protontest.User, sizes []int) {
+	srv.SetChildrenPageSize(budgetPageSize)
+	for _, n := range sizes {
+		seedFiles(u, u.SeedFolder(u.RootID(), fmt.Sprintf("n%d", n)), n)
+	}
+}
+
+// TestBudgetList checks listing a folder of N files costs one request
+// per page of child IDs and one per batch of 100 nodes, within spec
+// 9.4's P + ceil(N/100) + 1. The Fs is rooted at the folder, so finding
+// it is part of NewFs.
+//
+// A second listing costs nothing with enable_caching at its default,
+// and with it off costs the same as the first, no more. The latter is
+// checked at the sizes which differ in kind - empty, one batch, several
+// pages and batches - to save time.
+//
+// Measured, per route (children pages + links batches):
+// N=0: 1+0, N=1: 1+1, N=100: 1+1, N=101: 1+2, N=250: 2+3 - exactly
+// P + ceil(N/100), so the bound is that, without the spec's +1. Warm: 0
+// at every N; with caching off the second listing repeats the first.
+func TestBudgetList(t *testing.T) {
+	ctx := context.Background()
+	remote, srv, u := setupFakeRemote(t)
+	seedBudgetFolders(srv, u, budgetSizes)
+	list := func(t *testing.T, f fs.Fs, n int) map[string]int {
+		t.Helper()
+		srv.ResetRequests()
+		entries, err := f.List(ctx, "")
+		require.NoError(t, err)
+		require.Len(t, entries, n)
+		return srv.Requests()
+	}
+	for _, n := range budgetSizes {
+		t.Run(fmt.Sprint(n), func(t *testing.T) {
+			root := fmt.Sprintf("n%d", n)
+			f := newFakeFs(t, remote, root)
+			cold := list(t, f, n)
+			pages, batches := childPages(n, budgetPageSize), ceilDiv(n, 100)
+			assert.LessOrEqual(t, cold[childrenRoute], pages, "%v", cold)
+			assert.LessOrEqual(t, cold[linksRoute], batches, "%v", cold)
+			assert.LessOrEqual(t, totalRequests(cold), pages+batches, "%v", cold)
+
+			assert.Empty(t, list(t, f, n), "warm listing")
+
+			if n == 100 || n == 101 {
+				return
+			}
+			t.Setenv(fakeEnvPrefix+"ENABLE_CACHING", "false")
+			uncached := newFakeFs(t, remote, root)
+			assert.Equal(t, cold, list(t, uncached, n), "first listing without caching")
+			assert.Equal(t, cold, list(t, uncached, n), "second listing without caching")
+		})
+	}
+}
+
+// TestBudgetNoopSync checks a cold sync of a tree already in sync, by
+// size and modification time and by --checksum, lists each folder once
+// and transfers nothing: SHA-1s come from the metadata.
+//
+// For D = 6 folders (the root and 5 below it) and N = 200 files (40 per
+// folder), the plan's bound is 2*D + ceil(N/100) + 5 = 19. Measured, per
+// route, either way: 6 children pages + 6 links batches = 12, one page
+// and one batch per folder, as no folder has more than 100 children.
+// The bound is that: 2*D.
+func TestBudgetNoopSync(t *testing.T) {
+	const (
+		folders        = 5
+		filesPerFolder = 40
+		budget         = 2 * (folders + 1)
+	)
+	remote, srv, u := setupFakeRemote(t)
+	local := t.TempDir()
+	modTime := time.Now().Add(-time.Hour).Truncate(time.Second)
+	for d := range folders {
+		dir := fmt.Sprintf("dir%d", d)
+		id := u.SeedFolder(u.RootID(), dir)
+		require.NoError(t, os.Mkdir(filepath.Join(local, dir), 0o777))
+		for i := range filesPerFolder {
+			name := fmt.Sprintf("file%02d.txt", i)
+			content := []byte(fmt.Sprintf("content of %s/%s", dir, name))
+			u.SeedFile(id, name, content, modTime)
+			p := filepath.Join(local, dir, name)
+			require.NoError(t, os.WriteFile(p, content, 0o666))
+			require.NoError(t, os.Chtimes(p, modTime, modTime))
+		}
+	}
+
+	for _, checksum := range []bool{false, true} {
+		t.Run(fmt.Sprintf("checksum=%v", checksum), func(t *testing.T) {
+			ctx, ci := fs.AddConfig(context.Background())
+			ci.CheckSum = checksum
+			src, err := fs.NewFs(ctx, local)
+			require.NoError(t, err)
+			f := newFakeFs(t, remote, "")
+			srv.ResetRequests()
+			require.NoError(t, fssync.Sync(ctx, f, src, false))
+
+			got := srv.Requests()
+			assert.LessOrEqual(t, totalRequests(got), budget, "%v", got)
+			assert.Zero(t, blockTransfers(got), "%v", got)
+			assert.Zero(t, got[draftRoute]+got[revisionDraftRoute], "drafts created: %v", got)
+			for route := range got {
+				assert.NotContains(t, route, "/revisions/", "file content read: %v", got)
+			}
+		})
+	}
+}
+
+// TestBudgetNewObject checks finding a file three folders deep, cold,
+// costs one listing, a children page and a links batch, per folder on
+// its path.
+//
+// The plan's bound is 2 per level + 1 = 7 for three levels. Measured
+// for a/b/c/file.txt, per route: 4 children pages + 4 links batches =
+// 8, as the root is listed too - it is a fourth folder on the path,
+// and Proton has no route to look a child up by name. The bound is 2
+// per folder listed: 2*4.
+func TestBudgetNewObject(t *testing.T) {
+	ctx := context.Background()
+	remote, srv, u := setupFakeRemote(t)
+	parent := u.RootID()
+	for _, name := range []string{"a", "b", "c"} {
+		seedFiles(u, parent, 3)
+		parent = u.SeedFolder(parent, name)
+	}
+	u.SeedFile(parent, "file.txt", []byte("deep"), time.Now())
+
+	f := newFakeFs(t, remote, "")
+	srv.ResetRequests()
+	o, err := f.NewObject(ctx, "a/b/c/file.txt")
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), o.Size())
+
+	got := srv.Requests()
+	const listed = 4 // the root, a, b and c
+	assert.LessOrEqual(t, got[childrenRoute], listed, "%v", got)
+	assert.LessOrEqual(t, got[linksRoute], listed, "%v", got)
+	assert.LessOrEqual(t, totalRequests(got), 2*listed, "%v", got)
+}
+
+// TestBudgetUpload checks the requests of uploading new files to a
+// folder: one listing of the folder to check the name is free, the
+// first time only, then per file a draft, its verification code, at
+// most one block URL request per block, one PUT per block, the commit
+// and one read of the committed node.
+//
+// Measured, per route, into a folder of 3 files:
+//
+//	small.txt, cold: children 1 + links 1 (the listing), draft 1,
+//	verifier 1, block URLs 1, block PUTs 1, commit 1, links 1 = 8
+//	big.bin, 3 blocks: draft 1, verifier 1, block URLs 1-3 (as blocks
+//	are ready), block PUTs 3, commit 1, links 1 = 8-10
+//	again.txt: as small.txt without the listing = 6
+func TestBudgetUpload(t *testing.T) {
+	ctx := context.Background()
+	remote, srv, u := setupFakeRemote(t)
+	seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
+	f := newFakeFs(t, remote, "dir")
+
+	for _, test := range []struct {
+		name   string
+		size   int
+		blocks int
+		listed bool // the folder is listed to check the name is free
+	}{
+		{"small.txt", 5, 1, true},
+		{"big.bin", 2*blockSize + 5, 3, false},
+		{"again.txt", 5, 1, false},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			srv.ResetRequests()
+			content := patternedContent(test.size, 1)
+			_, err := operations.Copy(ctx, f, nil, test.name, object.NewMemoryObject(test.name, time.Now(), content))
+			require.NoError(t, err)
+
+			got := srv.Requests()
+			listing := 0
+			if test.listed {
+				listing = 2 // a children page and a links batch
+				assert.Equal(t, 1, got[childrenRoute], "%v", got)
+			} else {
+				assert.Zero(t, got[childrenRoute], "folder listed again: %v", got)
+			}
+			assert.Equal(t, 1, got[draftRoute], "%v", got)
+			assert.Equal(t, 1, got[verifierRoute], "%v", got)
+			assert.GreaterOrEqual(t, got[blockURLsRoute], 1, "%v", got)
+			assert.LessOrEqual(t, got[blockURLsRoute], test.blocks, "%v", got)
+			assert.Equal(t, test.blocks, got[blockPutRoute], "%v", got)
+			assert.Equal(t, 1, got[commitRoute], "%v", got)
+			assert.Equal(t, listing/2+1, got[linksRoute], "%v", got)
+			assert.LessOrEqual(t, totalRequests(got), listing+4+2*test.blocks, "%v", got)
+		})
+	}
+}
+
+// TestBudgetMkdir checks making folders side by side lists their
+// parent once, to check the first name is free, and then costs one
+// request per folder.
+//
+// Measured, per route, into a folder of 3 files: x: children 1 +
+// links 1 + create 1 = 3; y and z: create 1 each.
+func TestBudgetMkdir(t *testing.T) {
+	ctx := context.Background()
+	remote, srv, u := setupFakeRemote(t)
+	seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
+	f := newFakeFs(t, remote, "dir")
+	srv.ResetRequests()
+	for _, name := range []string{"x", "y", "z"} {
+		require.NoError(t, f.Mkdir(ctx, name))
+	}
+	got := srv.Requests()
+	assert.Equal(t, map[string]int{childrenRoute: 1, linksRoute: 1, createFolderRoute: 3}, got)
+
+	entries, err := f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Len(t, entries, 6, "the cached listing holds the new folders")
+}
+
+// TestBudgetNoLegacyRoutes checks a run of everyday operations requests
+// no removed share-scoped route. TestMain checks the same of every
+// test in the package, through every fake server.
+func TestBudgetNoLegacyRoutes(t *testing.T) {
+	ctx := context.Background()
+	remote, srv, u := setupFakeRemote(t)
+	seedFiles(u, u.SeedFolder(u.RootID(), "dir"), 3)
+	f := newFakeFs(t, remote, "")
+	o, err := operations.Copy(ctx, f, nil, "dir/new.txt", object.NewMemoryObject("dir/new.txt", time.Now(), []byte("new")))
+	require.NoError(t, err)
+	assert.Equal(t, "new", readObject(t, o))
+	_, err = operations.Move(ctx, f, nil, "moved/new.txt", o)
+	require.NoError(t, err)
+	require.NoError(t, operations.Purge(ctx, f, "dir"))
+	_, err = f.List(ctx, "moved")
+	require.NoError(t, err)
+	_, err = f.Features().About(ctx)
+	require.NoError(t, err)
+
+	for route := range srv.Requests() {
+		if strings.Contains(route, "/drive/shares/") {
+			assert.Equal(t, "GET /drive/shares/{shareID}", route)
+		}
+	}
+	assert.Empty(t, protontest.LegacyRequests())
+}

backend/protondrive/cache.go

@@ -0,0 +1,221 @@
+package protondrive
+
+import (
+	"slices"
+	"sync"
+)
+
+// nodeCache holds decrypted nodes and folder listings in memory.
+//
+// A nil nodeCache, as newNodeCache returns when caching is disabled,
+// holds nothing: its lookups miss and its updates do nothing. Cached
+// nodes are shared and must not be modified.
+type nodeCache struct {
+	mu       sync.Mutex
+	nodes    map[string]*node       // by ID
+	listings map[string][]string    // IDs of the complete listing of a folder, by folder ID
+	fetching map[string]*fetchState // folders being listed from the server, by folder ID
+}
+
+// fetchState tracks the listings of a folder being fetched, so that
+// one which began before a change to the folder's children isn't
+// cached over it.
+type fetchState struct {
+	count   int    // listings in progress
+	changes uint64 // changes to the folder's children since the first began
+}
+
+// newNodeCache returns an empty cache, or nil if enabled is false.
+func newNodeCache(enabled bool) *nodeCache {
+	if !enabled {
+		return nil
+	}
+	return &nodeCache{
+		nodes:    make(map[string]*node),
+		listings: make(map[string][]string),
+		fetching: make(map[string]*fetchState),
+	}
+}
+
+// get returns the node id, or nil if it isn't cached.
+func (c *nodeCache) get(id string) *node {
+	if c == nil {
+		return nil
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	return c.nodes[id]
+}
+
+// put caches n, replacing any node with its ID. If the cached node had
+// another parent, the listings of both parents are forgotten.
+func (c *nodeCache) put(n *node) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if old := c.nodes[n.ID]; old != nil && old.ParentID != n.ParentID {
+		c.childrenChangedLocked(old.ParentID)
+		c.childrenChangedLocked(n.ParentID)
+	}
+	c.nodes[n.ID] = n
+}
+
+// children returns the cached listing of folder parentID. It reports
+// false if the listing, or any node in it, isn't cached.
+func (c *nodeCache) children(parentID string) ([]*node, bool) {
+	if c == nil {
+		return nil, false
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	ids, ok := c.listings[parentID]
+	if !ok {
+		return nil, false
+	}
+	out := make([]*node, 0, len(ids))
+	for _, id := range ids {
+		n := c.nodes[id]
+		if n == nil {
+			return nil, false
+		}
+		out = append(out, n)
+	}
+	return out, true
+}
+
+// setChildren caches nodes as the complete listing of folder parentID,
+// and each of the nodes.
+func (c *nodeCache) setChildren(parentID string, nodes []*node) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.setChildrenLocked(parentID, nodes)
+}
+
+// setChildrenLocked is setChildren with c.mu held.
+func (c *nodeCache) setChildrenLocked(parentID string, nodes []*node) {
+	ids := make([]string, len(nodes))
+	for i, n := range nodes {
+		ids[i] = n.ID
+		c.nodes[n.ID] = n
+	}
+	c.listings[parentID] = ids
+}
+
+// startFetch records that a listing of folder parentID is being read
+// from the server and returns the token to pass to endFetch, which
+// must be called.
+func (c *nodeCache) startFetch(parentID string) (token uint64) {
+	if c == nil {
+		return 0
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	st := c.fetching[parentID]
+	if st == nil {
+		st = &fetchState{}
+		c.fetching[parentID] = st
+	}
+	st.count++
+	return st.changes
+}
+
+// endFetch ends a listing of folder parentID begun by startFetch with
+// token. If complete is set, nodes are cached as the listing, unless
+// the folder's children changed since it began, when what it read may
+// be out of date.
+func (c *nodeCache) endFetch(parentID string, token uint64, nodes []*node, complete bool) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	st := c.fetching[parentID]
+	if complete && st.changes == token {
+		c.setChildrenLocked(parentID, nodes)
+	}
+	st.count--
+	if st.count == 0 {
+		delete(c.fetching, parentID)
+	}
+}
+
+// childrenChangedLocked forgets the listing of folder parentID, and
+// marks its listings being fetched as out of date. c.mu must be held.
+func (c *nodeCache) childrenChangedLocked(parentID string) {
+	delete(c.listings, parentID)
+	if st := c.fetching[parentID]; st != nil {
+		st.changes++
+	}
+}
+
+// add caches n, a node new to its parent, and adds it to the cached
+// listing of its parent, if there is one.
+func (c *nodeCache) add(n *node) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if old := c.nodes[n.ID]; old != nil && old.ParentID != n.ParentID {
+		c.childrenChangedLocked(old.ParentID)
+		c.childrenChangedLocked(n.ParentID)
+	}
+	c.nodes[n.ID] = n
+	if st := c.fetching[n.ParentID]; st != nil {
+		st.changes++
+	}
+	if ids, ok := c.listings[n.ParentID]; ok && !slices.Contains(ids, n.ID) {
+		c.listings[n.ParentID] = append(ids, n.ID)
+	}
+}
+
+// invalidate forgets the node id, its listing if it is a folder, and
+// the listing of its parent.
+func (c *nodeCache) invalidate(id string) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if n := c.nodes[id]; n != nil {
+		c.childrenChangedLocked(n.ParentID)
+	} else {
+		// Its parent is unknown, and any listing being fetched may
+		// hold it.
+		for _, st := range c.fetching {
+			st.changes++
+		}
+	}
+	delete(c.nodes, id)
+	c.childrenChangedLocked(id)
+}
+
+// invalidateChildren forgets the listing of folder parentID, but not
+// the nodes in it.
+func (c *nodeCache) invalidateChildren(parentID string) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.childrenChangedLocked(parentID)
+}
+
+// clear forgets everything.
+func (c *nodeCache) clear() {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	clear(c.nodes)
+	clear(c.listings)
+	for _, st := range c.fetching {
+		st.changes++
+	}
+}

backend/protondrive/cache_internal_test.go

@@ -0,0 +1,184 @@
+package protondrive
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNodeCacheDisabled(t *testing.T) {
+	c := newNodeCache(false)
+	require.Nil(t, c)
+	n := &node{ID: "a", ParentID: "root"}
+	c.put(n)
+	c.setChildren("root", []*node{n})
+	assert.Nil(t, c.get("a"))
+	_, ok := c.children("root")
+	assert.False(t, ok)
+	c.invalidate("a")
+	c.invalidateChildren("root")
+	c.clear()
+}
+
+func TestNodeCache(t *testing.T) {
+	c := newNodeCache(true)
+	dir := &node{ID: "dir", ParentID: "root", IsFolder: true}
+	file := &node{ID: "file", ParentID: "dir"}
+
+	assert.Nil(t, c.get("dir"))
+	_, ok := c.children("root")
+	assert.False(t, ok)
+
+	c.setChildren("root", []*node{dir})
+	c.setChildren("dir", []*node{file})
+	assert.Same(t, dir, c.get("dir"))
+	assert.Same(t, file, c.get("file"))
+	got, ok := c.children("dir")
+	require.True(t, ok)
+	assert.Equal(t, []*node{file}, got)
+
+	// An empty listing is a listing.
+	c.setChildren("empty", nil)
+	got, ok = c.children("empty")
+	assert.True(t, ok)
+	assert.Empty(t, got)
+
+	// put replaces the node in the listing.
+	renamed := &node{ID: "file", ParentID: "dir", Name: "new"}
+	c.put(renamed)
+	got, ok = c.children("dir")
+	require.True(t, ok)
+	assert.Equal(t, []*node{renamed}, got)
+
+	// put of a node with another parent forgets both listings.
+	c.setChildren("other", nil)
+	moved := &node{ID: "file", ParentID: "other"}
+	c.put(moved)
+	assert.Same(t, moved, c.get("file"))
+	_, ok = c.children("dir")
+	assert.False(t, ok)
+	_, ok = c.children("other")
+	assert.False(t, ok)
+	_, ok = c.children("root")
+	assert.True(t, ok)
+	c.setChildren("dir", []*node{renamed})
+
+	// invalidate forgets the node and its parent's listing.
+	c.invalidate("file")
+	assert.Nil(t, c.get("file"))
+	_, ok = c.children("dir")
+	assert.False(t, ok)
+	_, ok = c.children("root")
+	assert.True(t, ok)
+
+	// and a folder's own listing.
+	c.setChildren("dir", []*node{file})
+	c.invalidate("dir")
+	_, ok = c.children("dir")
+	assert.False(t, ok)
+	_, ok = c.children("root")
+	assert.False(t, ok)
+	assert.Same(t, file, c.get("file"))
+
+	c.setChildren("dir", []*node{file})
+	c.invalidateChildren("dir")
+	_, ok = c.children("dir")
+	assert.False(t, ok)
+	assert.Same(t, file, c.get("file"))
+
+	c.setChildren("dir", []*node{file})
+	c.clear()
+	assert.Nil(t, c.get("file"))
+	_, ok = c.children("dir")
+	assert.False(t, ok)
+}
+
+func TestNodeCacheAdd(t *testing.T) {
+	c := newNodeCache(true)
+	a := &node{ID: "a", ParentID: "dir"}
+	b := &node{ID: "b", ParentID: "dir"}
+
+	// Without a listing of its parent, only the node is cached.
+	c.add(a)
+	assert.Same(t, a, c.get("a"))
+	_, ok := c.children("dir")
+	assert.False(t, ok)
+
+	c.setChildren("dir", []*node{a})
+	c.add(b)
+	got, ok := c.children("dir")
+	require.True(t, ok)
+	assert.Equal(t, []*node{a, b}, got)
+
+	// A node already listed is replaced, not listed twice.
+	b2 := &node{ID: "b", ParentID: "dir", Name: "new"}
+	c.add(b2)
+	got, ok = c.children("dir")
+	require.True(t, ok)
+	assert.Equal(t, []*node{a, b2}, got)
+
+	newNodeCache(false).add(a)
+}
+
+func TestNodeCacheFetch(t *testing.T) {
+	a := &node{ID: "a", ParentID: "dir"}
+	listed := func(c *nodeCache) bool {
+		_, ok := c.children("dir")
+		return ok
+	}
+
+	c := newNodeCache(true)
+	token := c.startFetch("dir")
+	c.endFetch("dir", token, []*node{a}, true)
+	assert.True(t, listed(c), "unchanged")
+	assert.Empty(t, c.fetching)
+
+	c = newNodeCache(true)
+	token = c.startFetch("dir")
+	c.endFetch("dir", token, []*node{a}, false)
+	assert.False(t, listed(c), "incomplete")
+	assert.Empty(t, c.fetching)
+
+	for name, change := range map[string]func(c *nodeCache){
+		"add":                func(c *nodeCache) { c.add(&node{ID: "b", ParentID: "dir"}) },
+		"invalidate child":   func(c *nodeCache) { c.invalidate("a") },
+		"invalidate unknown": func(c *nodeCache) { c.invalidate("unknown") },
+		"invalidate folder":  func(c *nodeCache) { c.invalidate("dir") },
+		"invalidateChildren": func(c *nodeCache) { c.invalidateChildren("dir") },
+		"put moved in":       func(c *nodeCache) { c.put(&node{ID: "other", ParentID: "dir"}) },
+		"put moved out":      func(c *nodeCache) { c.put(&node{ID: "a", ParentID: "elsewhere"}) },
+		"clear":              func(c *nodeCache) { c.clear() },
+	} {
+		t.Run(name, func(t *testing.T) {
+			c := newNodeCache(true)
+			c.put(a)
+			c.put(&node{ID: "other", ParentID: "elsewhere"})
+			first := c.startFetch("dir")
+			second := c.startFetch("dir")
+			change(c)
+			c.endFetch("dir", first, []*node{a}, true)
+			assert.False(t, listed(c), "first listing cached after a change")
+			// A fetch begun after the change is cached.
+			third := c.startFetch("dir")
+			c.endFetch("dir", second, []*node{a}, true)
+			assert.False(t, listed(c), "second listing cached after a change")
+			c.endFetch("dir", third, []*node{a}, true)
+			assert.True(t, listed(c))
+			assert.Empty(t, c.fetching, "fetch state kept")
+		})
+	}
+
+	// Changes elsewhere don't matter.
+	c = newNodeCache(true)
+	c.put(&node{ID: "x", ParentID: "other"})
+	token = c.startFetch("dir")
+	c.add(&node{ID: "y", ParentID: "other"})
+	c.invalidate("x")
+	c.invalidateChildren("other")
+	c.endFetch("dir", token, []*node{a}, true)
+	assert.True(t, listed(c))
+
+	var disabled *nodeCache
+	disabled.endFetch("dir", disabled.startFetch("dir"), nil, true)
+}

backend/protondrive/download.go

@@ -0,0 +1,786 @@
+package protondrive
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/base64"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"slices"
+	"strconv"
+	"sync"
+
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/rclone/rclone/backend/protondrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+var (
+	// errIntegrity is returned when a file's content doesn't match
+	// what its uploader signed or claimed.
+	errIntegrity = errors.New("integrity check failed")
+
+	// errBlockHash is returned when a block doesn't match its hash
+	// however many times it is downloaded.
+	errBlockHash = errors.New("block hash mismatch")
+
+	// errDownloadClosed is returned by Read after Close.
+	errDownloadClosed = errors.New("download closed")
+)
+
+// signatureHint ends an error for a signature which fails, telling the
+// user how to read the file anyway.
+const signatureHint = "set ignore_signature_errors to read the file anyway"
+
+const (
+	// defaultBlockSize is the plaintext size of every block but the
+	// last of a file the client uploads.
+	defaultBlockSize = 4 << 20
+
+	// blocksPerPage is the most blocks one revision request lists.
+	blocksPerPage = 20
+
+	// maxBlockSize is the largest block accepted, encrypted or
+	// decrypted.
+	maxBlockSize = 8 << 20
+
+	// blockAttempts is how many times a block whose hash doesn't match
+	// is downloaded.
+	blockAttempts = 3
+
+	// downloadAhead is the most blocks of one download fetched or
+	// waiting to be read, besides the one being read.
+	downloadAhead = 4
+
+	// minUnclaimedBlockSize bounds the blocks of a file which doesn't
+	// record its block sizes: there are at most its encrypted size over
+	// this, plus one.
+	minUnclaimedBlockSize = 1 << 20
+
+	// maxFileBlocks is the most blocks a file may list: 2 TiB in
+	// 4 MiB blocks, beyond any one file the largest Proton plan, of a
+	// few TB in all, could hold. A full read holds the metadata of
+	// every block until the manifest is checked, so this bounds how
+	// many whatever the server claims: about 250-350 MiB at the cap
+	// with realistic tokens and URLs. Their length isn't bounded.
+	maxFileBlocks = 1 << 19
+
+	// contentKeyFallbackUntil is the Unix time until which a content
+	// key packet may be signed by any of the account's address keys,
+	// as in the SDK.
+	contentKeyFallbackUntil = 1672531200 // 2023-01-01
+)
+
+// open returns a reader of limit bytes of the content of file n from
+// offset, or to the end if limit < 0.
+//
+// A read from offset 0 to the end lists every block and checks the
+// manifest signature before releasing any byte: if it fails, the first
+// Read returns errIntegrity, unless IgnoreSignatureErrors is set, when
+// it is logged. A ranged read checks each block against its listed
+// hash but can't check the manifest. A node key or content key
+// signature which fails is errIntegrity from open, on the same terms.
+// The reader must be closed.
+func (c *client) open(ctx context.Context, n *node, offset, limit int64) (io.ReadCloser, error) {
+	return c.openRead(ctx, n, offset, limit, offset == 0 && limit < 0)
+}
+
+// openWhole is open for limit bytes from offset 0 of a range which
+// covers the whole file, checked as a full read is.
+func (c *client) openWhole(ctx context.Context, n *node, limit int64) (io.ReadCloser, error) {
+	return c.openRead(ctx, n, 0, limit, true)
+}
+
+// openRead is open, listing every block and checking the manifest
+// before releasing any byte if full is set, which needs offset 0.
+func (c *client) openRead(ctx context.Context, n *node, offset, limit int64, full bool) (io.ReadCloser, error) {
+	if n.IsFolder || n.link == nil || n.link.File == nil || n.RevisionID == "" {
+		return nil, fmt.Errorf("open %q: not a file with content", n.Name)
+	}
+	if offset < 0 {
+		return nil, fmt.Errorf("open %q: negative offset %d", n.Name, offset)
+	}
+	sk, err := c.contentKey(n, "reading")
+	if err != nil {
+		return nil, fmt.Errorf("open %q: %w", n.Name, err)
+	}
+	var sizes []int64
+	if n.X != nil {
+		sizes = n.X.BlockSizes
+	}
+	full = full && offset == 0
+	if len(sizes) > c.maxFileBlocks {
+		return nil, fmt.Errorf("open %q: %w: %d blocks recorded, more than %d blocks", n.Name, errIntegrity, len(sizes), c.maxFileBlocks)
+	}
+	if sizes != nil {
+		var total int64
+		for i, size := range sizes {
+			// Bounding each size bounds the total too.
+			if size > maxBlockSize {
+				return nil, fmt.Errorf("open %q: %w: block %d recorded as %d bytes", n.Name, errIntegrity, i+1, size)
+			}
+			total += size
+		}
+		switch {
+		case full:
+			// A full read always lists the blocks, to reach the
+			// manifest.
+		case offset >= total:
+			limit = 0
+		case limit >= 0:
+			limit = min(limit, total-offset)
+		}
+	}
+	if limit == 0 {
+		return io.NopCloser(bytes.NewReader(nil)), nil
+	}
+	// Without recorded block sizes only the sizes learnt by earlier
+	// reads tell which block holds offset, as old clients didn't all
+	// write 4 MiB blocks, so the read starts at the last block they
+	// reach and discards the bytes before offset.
+	first, skip, last := 1, offset, -1
+	var startHash []byte
+	switch {
+	case sizes != nil:
+		first, skip = blockAt(sizes, offset)
+		if limit > 0 && !full {
+			last, _ = blockAt(sizes, offset+limit-1)
+		}
+	case !full:
+		first, skip, last, startHash = c.learnt.locate(n.RevisionID, offset, limit)
+	}
+	maxBlocks := len(sizes)
+	if sizes == nil {
+		maxBlocks = int(min(n.EncryptedSize/minUnclaimedBlockSize+1, 1<<30))
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	r := &blockReader{
+		c:         c,
+		n:         n,
+		sk:        sk,
+		ctx:       ctx,
+		cancel:    cancel,
+		slots:     make(chan chan blockResult, downloadAhead),
+		sizes:     sizes,
+		startHash: startHash,
+		skip:      skip,
+		remaining: limit,
+	}
+	r.wg.Add(1)
+	go r.list(first, last, maxBlocks, full)
+	return r, nil
+}
+
+// blockAt returns the index, from 1, of the block holding plaintext
+// offset pos of a file with block sizes sizes, and the offset of pos in
+// it.
+func blockAt(sizes []int64, pos int64) (index int, offset int64) {
+	for i, size := range sizes {
+		if pos < size {
+			return i + 1, pos
+		}
+		pos -= size
+	}
+	return len(sizes) + 1, pos
+}
+
+// contentKey returns the content session key of file n for op,
+// "reading" or "updating", checking its signature as the SDK's
+// decryptContentKeyPacket does, except that a missing signature fails
+// too: anyone with the node's public key can make a content key packet.
+//
+// An unverified node key or content key is errIntegrity unless
+// IgnoreSignatureErrors is set, when it is logged once per node and op:
+// whoever made the key could read what is written with it and make up
+// the content read.
+func (c *client) contentKey(n *node, op string) (*crypto.SessionKey, error) {
+	hint := signatureHint
+	if op != "reading" {
+		hint = "set ignore_signature_errors to update the file anyway"
+	}
+	if n.keys.unverified != nil {
+		if !c.opt.IgnoreSignatureErrors {
+			return nil, fmt.Errorf("node key: %w: %w: %s", errIntegrity, n.keys.unverified, hint)
+		}
+		c.reportUnverified(op, n, "")
+	}
+	f := n.link.File
+	now := c.s.serverTime()
+	sk, verifyErr, err := decryptContentKey(f.ContentKeyPacket, f.ContentKeyPacketSignature, n.keys.keyRing, n.keyVerifyKR, now)
+	if err != nil {
+		return nil, err
+	}
+	if verifyErr != nil && !isNotSigned(verifyErr) && n.CreateTime.Unix() <= contentKeyFallbackUntil {
+		// Old clients signed with an address key, and a rename or
+		// move could change the node's signer without signing the
+		// content key packet again.
+		krs := make([]*crypto.KeyRing, 0, len(c.acct.addresses))
+		for _, addr := range c.acct.addresses {
+			krs = append(krs, addr.keyRing)
+		}
+		if allKR, err := joinKeyRings(krs...); err == nil && verifyDetached(sk.Key, f.ContentKeyPacketSignature, allKR, now) == nil {
+			fs.Debugf(nil, "protondrive: %q (ID %s): content key signed by another address of the account", n.Name, n.ID)
+			verifyErr = nil
+		}
+	}
+	if verifyErr != nil {
+		verifyErr = fmt.Errorf("content key: %w: %w", errIntegrity, verifyErr)
+		if !c.opt.IgnoreSignatureErrors {
+			return nil, fmt.Errorf("%w: %s", verifyErr, hint)
+		}
+		c.reportOnce("content key "+op, n.ID, func() {
+			fs.Errorf(nil, "protondrive: %s %q (ID %s): %v", op, n.Name, n.ID, verifyErr)
+		})
+	}
+	return sk, nil
+}
+
+// revisionBlocks returns the active revision of file n with at most
+// pageSize of its blocks from block index from.
+func (c *client) revisionBlocks(ctx context.Context, n *node, from, pageSize int) (*api.Revision, error) {
+	opts := &rest.Opts{
+		Method: "GET",
+		Path:   c.volumePath("/drive/v2/volumes", "files", n.ID, "revisions", n.RevisionID),
+		Parameters: url.Values{
+			"FromBlockIndex": {strconv.Itoa(from)},
+			"PageSize":       {strconv.Itoa(pageSize)},
+		},
+	}
+	var resp api.RevisionResponse
+	if err := c.apiCall(ctx, opts, nil, &resp); err != nil {
+		return nil, fmt.Errorf("list blocks: %w", err)
+	}
+	return &resp.Revision, nil
+}
+
+// blockURL returns block index of file n with a new download URL.
+func (c *client) blockURL(ctx context.Context, n *node, index int) (api.Block, error) {
+	rev, err := c.revisionBlocks(ctx, n, index, 1)
+	if err != nil {
+		return api.Block{}, err
+	}
+	if len(rev.Blocks) == 0 || rev.Blocks[0].Index != index {
+		return api.Block{}, fmt.Errorf("list blocks: block %d is missing", index)
+	}
+	return rev.Blocks[0], nil
+}
+
+// downloadBlock returns the encrypted block at bareURL, which token
+// unlocks, and its SHA-256. The block must be closed.
+func (c *client) downloadBlock(ctx context.Context, bareURL, token string) (enc *pool.RW, sum []byte, err error) {
+	c.downloadTokens.Get()
+	defer c.downloadTokens.Put()
+	opts := &rest.Opts{
+		Method:  "GET",
+		RootURL: bareURL,
+		// The storage host gets the block's token, never the
+		// session's credentials, which only session.call adds.
+		ExtraHeaders: map[string]string{"pm-storage-token": token},
+	}
+	err = c.pacedCall(ctx, c.storagePacer, func() (*http.Response, error) {
+		if enc != nil {
+			_ = enc.Close()
+		}
+		enc = pool.NewRW(pool.Global())
+		resp, err := c.s.srv.Call(ctx, opts)
+		if err != nil {
+			return resp, err
+		}
+		defer func() { _ = resp.Body.Close() }()
+		if resp.ContentLength > maxBlockSize {
+			return nil, fmt.Errorf("block is larger than %d bytes", maxBlockSize)
+		}
+		h := sha256.New()
+		size, err := enc.ReadFrom(io.TeeReader(io.LimitReader(resp.Body, maxBlockSize+1), h))
+		if err != nil {
+			return resp, err
+		}
+		if size > maxBlockSize {
+			return nil, fmt.Errorf("block is larger than %d bytes", maxBlockSize)
+		}
+		sum = h.Sum(nil)
+		return resp, nil
+	})
+	if err != nil {
+		if enc != nil {
+			_ = enc.Close()
+		}
+		return nil, nil, err
+	}
+	return enc, sum, nil
+}
+
+// blockResult is the outcome of a block of a download, or its end.
+type blockResult struct {
+	index int
+	hash  []byte   // the block's SHA-256
+	plain *pool.RW // the decrypted block; nil on error or at the end
+	err   error
+	end   bool // no more blocks follow; err is the verdict on the file
+}
+
+// blockReader reads a range of a file, fetching blocks ahead of the
+// reader in the background.
+type blockReader struct {
+	c      *client
+	n      *node
+	sk     *crypto.SessionKey
+	ctx    context.Context
+	cancel context.CancelFunc
+	slots  chan chan blockResult // results to read, in order
+	wg     sync.WaitGroup        // the lister and the fetches
+	sizes  []int64               // claimed plaintext block sizes; nil if not recorded
+	// startHash is the hash the first block had when its size was
+	// learnt, if it was.
+	startHash []byte
+
+	mu        sync.Mutex // held by Read and Close
+	cur       *pool.RW   // block being read
+	skip      int64      // bytes to discard before the range, from the first block on
+	remaining int64      // bytes left to read; negative for all
+	err       error      // returned by every Read once set
+	closed    bool
+}
+
+// list lists the blocks first to last (to the end if last < 0) and
+// fetches them, then sends the end of the download. If verify is set,
+// the list having started at the first block, it lists every block and
+// checks the manifest before fetching any.
+//
+// A list of more than maxBlocks blocks is an error.
+func (r *blockReader) list(first, last, maxBlocks int, verify bool) {
+	defer r.wg.Done()
+	end := make(chan blockResult, 1)
+	end <- blockResult{end: true, err: r.listBlocks(first, last, maxBlocks, verify)}
+	select {
+	case r.slots <- end:
+	case <-r.ctx.Done():
+	}
+}
+
+// listedBlock is a block of the list with its decoded hash.
+type listedBlock struct {
+	block api.Block
+	hash  []byte
+}
+
+// listBlocks does the work of list, returning the verdict on the file.
+func (r *blockReader) listBlocks(first, last, maxBlocks int, verify bool) error {
+	var (
+		hashes    [][]byte
+		listed    []listedBlock // blocks waiting for the manifest check, if verify
+		manifest  string
+		signer    string
+		index     = first
+		firstPage = true
+	)
+	for {
+		pageSize := blocksPerPage
+		if last > 0 {
+			pageSize = min(pageSize, last-index+1)
+		}
+		rev, err := r.c.revisionBlocks(r.ctx, r.n, index, pageSize)
+		if err != nil {
+			return err
+		}
+		if firstPage && verify {
+			manifest, signer = rev.ManifestSignature, rev.SignatureEmail
+			// As in the SDK, the manifest starts with the thumbnails.
+			for _, th := range rev.Thumbnails {
+				hash, err := decodeBlockHash(th.Hash)
+				if err != nil {
+					return fmt.Errorf("thumbnail: %w", err)
+				}
+				hashes = append(hashes, hash)
+			}
+		}
+		firstPage = false
+		if len(rev.Blocks) == 0 {
+			break
+		}
+		for _, b := range rev.Blocks {
+			switch {
+			case index > r.c.maxFileBlocks:
+				return fmt.Errorf("%w: more than %d blocks listed, more than any file can have", errIntegrity, r.c.maxFileBlocks)
+			case b.Index != index:
+				return fmt.Errorf("%w: block %d listed where block %d belongs", errIntegrity, b.Index, index)
+			case index > maxBlocks && r.sizes != nil:
+				return fmt.Errorf("%w: more blocks listed than the %d the file records", errIntegrity, maxBlocks)
+			case index > maxBlocks:
+				return fmt.Errorf("%w: more blocks listed than the file's encrypted size of %d allows", errIntegrity, r.n.EncryptedSize)
+			}
+			hash, err := decodeBlockHash(b.Hash)
+			if err != nil {
+				return fmt.Errorf("block %d: %w", index, err)
+			}
+			if index == first && r.startHash != nil && !bytes.Equal(hash, r.startHash) {
+				// The sizes placing the range came from another block
+				// list of the same revision.
+				return fmt.Errorf("%w: block %d listed with another hash than when it was read before", errIntegrity, index)
+			}
+			if verify {
+				// Nothing is released before the manifest vouches for
+				// every block's hash and place.
+				hashes = append(hashes, hash)
+				listed = append(listed, listedBlock{block: b, hash: hash})
+			} else if !r.fetch(b, hash) {
+				return r.ctx.Err()
+			}
+			if index == last {
+				return nil
+			}
+			index++
+		}
+	}
+	if r.sizes != nil && index <= len(r.sizes) {
+		return fmt.Errorf("%w: %d blocks listed of the %d the file records", errIntegrity, index-1, len(r.sizes))
+	}
+	if !verify {
+		return nil
+	}
+	if err := r.verifyManifest(hashes, manifest, signer); err != nil {
+		return err
+	}
+	for _, lb := range listed {
+		if !r.fetch(lb.block, lb.hash) {
+			return r.ctx.Err()
+		}
+	}
+	return nil
+}
+
+// decodeBlockHash returns the base64 SHA-256 hash.
+func decodeBlockHash(b64 string) ([]byte, error) {
+	hash, err := base64.StdEncoding.DecodeString(b64)
+	if err == nil && len(hash) != sha256.Size {
+		err = fmt.Errorf("%d bytes long", len(hash))
+	}
+	if err != nil {
+		return nil, fmt.Errorf("bad hash: %w", err)
+	}
+	return hash, nil
+}
+
+// verifyManifest checks the manifest signature of the file's block
+// hashes against the keys of signer, or the node key if it is empty.
+func (r *blockReader) verifyManifest(hashes [][]byte, manifest, signer string) error {
+	verifyKR, err := r.c.verifyKeys(r.ctx, signer, r.n.keys.keyRing)
+	if err != nil {
+		return fmt.Errorf("verify manifest: %w", err)
+	}
+	err = verifyManifest(hashes, manifest, verifyKR, r.c.s.serverTime())
+	if err == nil {
+		return nil
+	}
+	err = fmt.Errorf("%w: %w", errIntegrity, err)
+	if !r.c.opt.IgnoreSignatureErrors {
+		return fmt.Errorf("%w: %s", err, signatureHint)
+	}
+	fs.Errorf(nil, "protondrive: %q (ID %s): %v", r.n.Name, r.n.ID, err)
+	return nil
+}
+
+// fetch queues block b, whose SHA-256 is hash, for the reader and
+// fetches it in the background. It returns false if the download is
+// cancelled first.
+func (r *blockReader) fetch(b api.Block, hash []byte) bool {
+	slot := make(chan blockResult, 1)
+	select {
+	case r.slots <- slot:
+	case <-r.ctx.Done():
+		return false
+	}
+	r.wg.Add(1)
+	go func() {
+		defer r.wg.Done()
+		slot <- r.fetchBlock(b, hash)
+	}()
+	return true
+}
+
+// fetchBlock downloads and decrypts block b, whose SHA-256 is hash,
+// requesting a new URL after a 404 (once) or a hash mismatch.
+func (r *blockReader) fetchBlock(b api.Block, hash []byte) blockResult {
+	index := b.Index
+	refetched := false
+	for attempt := 1; ; {
+		plain, err := r.fetchOnce(b, hash)
+		var apiErr *api.Error
+		switch {
+		case err == nil:
+			return blockResult{index: index, hash: hash, plain: plain}
+		case r.ctx.Err() != nil:
+			return blockResult{index: index, err: r.ctx.Err()}
+		case errors.As(err, &apiErr) && apiErr.Status == http.StatusNotFound && !refetched:
+			// The URL may have expired.
+			refetched = true
+		case errors.Is(err, errBlockHash) && attempt < blockAttempts:
+			// The bytes may have been damaged on the way.
+			attempt++
+		default:
+			return blockResult{index: index, err: fmt.Errorf("block %d: %w", index, err)}
+		}
+		fs.Debugf(nil, "protondrive: %q (ID %s): block %d: %v: retrying with a new URL", r.n.Name, r.n.ID, index, err)
+		// hash stays the listed one, which the manifest covers.
+		if b, err = r.c.blockURL(r.ctx, r.n, index); err != nil {
+			return blockResult{index: index, err: fmt.Errorf("block %d: %w", index, err)}
+		}
+		if relisted, err := decodeBlockHash(b.Hash); err != nil || !bytes.Equal(relisted, hash) {
+			return blockResult{index: index, err: fmt.Errorf("block %d: %w: listed again with another hash", index, errIntegrity)}
+		}
+	}
+}
+
+// fetchOnce downloads block b and checks it against hash and its
+// claimed size, returning it decrypted.
+func (r *blockReader) fetchOnce(b api.Block, hash []byte) (*pool.RW, error) {
+	enc, sum, err := r.c.downloadBlock(r.ctx, b.BareURL, b.Token)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { _ = enc.Close() }()
+	if !bytes.Equal(sum, hash) {
+		return nil, errBlockHash
+	}
+	dec, err := blockDecrypter(enc, r.sk)
+	if err != nil {
+		return nil, err
+	}
+	plain := pool.NewRW(pool.Global())
+	size, err := plain.ReadFrom(io.LimitReader(dec, maxBlockSize+1))
+	switch {
+	case err != nil:
+		err = fmt.Errorf("decrypt block: %w", err)
+	case size > maxBlockSize:
+		err = fmt.Errorf("block decrypts to more than %d bytes", maxBlockSize)
+	case r.sizes != nil && size != r.sizes[b.Index-1]:
+		err = fmt.Errorf("%w: block is %d bytes, the file records %d", errIntegrity, size, r.sizes[b.Index-1])
+	}
+	if err != nil {
+		_ = plain.Close()
+		return nil, err
+	}
+	return plain, nil
+}
+
+// Read reads the range of the file.
+func (r *blockReader) Read(p []byte) (int, error) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if len(p) == 0 {
+		return 0, nil
+	}
+	for {
+		switch {
+		case r.closed:
+			return 0, errDownloadClosed
+		case r.err != nil:
+			return 0, r.err
+		case r.remaining == 0:
+			return 0, io.EOF
+		}
+		if r.cur == nil {
+			if err := r.next(); err != nil {
+				r.err = err
+				r.cancel()
+			}
+			continue
+		}
+		if r.remaining > 0 && int64(len(p)) > r.remaining {
+			p = p[:r.remaining]
+		}
+		n, err := r.cur.Read(p)
+		if n > 0 {
+			if r.remaining > 0 {
+				r.remaining -= int64(n)
+			}
+			return n, nil
+		}
+		if err != nil && err != io.EOF {
+			r.err = err
+			continue
+		}
+		_ = r.cur.Close()
+		r.cur = nil
+	}
+}
+
+// next makes the next block holding bytes of the range current,
+// returning io.EOF at the end.
+func (r *blockReader) next() error {
+	for {
+		var slot chan blockResult
+		select {
+		case slot = <-r.slots:
+		case <-r.ctx.Done():
+			return r.ctx.Err()
+		}
+		res := <-slot
+		switch {
+		case res.err != nil && r.ctx.Err() != nil:
+			return r.ctx.Err()
+		case res.err != nil:
+			return res.err
+		case res.end:
+			return io.EOF
+		}
+		if r.sizes == nil {
+			if err := r.c.learnt.learn(r.n.RevisionID, res.index, res.hash, res.plain.Size()); err != nil {
+				_ = res.plain.Close()
+				return err
+			}
+		}
+		if size := res.plain.Size(); r.skip > 0 && r.skip >= size {
+			r.skip -= size
+			_ = res.plain.Close()
+			continue
+		}
+		if r.skip > 0 {
+			if _, err := res.plain.Seek(r.skip, io.SeekStart); err != nil {
+				_ = res.plain.Close()
+				return err
+			}
+			r.skip = 0
+		}
+		r.cur = res.plain
+		return nil
+	}
+}
+
+// Close stops the download and frees its buffers. It may be called
+// more than once.
+func (r *blockReader) Close() error {
+	r.cancel()
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.closed {
+		return nil
+	}
+	r.closed = true
+	r.wg.Wait()
+	if r.cur != nil {
+		_ = r.cur.Close()
+		r.cur = nil
+	}
+	// Every queued slot has its result now.
+	for {
+		select {
+		case slot := <-r.slots:
+			if res := <-slot; res.plain != nil {
+				_ = res.plain.Close()
+			}
+		default:
+			return nil
+		}
+	}
+}
+
+// check interfaces
+var _ io.ReadCloser = (*blockReader)(nil)
+
+const (
+	// learntRevisions and learntBlocks bound the block sizes a client
+	// learns: of at most this many revisions, and this many blocks in
+	// all, about 2.5 MiB.
+	learntRevisions = 16
+	learntBlocks    = 1 << 16
+)
+
+// learntBlock is the plaintext size of a block read, with its hash.
+type learntBlock struct {
+	hash [sha256.Size]byte
+	size int64
+}
+
+// blockSizeCache holds the plaintext sizes of the blocks, from the
+// first on, read of revisions which don't record their block sizes, so
+// a later ranged read, as each chunk of a multi-thread copy is, can
+// start at the block holding its offset. The zero value is empty.
+type blockSizeCache struct {
+	mu     sync.Mutex
+	revs   map[string][]learntBlock // by revision ID
+	order  []string                 // revision IDs, oldest first
+	blocks int                      // learnt in all
+}
+
+// locate returns where a read of limit bytes (to the end if negative)
+// from offset of revision id starts, as for blockReader: the first
+// block, the bytes of it to skip, the last block or -1 if not known,
+// and the hash the first block had when its size was learnt, or nil if
+// it wasn't.
+func (b *blockSizeCache) locate(id string, offset, limit int64) (first int, skip int64, last int, startHash []byte) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	blocks := b.revs[id]
+	first, skip, last = len(blocks)+1, offset, -1
+	for i, lb := range blocks {
+		if skip < lb.size {
+			first = i + 1
+			startHash = slices.Clone(lb.hash[:])
+			break
+		}
+		skip -= lb.size
+	}
+	if first > len(blocks) || limit <= 0 {
+		return first, skip, last, startHash
+	}
+	end := skip + limit - 1
+	for i := first - 1; i < len(blocks); i++ {
+		if end < blocks[i].size {
+			return first, skip, i + 1, startHash
+		}
+		end -= blocks[i].size
+	}
+	return first, skip, last, startHash
+}
+
+// learn records that block index of revision id, whose SHA-256 is hash,
+// is size bytes. A block learnt before with another hash or size is
+// errIntegrity: the server listed two blocks for one place.
+func (b *blockSizeCache) learn(id string, index int, hash []byte, size int64) error {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	blocks, ok := b.revs[id]
+	if index <= len(blocks) {
+		if lb := blocks[index-1]; !bytes.Equal(lb.hash[:], hash) || lb.size != size {
+			return fmt.Errorf("%w: block %d listed with another hash than when it was read before", errIntegrity, index)
+		}
+		return nil
+	}
+	if index != len(blocks)+1 || len(hash) != sha256.Size {
+		return nil
+	}
+	if !ok {
+		if b.revs == nil {
+			b.revs = make(map[string][]learntBlock)
+		}
+		for len(b.order) >= learntRevisions {
+			b.forgetOldest()
+		}
+		b.order = append(b.order, id)
+	}
+	for b.blocks >= learntBlocks && len(b.order) > 1 && b.order[0] != id {
+		b.forgetOldest()
+	}
+	if b.blocks >= learntBlocks {
+		return nil
+	}
+	lb := learntBlock{size: size}
+	copy(lb.hash[:], hash)
+	b.revs[id] = append(blocks, lb)
+	b.blocks++
+	return nil
+}
+
+// forgetOldest drops the revision learnt first.
+func (b *blockSizeCache) forgetOldest() {
+	b.blocks -= len(b.revs[b.order[0]])
+	delete(b.revs, b.order[0])
+	b.order = b.order[1:]
+}

backend/protondrive/download_internal_test.go

@@ -0,0 +1,852 @@
+package protondrive
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"math"
+	"math/rand/v2"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	storagePattern  = "GET /storage/blocks/{id}"
+	revisionPattern = "GET /drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}"
+
+	// bigSize is the size of a file of three blocks, the last partial.
+	bigSize = 9<<20 + 123
+)
+
+// testContent returns n bytes of reproducible random content.
+func testContent(n int) []byte {
+	r := rand.New(rand.NewPCG(uint64(n), 1))
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = byte(r.Uint32())
+	}
+	return b
+}
+
+// seedFile seeds file name with content in the root of u and returns
+// its node as c reads it.
+func seedFile(t *testing.T, c *client, u *protontest.User, name string, content []byte) *node {
+	t.Helper()
+	return getTestNode(t, c, u.SeedFile(u.RootID(), name, content, time.Now()))
+}
+
+// getTestNode returns node id as c reads it.
+func getTestNode(t *testing.T, c *client, id string) *node {
+	t.Helper()
+	n, err := c.getNode(context.Background(), id)
+	require.NoError(t, err)
+	return n
+}
+
+// withoutBlockSizes returns a copy of n whose extended attributes
+// don't record its block sizes, as some old clients wrote them.
+func withoutBlockSizes(n *node) *node {
+	cp := *n
+	cp.X = nil
+	return &cp
+}
+
+// readRange opens n at offset for limit bytes and reads it to the end.
+func readRange(t *testing.T, c *client, n *node, offset, limit int64) ([]byte, error) {
+	t.Helper()
+	rc, err := c.open(context.Background(), n, offset, limit)
+	if err != nil {
+		return nil, err
+	}
+	data, err := io.ReadAll(rc)
+	require.NoError(t, rc.Close())
+	return data, err
+}
+
+// roundTripFunc is an http.RoundTripper calling itself.
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+// RoundTrip calls f.
+func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+// newTestClientTransport returns a client for a new session of u whose
+// requests go through the transport wrap returns.
+func newTestClientTransport(t *testing.T, srv *protontest.Server, u *protontest.User, opt clientOptions, wrap func(next http.RoundTripper) http.RoundTripper) *client {
+	t.Helper()
+	base := newAccountSession(srv, u)
+	s := newSession(&http.Client{Transport: wrap(srv.Client().Transport)}, srv.URL(), testAppVersion, base.store)
+	c, err := newClient(context.Background(), s, opt)
+	require.NoError(t, err)
+	t.Cleanup(c.close)
+	return c
+}
+
+// leakBaseline returns the goroutine count and pool buffers in use with
+// no idle connections, for requireNoLeaks.
+func leakBaseline(srv *protontest.Server) (goroutines, buffers int) {
+	srv.Client().CloseIdleConnections()
+	return runtime.NumGoroutine(), pool.Global().InUse()
+}
+
+// requireNoLeaks fails the test unless the goroutine count returns to
+// goroutines within 2s and the pool buffers in use are back to buffers.
+func requireNoLeaks(t *testing.T, srv *protontest.Server, goroutines, buffers int) {
+	t.Helper()
+	require.Eventually(t, func() bool {
+		srv.Client().CloseIdleConnections()
+		return runtime.NumGoroutine() <= goroutines
+	}, 2*time.Second, 20*time.Millisecond, "goroutines left running")
+	assert.Equal(t, buffers, pool.Global().InUse(), "pool buffers not returned")
+}
+
+func TestDownloadSizes(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	for _, size := range []int{0, 1, 4<<20 - 1, 4 << 20, 4<<20 + 1, 8 << 20, bigSize} {
+		t.Run(strconv.Itoa(size), func(t *testing.T) {
+			content := testContent(size)
+			n := seedFile(t, c, u, "f"+strconv.Itoa(size), content)
+			data, err := readRange(t, c, n, 0, -1)
+			require.NoError(t, err)
+			assert.True(t, bytes.Equal(content, data), "content differs")
+		})
+	}
+}
+
+func TestDownloadRanges(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	content := testContent(bigSize)
+	claimed := seedFile(t, c, u, "big", content)
+	unrecorded := withoutBlockSizes(claimed)
+	for _, tc := range []struct {
+		offset, limit int64
+		blocks        int // blocks fetched with the block sizes claimed
+	}{
+		{0, -1, 3},
+		{0, 1, 1},
+		{0, 0, 0},
+		{4<<20 - 1, 2, 2},
+		{4 << 20, -1, 2},
+		{bigSize - 1, -1, 1},
+		{bigSize, -1, 0},
+		{8 << 20, 100, 1},
+		{bigSize - 10, 100, 1},
+	} {
+		want := content[tc.offset:]
+		if tc.limit >= 0 {
+			want = want[:min(tc.limit, int64(len(want)))]
+		}
+		for _, v := range []struct {
+			name string
+			n    *node
+		}{{"claimed", claimed}, {"unrecorded", unrecorded}} {
+			t.Run(v.name+"/"+strconv.FormatInt(tc.offset, 10)+"+"+strconv.FormatInt(tc.limit, 10), func(t *testing.T) {
+				srv.ResetRequests()
+				data, err := readRange(t, c, v.n, tc.offset, tc.limit)
+				require.NoError(t, err)
+				assert.True(t, bytes.Equal(want, data), "got %d bytes, want %d", len(data), len(want))
+				fetched := srv.Requests()[storagePattern]
+				if v.n == claimed {
+					assert.Equal(t, tc.blocks, fetched, "blocks fetched")
+					return
+				}
+				// The block sizes learnt by the first, full, read
+				// place the range; blocks past it may be fetched ahead
+				// before the reader is closed.
+				assert.GreaterOrEqual(t, fetched, tc.blocks, "blocks fetched")
+				assert.LessOrEqual(t, fetched, 3, "blocks fetched")
+			})
+		}
+	}
+}
+
+func TestDownloadRangesUnrecordedBlockSizes(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	const size = 6<<20 + 5
+	content := testContent(size)
+	// Blocks of 1 MiB, which a reader assuming 4 MiB blocks would
+	// place wrongly.
+	n := getTestNode(t, c, u.SeedFileInBlocks(u.RootID(), "small-blocks", content, 1<<20))
+	require.Nil(t, n.X.BlockSizes)
+	for _, tc := range []struct{ offset, limit int64 }{
+		{0, -1},
+		{0, 3 << 20},
+		{10, 100},
+		{1<<20 - 3, 10},
+		{4<<20 + 10, 100},
+		{4<<20 + 10, -1},
+		{2<<20 + 7, 3 << 20},
+		{size - 1, -1},
+		{size, -1},
+	} {
+		t.Run(strconv.FormatInt(tc.offset, 10)+"+"+strconv.FormatInt(tc.limit, 10), func(t *testing.T) {
+			want := content[tc.offset:]
+			if tc.limit >= 0 {
+				want = want[:min(tc.limit, int64(len(want)))]
+			}
+			data, err := readRange(t, c, n, tc.offset, tc.limit)
+			require.NoError(t, err)
+			assert.True(t, bytes.Equal(want, data), "got %d bytes, want %d", len(data), len(want))
+		})
+	}
+}
+
+func TestDownloadChunksUnrecordedBlockSizes(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	const (
+		blocks     = 32
+		chunk      = 4 << 20
+		blockBytes = 1 << 20
+	)
+	content := testContent(blocks * blockBytes)
+	id := u.SeedFileInBlocks(u.RootID(), "chunks", content, blockBytes)
+	n := getTestNode(t, c, id)
+	require.Nil(t, n.X.BlockSizes)
+
+	// A multi-thread copy opens each chunk as a range of its own.
+	srv.ResetRequests()
+	for offset := int64(0); offset < int64(len(content)); offset += chunk {
+		data, err := readRange(t, c, n, offset, chunk)
+		require.NoError(t, err)
+		require.True(t, bytes.Equal(content[offset:offset+chunk], data), "chunk at %d", offset)
+	}
+	fetched := srv.Requests()[storagePattern]
+	t.Logf("%d blocks fetched for %d chunks of a file of %d blocks", fetched, len(content)/chunk, blocks)
+	// Reading each chunk from the first block would fetch about
+	// blocks^2/(2*blocks per chunk) = 128; each chunk may fetch up to
+	// downloadAhead blocks past its end.
+	assert.LessOrEqual(t, fetched, blocks+len(content)/chunk*downloadAhead, "chunks read from the first block")
+
+	// Once the sizes are known, a range fetches only its own blocks.
+	srv.ResetRequests()
+	data, err := readRange(t, c, n, 10*blockBytes+5, 2*blockBytes)
+	require.NoError(t, err)
+	assert.Equal(t, content[10*blockBytes+5:12*blockBytes+5], data)
+	assert.Equal(t, 3, srv.Requests()[storagePattern], "blocks fetched")
+
+	// A block list which changed since then is an integrity error.
+	u.Corrupt(id, "swap-blocks")
+	_, err = readRange(t, c, n, 5, 10)
+	require.ErrorIs(t, err, errIntegrity)
+	assert.Contains(t, err.Error(), "listed with another hash")
+}
+
+func TestDownloadLegacyFile(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	content := testContent(5<<20 + 7)
+	n := getTestNode(t, c, u.SeedLegacyFile(u.RootID(), "legacy", content))
+	data, err := readRange(t, c, n, 0, -1)
+	require.NoError(t, err)
+	assert.True(t, bytes.Equal(content, data), "content differs")
+	data, err = readRange(t, c, n, 4<<20+3, 10)
+	require.NoError(t, err)
+	assert.Equal(t, content[4<<20+3:4<<20+13], data)
+}
+
+func TestDownloadManifestSigners(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	other := srv.AddUser("other", "pass")
+	content := testContent(1000)
+	for name, id := range map[string]string{
+		"other user": u.SeedFileSignedBy(other, u.RootID(), "other", content, time.Now()),
+		"anonymous":  u.SeedAnonymousFile(u.RootID(), "anonymous", content, time.Now()),
+	} {
+		t.Run(name, func(t *testing.T) {
+			data, err := readRange(t, c, getTestNode(t, c, id), 0, -1)
+			require.NoError(t, err)
+			assert.Equal(t, content, data)
+		})
+	}
+}
+
+func TestDownloadContentKey(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	const second = "second@proton.test"
+	u.AddAddress(second)
+	c := newTestClientFor(t, srv, u, clientOptions{})
+	ignoring := newTestClientFor(t, srv, u, clientOptions{IgnoreSignatureErrors: true})
+	before2023 := time.Date(2022, 6, 1, 0, 0, 0, 0, time.UTC)
+	content := testContent(1000)
+	for _, tc := range []struct {
+		name    string
+		signer  string
+		created time.Time
+		ok      bool
+	}{
+		{"unsigned", "", time.Now(), false},
+		{"unsigned before 2023", "", before2023, false},
+		{"signer address", u.Email(), time.Now(), true},
+		{"other address before 2023", second, before2023, true},
+		{"other address after 2023", second, time.Now(), false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			id := u.SeedFileContentKeySignedBy(tc.signer, tc.created, u.RootID(), tc.name, content)
+			data, err := readRange(t, c, getTestNode(t, c, id), 0, -1)
+			if !tc.ok {
+				require.ErrorIs(t, err, errIntegrity)
+				assert.Contains(t, err.Error(), "ignore_signature_errors")
+				data, err = readRange(t, ignoring, getTestNode(t, ignoring, id), 0, -1)
+			}
+			require.NoError(t, err)
+			assert.Equal(t, content, data)
+		})
+	}
+}
+
+func TestDownloadNodeKeySignature(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	content := testContent(1000)
+	id := u.SeedFile(u.RootID(), "f", content, time.Now())
+	u.Corrupt(id, "passphrase-signature")
+	n := getTestNode(t, c, id)
+
+	data, err := readRange(t, c, n, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+	assert.Contains(t, err.Error(), "ignore_signature_errors")
+	assert.Empty(t, data)
+	_, err = readRange(t, c, n, 10, 10)
+	require.ErrorIs(t, err, errIntegrity, "a ranged read")
+
+	ignoring := newTestClientFor(t, srv, u, clientOptions{IgnoreSignatureErrors: true})
+	data, err = readRange(t, ignoring, getTestNode(t, ignoring, id), 0, -1)
+	require.NoError(t, err)
+	assert.Equal(t, content, data)
+}
+
+func TestDownloadBlockHashMismatch(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	n := seedFile(t, c, u, "f", testContent(1<<20))
+	u.Corrupt(n.ID, "block-hash")
+	srv.ResetRequests()
+	data, err := readRange(t, c, n, 0, -1)
+	require.ErrorIs(t, err, errBlockHash)
+	assert.Empty(t, data)
+	assert.Equal(t, 3, srv.Requests()[storagePattern], "block attempts")
+	// The listing page, the empty page ending the list and a new URL
+	// for each retry.
+	assert.Equal(t, 4, srv.Requests()[revisionPattern])
+}
+
+func TestDownloadSwappedBlocks(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	n := seedFile(t, c, u, "f", testContent(bigSize))
+	u.Corrupt(n.ID, "swap-blocks")
+	_, err := readRange(t, c, n, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+}
+
+func TestDownloadTruncatedBlocks(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	n := seedFile(t, c, u, "f", testContent(bigSize))
+	u.Corrupt(n.ID, "truncate-blocks")
+	_, err := readRange(t, c, n, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+	// Without block sizes, only the manifest shows the loss.
+	_, err = readRange(t, c, withoutBlockSizes(n), 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+}
+
+func TestDownloadBadManifest(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	content := testContent(bigSize)
+	n := seedFile(t, c, u, "f", content)
+	u.Corrupt(n.ID, "manifest")
+
+	data, err := readRange(t, c, n, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+	assert.Contains(t, err.Error(), "ignore_signature_errors")
+	assert.Empty(t, data, "no byte comes before the verdict")
+
+	// A ranged read can't check the manifest.
+	data, err = readRange(t, c, n, 0, 100)
+	require.NoError(t, err)
+	assert.Equal(t, content[:100], data)
+
+	ignoring := newTestClientFor(t, srv, u, clientOptions{IgnoreSignatureErrors: true})
+	data, err = readRange(t, ignoring, getTestNode(t, ignoring, n.ID), 0, -1)
+	require.NoError(t, err)
+	assert.True(t, bytes.Equal(content, data), "content differs")
+}
+
+func TestDownloadEmptyFileBadManifest(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	n := seedFile(t, c, u, "empty", nil)
+	u.Corrupt(n.ID, "manifest")
+	_, err := readRange(t, c, n, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+}
+
+func TestDownloadStorage404Refetch(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	content := testContent(1 << 20)
+	n := seedFile(t, c, u, "f", content)
+
+	srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound, Times: 1})
+	srv.ResetRequests()
+	data, err := readRange(t, c, n, 0, -1)
+	require.NoError(t, err)
+	assert.Equal(t, content, data)
+	assert.Equal(t, 2, srv.Requests()[storagePattern])
+	// The listing page, the new URL and the empty page ending the list.
+	assert.Equal(t, 3, srv.Requests()[revisionPattern])
+
+	// The URL is requested again only once.
+	srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound})
+	srv.ResetRequests()
+	_, err = readRange(t, c, n, 0, -1)
+	require.Error(t, err)
+	assert.Equal(t, 2, srv.Requests()[storagePattern])
+}
+
+func TestDownloadRefetchOtherHash(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	n := seedFile(t, c, u, "f", testContent(1<<20))
+
+	// The block's URL expires, and the server lists it again with
+	// another block, which the checked manifest doesn't cover.
+	srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusNotFound, Times: 1, Delay: 300 * time.Millisecond})
+	done := make(chan struct{})
+	var data []byte
+	var err error
+	go func() {
+		defer close(done)
+		data, err = readRange(t, c, n, 0, -1)
+	}()
+	require.Eventually(t, func() bool { return srv.Requests()[storagePattern] == 1 }, 5*time.Second, 5*time.Millisecond)
+	u.Corrupt(n.ID, "foreign-block")
+	<-done
+	require.ErrorIs(t, err, errIntegrity)
+	assert.Contains(t, err.Error(), "listed again with another hash")
+	assert.Empty(t, data)
+}
+
+func TestDownloadRetries5xx(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	content := testContent(1 << 20)
+	n := seedFile(t, c, u, "f", content)
+	srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusServiceUnavailable, Times: 2})
+	srv.Inject(protontest.Fault{Pattern: "/drive/v2/volumes/{volumeID}/files/{linkID}/revisions/{revisionID}", Status: http.StatusBadGateway, Times: 1})
+	srv.ResetRequests()
+	data, err := readRange(t, c, n, 0, -1)
+	require.NoError(t, err)
+	assert.Equal(t, content, data)
+	assert.Equal(t, 3, srv.Requests()[storagePattern])
+}
+
+func TestDownloadStorageRetryAfter(t *testing.T) {
+	t.Parallel()
+	c, srv, u := newTestClient(t)
+	content := testContent(1000)
+	n := seedFile(t, c, u, "f", content)
+	srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Status: http.StatusTooManyRequests, RetryAfter: 1, Times: 1})
+	start := time.Now()
+	data, err := readRange(t, c, n, 0, -1)
+	require.NoError(t, err)
+	assert.Equal(t, content, data)
+	assert.GreaterOrEqual(t, time.Since(start), time.Second)
+}
+
+func TestDownloadStorageHeaders(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	var (
+		mu      sync.Mutex
+		headers []http.Header
+	)
+	c := newTestClientTransport(t, srv, u, clientOptions{}, func(next http.RoundTripper) http.RoundTripper {
+		return roundTripFunc(func(r *http.Request) (*http.Response, error) {
+			if strings.HasPrefix(r.URL.Path, "/storage/") {
+				mu.Lock()
+				headers = append(headers, r.Header.Clone())
+				mu.Unlock()
+			}
+			return next.RoundTrip(r)
+		})
+	})
+	n := seedFile(t, c, u, "f", testContent(4<<20+1))
+	_, err := readRange(t, c, n, 0, -1)
+	require.NoError(t, err)
+	require.Len(t, headers, 2)
+	for _, h := range headers {
+		assert.Empty(t, h.Get("Authorization"))
+		assert.Empty(t, h.Get("x-pm-uid"))
+		assert.NotEmpty(t, h.Get("pm-storage-token"))
+		assert.Equal(t, testAppVersion, h.Get("x-pm-appversion"))
+	}
+
+	// Block URLs on another host, as the real storage host is, get the
+	// same headers. The other host passes requests on to the fake.
+	var other []http.Header
+	storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		mu.Lock()
+		other = append(other, r.Header.Clone())
+		mu.Unlock()
+		req, err := http.NewRequestWithContext(r.Context(), "GET", srv.URL()+r.URL.Path, nil)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+		for _, k := range []string{"pm-storage-token", "x-pm-appversion"} {
+			req.Header.Set(k, r.Header.Get(k))
+		}
+		resp, err := srv.Client().Do(req)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusBadGateway)
+			return
+		}
+		defer func() { _ = resp.Body.Close() }()
+		w.WriteHeader(resp.StatusCode)
+		_, _ = io.Copy(w, resp.Body)
+	}))
+	t.Cleanup(storage.Close)
+	require.NotEqual(t, srv.URL(), storage.URL)
+	moved := newTestClientTransport(t, srv, u, clientOptions{}, func(next http.RoundTripper) http.RoundTripper {
+		return roundTripFunc(func(r *http.Request) (*http.Response, error) {
+			resp, err := next.RoundTrip(r)
+			if err != nil || !strings.Contains(r.URL.Path, "/revisions/") {
+				return resp, err
+			}
+			body, err := io.ReadAll(resp.Body)
+			_ = resp.Body.Close()
+			if err != nil {
+				return nil, err
+			}
+			body = bytes.ReplaceAll(body, []byte(srv.URL()+"/storage/"), []byte(storage.URL+"/storage/"))
+			resp.Body = io.NopCloser(bytes.NewReader(body))
+			resp.ContentLength = int64(len(body))
+			resp.Header.Del("Content-Length")
+			return resp, nil
+		})
+	})
+	_, err = readRange(t, moved, getTestNode(t, moved, n.ID), 0, -1)
+	require.NoError(t, err)
+	require.Len(t, other, 2)
+	for _, h := range other {
+		assert.Empty(t, h.Get("Authorization"))
+		assert.Empty(t, h.Get("x-pm-uid"))
+		assert.NotEmpty(t, h.Get("pm-storage-token"))
+	}
+}
+
+func TestDownloadBlockTooLarge(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	var fetches atomic.Int32
+	c := newTestClientTransport(t, srv, u, clientOptions{}, func(next http.RoundTripper) http.RoundTripper {
+		return roundTripFunc(func(r *http.Request) (*http.Response, error) {
+			if !strings.HasPrefix(r.URL.Path, "/storage/") {
+				return next.RoundTrip(r)
+			}
+			fetches.Add(1)
+			// An endless body of unknown length.
+			return &http.Response{
+				StatusCode:    http.StatusOK,
+				Header:        http.Header{},
+				Body:          io.NopCloser(zeroReader{}),
+				ContentLength: -1,
+				Request:       r,
+			}, nil
+		})
+	})
+	n := seedFile(t, c, u, "f", testContent(1000))
+	_, err := readRange(t, c, n, 0, -1)
+	require.ErrorContains(t, err, "larger than")
+	assert.NotErrorIs(t, err, errBlockHash)
+	assert.Equal(t, int32(1), fetches.Load(), "an oversized block is not fetched again")
+}
+
+// zeroReader reads zeros forever.
+type zeroReader struct{}
+
+// Read fills p with zeros.
+func (zeroReader) Read(p []byte) (int, error) {
+	clear(p)
+	return len(p), nil
+}
+
+// endlessBlockList returns a transport wrapper whose revision pages are
+// all full of copies of the file's first block, renumbered, counting
+// the pages in pages.
+func endlessBlockList(pages *atomic.Int32) func(next http.RoundTripper) http.RoundTripper {
+	var (
+		mu    sync.Mutex
+		first map[string]any
+	)
+	return func(next http.RoundTripper) http.RoundTripper {
+		return roundTripFunc(func(r *http.Request) (*http.Response, error) {
+			resp, err := next.RoundTrip(r)
+			if err != nil || !strings.Contains(r.URL.Path, "/revisions/") {
+				return resp, err
+			}
+			pages.Add(1)
+			var body map[string]any
+			err = json.NewDecoder(resp.Body).Decode(&body)
+			_ = resp.Body.Close()
+			if err != nil {
+				return nil, err
+			}
+			rev := body["Revision"].(map[string]any)
+			mu.Lock()
+			if blocks := rev["Blocks"].([]any); first == nil && len(blocks) > 0 {
+				first = blocks[0].(map[string]any)
+			}
+			from, _ := strconv.Atoi(r.URL.Query().Get("FromBlockIndex"))
+			size, _ := strconv.Atoi(r.URL.Query().Get("PageSize"))
+			var blocks []any
+			for i := range size {
+				b := make(map[string]any, len(first))
+				for k, v := range first {
+					b[k] = v
+				}
+				b["Index"] = from + i
+				blocks = append(blocks, b)
+			}
+			mu.Unlock()
+			rev["Blocks"] = blocks
+			data, err := json.Marshal(body)
+			if err != nil {
+				return nil, err
+			}
+			resp.Body = io.NopCloser(bytes.NewReader(data))
+			resp.ContentLength = int64(len(data))
+			resp.Header.Del("Content-Length")
+			return resp, nil
+		})
+	}
+}
+
+func TestDownloadEndlessBlockList(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	var pages atomic.Int32
+	c := newTestClientTransport(t, srv, u, clientOptions{EnableCaching: true}, endlessBlockList(&pages))
+	n := seedFile(t, c, u, "f", testContent(4<<20+1))
+	for name, n := range map[string]*node{"claimed": n, "unrecorded": withoutBlockSizes(n)} {
+		t.Run(name, func(t *testing.T) {
+			pages.Store(0)
+			data, err := readRange(t, c, n, 0, -1)
+			require.ErrorIs(t, err, errIntegrity)
+			assert.LessOrEqual(t, len(data), 5*(4<<20))
+			assert.LessOrEqual(t, pages.Load(), int32(2))
+		})
+	}
+}
+
+func TestDownloadBlockCap(t *testing.T) {
+	t.Parallel()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	var pages atomic.Int32
+	c := newTestClientTransport(t, srv, u, clientOptions{EnableCaching: true}, endlessBlockList(&pages))
+	c.maxFileBlocks = 45
+	n := seedFile(t, c, u, "f", testContent(4<<20+1))
+
+	// The encrypted size, which nothing signs, claims room for many
+	// more blocks than the cap.
+	huge := withoutBlockSizes(n)
+	huge.EncryptedSize = 1 << 50
+	data, err := readRange(t, c, huge, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+	assert.Contains(t, err.Error(), "more than 45 blocks")
+	assert.Empty(t, data)
+	assert.LessOrEqual(t, pages.Load(), int32(3), "listed past the cap")
+
+	// Recorded block sizes beyond the cap fail before any request.
+	claims := *n
+	x := *n.X
+	x.BlockSizes = make([]int64, 46)
+	claims.X = &x
+	pages.Store(0)
+	_, err = readRange(t, c, &claims, 0, -1)
+	require.ErrorIs(t, err, errIntegrity)
+	assert.Zero(t, pages.Load())
+}
+
+func TestDownloadCloseEarly(t *testing.T) {
+	c, srv, u := newTestClient(t)
+	n := seedFile(t, c, u, "f", testContent(8<<20))
+	goroutines, buffers := leakBaseline(srv)
+
+	rc, err := c.open(context.Background(), n, 0, -1)
+	require.NoError(t, err)
+	buf := make([]byte, 10)
+	_, err = io.ReadFull(rc, buf)
+	require.NoError(t, err)
+	require.NoError(t, rc.Close())
+	require.NoError(t, rc.Close())
+	_, err = rc.Read(buf)
+	require.Error(t, err)
+	requireNoLeaks(t, srv, goroutines, buffers)
+}
+
+func TestDownloadContextCancel(t *testing.T) {
+	c, srv, u := newTestClient(t)
+	n := seedFile(t, c, u, "f", testContent(8<<20))
+	// One of the blocks takes far longer than the test.
+	srv.Inject(protontest.Fault{Pattern: "/storage/blocks/{id}", Skip: 1, Times: 1, Delay: time.Minute})
+	goroutines, buffers := leakBaseline(srv)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	rc, err := c.open(ctx, n, 0, -1)
+	require.NoError(t, err)
+	time.AfterFunc(200*time.Millisecond, cancel)
+	start := time.Now()
+	data, err := io.ReadAll(rc)
+	require.ErrorIs(t, err, context.Canceled)
+	assert.Less(t, len(data), 8<<20)
+	assert.Less(t, time.Since(start), 10*time.Second)
+	require.NoError(t, rc.Close())
+	requireNoLeaks(t, srv, goroutines, buffers)
+}
+
+func TestDownloadHostileBlockSizes(t *testing.T) {
+	t.Parallel()
+	c, _, u := newTestClient(t)
+	small := testContent(1000)
+	big := testContent(4<<20 + 1)
+	for _, tc := range []struct {
+		name    string
+		content []byte
+		sizes   []int64
+	}{
+		{"overflowing", small, []int64{math.MaxInt64, 1}},
+		{"above the cap", small, []int64{maxBlockSize + 1}},
+		{"sum too small", big, []int64{1000}},
+		{"too few blocks", big, []int64{4 << 20}},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			n := getTestNode(t, c, u.SeedFileClaimingBlockSizes(u.RootID(), tc.name, tc.content, tc.sizes))
+			require.Equal(t, tc.sizes, n.X.BlockSizes)
+			_, err := readRange(t, c, n, 0, -1)
+			require.ErrorIs(t, err, errIntegrity)
+		})
+	}
+}
+
+// frozenFileDir holds files the old library (go-proton-api v1.0.4 and
+// Proton-API-Bridge v1.0.5) wrote, as the server stored them, one
+// directory per format: "legacy" (v3 PKESK content key, SEIPDv1 blocks)
+// and "v6" (v6 PKESK content key, SEIPDv2 blocks). The old library only
+// writes v6 content as a revision of a file another client created, so
+// in "v6" the node key and content key packet with its signature are
+// SDK-style stand-ins made with plain gopenpgp.
+const frozenFileDir = "testdata/oldlib/file"
+
+// frozenFile is a file a client holding Keys stored, with what it
+// encrypted. In a directory, file.json holds it but Stored, which is
+// in link.json, revision.json and block-1.bin, block-2.bin...
+type frozenFile struct {
+	Comment string
+	Email   string // address which signed everything, the owner's primary address
+	Keys    protontest.FixedKeys
+	Name    string
+	ModTime string // as the writer recorded it in the extended attributes
+	Content []byte
+	Stored  protontest.StoredFile `json:"-"`
+}
+
+// frozenFileParts returns the names of the files of f in a directory
+// and the values they hold.
+func frozenFileParts(f *frozenFile) map[string]any {
+	return map[string]any{"file.json": f, "link.json": &f.Stored.Link, "revision.json": &f.Stored.Revision}
+}
+
+// blockFileName returns the name of the file holding block i, from 0.
+func blockFileName(i int) string {
+	return "block-" + strconv.Itoa(i+1) + ".bin"
+}
+
+// loadFrozenFile returns the file in dir.
+func loadFrozenFile(t testing.TB, dir string) frozenFile {
+	t.Helper()
+	var f frozenFile
+	for name, v := range frozenFileParts(&f) {
+		data, err := os.ReadFile(filepath.Join(dir, name))
+		require.NoError(t, err)
+		require.NoError(t, json.Unmarshal(data, v), name)
+	}
+	for i := range f.Stored.Revision.Blocks {
+		data, err := os.ReadFile(filepath.Join(dir, blockFileName(i)))
+		require.NoError(t, err)
+		f.Stored.Revision.Blocks[i].Data = data
+	}
+	return f
+}
+
+// checkFrozenFile stores f for a user holding its keys and checks that
+// the client reads its metadata, verifying every signature, and
+// downloads its content.
+func checkFrozenFile(t *testing.T, f frozenFile) {
+	t.Helper()
+	srv := protontest.New(t)
+	u := srv.AddUserWithKeys(f.Email, "pass", f.Keys)
+	c := newTestClientFor(t, srv, u, clientOptions{})
+	n := getTestNode(t, c, u.SeedStoredFile(u.RootID(), f.Stored))
+	assert.Equal(t, f.Name, n.Name)
+	require.NotNil(t, n.X, "extended attributes")
+	modTime, err := time.Parse("2006-01-02T15:04:05-0700", f.ModTime)
+	require.NoError(t, err)
+	assert.True(t, modTime.Equal(n.modTime()), "modification time %v, want %v", n.modTime(), modTime)
+	assert.Equal(t, int64(len(f.Content)), n.size(true))
+	assert.Equal(t, sha1Of(f.Content), n.sha1())
+	data, err := readRange(t, c, n, 0, -1)
+	require.NoError(t, err)
+	assert.True(t, bytes.Equal(f.Content, data), "content differs: %d bytes, want %d", len(data), len(f.Content))
+	assert.Empty(t, c.reported, "problems reported")
+}
+
+func TestFrozenFileFixture(t *testing.T) {
+	for format, want := range map[string]struct{ pkesk, seipd int }{"legacy": {3, 1}, "v6": {6, 2}} {
+		t.Run(format, func(t *testing.T) {
+			f := loadFrozenFile(t, filepath.Join(frozenFileDir, format))
+			// The fixture is in the format its directory names.
+			keyPacket, err := base64.StdEncoding.DecodeString(f.Stored.Link.ContentKeyPacket)
+			require.NoError(t, err)
+			assert.Equal(t, want.pkesk, readPackets(t, keyPacket, 1)[0].(*packet.EncryptedKey).Version, "content key packet")
+			require.NotEmpty(t, f.Stored.Revision.Blocks)
+			for i, b := range f.Stored.Revision.Blocks {
+				assert.Equal(t, want.seipd, readPackets(t, b.Data, 1)[0].(*packet.SymmetricallyEncrypted).Version, "block %d", i+1)
+			}
+			checkFrozenFile(t, f)
+		})
+	}
+}

backend/protondrive/export_test.go

@@ -0,0 +1,25 @@
+package protondrive
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+)
+
+// ConfigBrowserSignIn runs the browser sign-in of rclone config for
+// remote name on m, handing the sign-in page to approve instead of
+// printing it.
+func ConfigBrowserSignIn(ctx context.Context, name string, m configmap.Mapper, approve func(signInURL string)) error {
+	out, err := configure(ctx, name, m, fs.ConfigIn{State: "browser"}, signInOptions{
+		showURL:      approve,
+		pollInterval: 10 * time.Millisecond,
+		pollTimeout:  10 * time.Second,
+	})
+	if err == nil && out != nil {
+		err = fmt.Errorf("config stopped in state %q", out.State)
+	}
+	return err
+}

backend/protondrive/fake_test.go

@@ -0,0 +1,75 @@
+package protondrive_test
+
+import (
+	"encoding/base64"
+	"net/http"
+	"os"
+	"testing"
+
+	"github.com/rclone/rclone/backend/protondrive"
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// fakeEnvPrefix starts the environment variables configuring the
+// TestProtonDriveFake: remote.
+const fakeEnvPrefix = "RCLONE_CONFIG_TESTPROTONDRIVEFAKE_"
+
+// setupFakeRemote starts a fake Proton server with one user and
+// configures the TestProtonDriveFake: remote, through environment
+// variables, with a saved session of that user.
+func setupFakeRemote(t *testing.T) (remoteName string, srv *protontest.Server, user *protontest.User) {
+	srv = protontest.New(t)
+	user = srv.AddUser("user", "pass")
+	sess := srv.NewSession(user)
+	t.Setenv(fakeEnvPrefix+"TYPE", "protondrive")
+	t.Setenv(fakeEnvPrefix+"ENDPOINT", srv.URL())
+	t.Setenv(fakeEnvPrefix+"CLIENT_UID", sess.UID)
+	t.Setenv(fakeEnvPrefix+"CLIENT_ACCESS_TOKEN", sess.AccessToken)
+	t.Setenv(fakeEnvPrefix+"CLIENT_REFRESH_TOKEN", sess.RefreshToken)
+	// The config holds the salted key password base64 encoded.
+	t.Setenv(fakeEnvPrefix+"CLIENT_SALTED_KEY_PASS", base64.StdEncoding.EncodeToString([]byte(user.KeyPassword())))
+	t.Setenv(fakeEnvPrefix+"DRAFT_CLIENT_UID", random.String(32))
+	return "TestProtonDriveFake:", srv, user
+}
+
+// TestSetupFakeRemote checks the configured session is accepted by the
+// fake server.
+func TestSetupFakeRemote(t *testing.T) {
+	remoteName, srv, user := setupFakeRemote(t)
+	assert.Equal(t, "TestProtonDriveFake:", remoteName)
+	assert.Equal(t, "protondrive", os.Getenv(fakeEnvPrefix+"TYPE"))
+	assert.Equal(t, srv.URL(), os.Getenv(fakeEnvPrefix+"ENDPOINT"))
+	assert.NotEmpty(t, os.Getenv(fakeEnvPrefix+"DRAFT_CLIENT_UID"))
+	keyPass, err := base64.StdEncoding.DecodeString(os.Getenv(fakeEnvPrefix + "CLIENT_SALTED_KEY_PASS"))
+	require.NoError(t, err)
+	assert.Equal(t, user.KeyPassword(), string(keyPass))
+	require.NotEmpty(t, os.Getenv(fakeEnvPrefix+"CLIENT_REFRESH_TOKEN"))
+
+	req, err := http.NewRequest("GET", os.Getenv(fakeEnvPrefix+"ENDPOINT")+"/core/v4/users", nil)
+	require.NoError(t, err)
+	req.Header.Set("x-pm-appversion", "external-drive-rclone@1.0.0-stable")
+	req.Header.Set("x-pm-uid", os.Getenv(fakeEnvPrefix+"CLIENT_UID"))
+	req.Header.Set("Authorization", "Bearer "+os.Getenv(fakeEnvPrefix+"CLIENT_ACCESS_TOKEN"))
+	resp, err := srv.Client().Do(req)
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+}
+
+// TestIntegrationFake runs integration tests against the fake server.
+func TestIntegrationFake(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	remoteName, _, _ := setupFakeRemote(t)
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:  remoteName,
+		NilObject:   (*protondrive.Object)(nil),
+		QuickTestOK: true,
+	})
+}

backend/protondrive/fuzz_internal_test.go

@@ -0,0 +1,367 @@
+package protondrive
+
+import (
+	"bytes"
+	"context"
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+	"unicode/utf16"
+	"unicode/utf8"
+
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/rclone/rclone/backend/protondrive/api"
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Fuzz targets for everything which parses bytes the server controls.
+// Each is seeded from the old library's fixtures in testdata/oldlib and
+// from payloads the fake server makes, and checks invariants of what
+// is accepted, not only that nothing panics.
+
+// oldlibFixture is the part of testdata/oldlib/file.json and
+// folder.json the fuzz targets use.
+type oldlibFixture struct {
+	NodeKey             string
+	NodePassphrasePlain string
+	NamePlain           string
+	ChildName           string
+	XAttr               string
+}
+
+// loadOldlibFixture returns testdata/oldlib/name.
+func loadOldlibFixture(t testing.TB, name string) oldlibFixture {
+	t.Helper()
+	data, err := os.ReadFile(filepath.Join("testdata/oldlib", name))
+	require.NoError(t, err)
+	var fx oldlibFixture
+	require.NoError(t, json.Unmarshal(data, &fx))
+	return fx
+}
+
+// fuzzLinks returns a client for a user of a fake server holding the
+// keys of the old library's v6 file fixture, the root folder's keys,
+// and the JSON of a link of each kind the client reads: the fixture
+// itself, and nodes the fake makes as SDK clients would.
+func fuzzLinks(f *testing.F) (*client, *nodeKeys, [][]byte) {
+	frozen := loadFrozenFile(f, filepath.Join(frozenFileDir, "v6"))
+	srv := protontest.New(f)
+	u := srv.AddUserWithKeys(frozen.Email, "pass", frozen.Keys)
+	c := newTestClientFor(f, srv, u, clientOptions{})
+	root := u.RootID()
+	ids := []string{
+		u.SeedStoredFile(root, frozen.Stored),
+		u.SeedFolder(root, "folder"),
+		u.SeedFile(root, "file.txt", []byte("content"), time.Now()),
+		u.SeedFile(root, "empty", nil, time.Now()),
+		u.SeedLegacyFile(root, "legacy", []byte("legacy")),
+		u.SeedAnonymousFile(root, "anonymous", []byte("anonymous"), time.Now()),
+		u.SeedAnonymousFolder(root, "anonymous folder"),
+		u.SeedFileClaimingBlockSizes(root, "claims", []byte("claims"), []int64{1, 2, 3}),
+		u.SeedDoc(root, "doc"),
+	}
+	ctx := context.Background()
+	links, err := c.fetchLinks(ctx, ids)
+	require.NoError(f, err)
+	require.Len(f, links, len(ids))
+	var seeds [][]byte
+	for _, d := range links {
+		data, err := json.Marshal(d)
+		require.NoError(f, err)
+		seeds = append(seeds, data)
+	}
+	keys, err := c.folderKeysOf(ctx, root)
+	require.NoError(f, err)
+	return c, keys, seeds
+}
+
+func FuzzValidateName(f *testing.F) {
+	for _, name := range []string{
+		loadOldlibFixture(f, "file.json").NamePlain,
+		loadOldlibFixture(f, "folder.json").NamePlain,
+		loadOldlibFixture(f, "folder.json").ChildName,
+		"file.txt", ".", "..", "...", "", "a/b", "/", "a\x00b", " ", "é", "é",
+		strings.Repeat("a", maxNameLength), strings.Repeat("a", maxNameLength+1),
+		strings.Repeat("\U0001F600", maxNameLength/2), strings.Repeat("\U0001F600", maxNameLength/2+1),
+		"\xff\xfe",
+	} {
+		f.Add(name)
+	}
+	f.Fuzz(func(t *testing.T, name string) {
+		if validateName(name) != nil {
+			return
+		}
+		assert.NotEmpty(t, name)
+		assert.True(t, utf8.ValidString(name), "%q", name)
+		assert.NotEqual(t, ".", name)
+		assert.NotEqual(t, "..", name)
+		assert.NotContains(t, name, "/")
+		assert.NotContains(t, name, "\x00")
+		assert.LessOrEqual(t, len(utf16.Encode([]rune(name))), maxNameLength)
+	})
+}
+
+func FuzzDecodeLink(f *testing.F) {
+	c, rootKeys, seeds := fuzzLinks(f)
+	for _, seed := range seeds {
+		f.Add(seed)
+		// Hostile values in links which otherwise decrypt.
+		var d api.LinkDetails
+		require.NoError(f, json.Unmarshal(seed, &d))
+		for _, change := range []func(d *api.LinkDetails){
+			func(d *api.LinkDetails) { d.Link.LinkID = "" },
+			func(d *api.LinkDetails) { d.Link.Type = 3 - d.Link.Type },
+			func(d *api.LinkDetails) {
+				if d.File != nil && d.File.ActiveRevision != nil {
+					d.File.ActiveRevision.EncryptedSize = -5
+				}
+			},
+		} {
+			changed := d
+			changed.File = cloneFile(d.File)
+			change(&changed)
+			data, err := json.Marshal(changed)
+			require.NoError(f, err)
+			f.Add(data)
+		}
+	}
+	// Shapes the client can't decrypt.
+	for _, format := range []string{"legacy", "v6"} {
+		data, err := os.ReadFile(filepath.Join(frozenFileDir, format, "link.json"))
+		require.NoError(f, err)
+		f.Add(data)
+	}
+	f.Add([]byte(`{}`))
+	f.Add([]byte(`{"Link":{"LinkID":"x","Type":1,"ParentLinkID":"p"},"Folder":{}}`))
+	ctx := context.Background()
+	f.Fuzz(func(t *testing.T, data []byte) {
+		var d api.LinkDetails
+		if json.Unmarshal(data, &d) != nil {
+			return
+		}
+		n, unreadable, err := c.decryptNode(ctx, &d, rootKeys)
+		if err != nil || unreadable != nil {
+			assert.Nil(t, n)
+			return
+		}
+		require.NotNil(t, n)
+		assert.NotEmpty(t, n.ID)
+		assert.Equal(t, d.Link.LinkID, n.ID)
+		assert.True(t, utf8.ValidString(n.Name), "%q", n.Name)
+		if d.Link.ParentLinkID != "" {
+			assert.NoError(t, validateName(n.Name))
+		}
+		assert.True(t, n.IsFolder == (d.Link.Type == api.LinkTypeFolder))
+		for _, original := range []bool{true, false} {
+			assert.GreaterOrEqual(t, n.size(original), int64(0), "original=%v", original)
+		}
+		if n.X != nil {
+			checkXAttrs(t, n.X)
+		}
+	})
+}
+
+// cloneFile returns a copy of file, sharing nothing changeable with it.
+func cloneFile(file *api.File) *api.File {
+	if file == nil {
+		return nil
+	}
+	out := *file
+	if file.ActiveRevision != nil {
+		rev := *file.ActiveRevision
+		out.ActiveRevision = &rev
+	}
+	return &out
+}
+
+// checkXAttrs checks the invariants of parsed extended attributes.
+//
+// That the block sizes add up to the size isn't one: as in the SDK
+// neither is checked against the other, and a read checks each block
+// against its recorded size, so a file whose attributes disagree fails
+// to read rather than reading wrong.
+func checkXAttrs(t *testing.T, x *xattrs) {
+	t.Helper()
+	assert.GreaterOrEqual(t, x.Size, int64(0))
+	if x.NoSize {
+		assert.Zero(t, x.Size)
+	}
+	if x.BlockSizes != nil {
+		assert.NotEmpty(t, x.BlockSizes)
+	}
+	for i, size := range x.BlockSizes {
+		assert.Positive(t, size, "block %d", i+1)
+	}
+	if !x.ModTime.IsZero() {
+		assert.Equal(t, time.UTC, x.ModTime.Location())
+		// RFC 3339 years, moved by a zone offset of under a day.
+		assert.True(t, x.ModTime.Year() >= -1 && x.ModTime.Year() <= 10000, "%v", x.ModTime)
+	}
+	assert.NotContains(t, x.Extra, "Common")
+}
+
+func FuzzCheckKeyS2K(f *testing.F) {
+	f.Add(unarmor(f, loadOldlibFixture(f, "file.json").NodeKey))
+	now := time.Now()
+	parent, signer := newTestKeyRing(f, now), newTestKeyRing(f, now)
+	_, _, _, keys := testNodeKeys(f, parent, signer, true, now)
+	for _, memExp := range []byte{3, 17, 18, 31} {
+		f.Add(unarmor(f, argon2Locked(f, keys, 1, 1, memExp)))
+	}
+	f.Fuzz(func(t *testing.T, data []byte) {
+		// Whatever the bytes, the check never runs a KDF and never
+		// panics; a key it lets through still parses or fails to.
+		if checkKeyS2K(data) == nil {
+			_, _ = crypto.NewKey(data)
+		}
+	})
+}
+
+func FuzzDecryptXAttrs(f *testing.F) {
+	fx := loadOldlibFixture(f, "file.json")
+	key, err := unlockKey(fx.NodeKey, []byte(fx.NodePassphrasePlain))
+	require.NoError(f, err)
+	nodeKR, err := crypto.NewKeyRing(key)
+	require.NoError(f, err)
+	now := time.Now()
+
+	// The old library's attributes, encrypted and in plain text.
+	res, _, _, err := decryptMessage(fx.XAttr, nodeKR, nil, now, maxXAttrsSize)
+	require.NoError(f, err)
+	f.Add(fx.XAttr, res.Bytes())
+	// Attributes as the fake writes them for SDK clients.
+	c, _, seeds := fuzzLinks(f)
+	for _, seed := range seeds {
+		var d api.LinkDetails
+		require.NoError(f, json.Unmarshal(seed, &d))
+		if d.File == nil || d.File.ActiveRevision == nil || d.File.ActiveRevision.XAttr == "" {
+			continue
+		}
+		n, err := c.getNode(context.Background(), d.Link.LinkID)
+		require.NoError(f, err)
+		res, _, _, err := decryptMessage(d.File.ActiveRevision.XAttr, n.keys.keyRing, nil, now, maxXAttrsSize)
+		require.NoError(f, err)
+		f.Add("", res.Bytes())
+	}
+	for _, plain := range []string{
+		`{"Common":{"ModificationTime":"2021-09-16T07:40:54+0000","Size":-1,"BlockSizes":[4194304,-1],"Digests":{"SHA1":"x"}}}`,
+		`{"Common":{"Size":1152921504606846976,"BlockSizes":[1152921504606846976,1]}}`,
+		`{"Common":{"ModificationTime":"99999-01-01T00:00:00Z"}}`,
+		`{"Common":{"ModificationTime":"0000-01-01T00:00:00+23:59"}}`,
+		`{"Common":{"BlockSizes":[]},"Media":{"Width":1}}`,
+		`{"Common":{"BlockSizes":[` + strings.Repeat("4194304,", 10000) + `1]}}`,
+		strings.Repeat(`{"Common":`, 20000) + strings.Repeat("}", 20000),
+		`{"Common":` + strings.Repeat("[", 100000) + strings.Repeat("]", 100000) + `}`,
+	} {
+		f.Add("", []byte(plain))
+	}
+
+	f.Fuzz(func(t *testing.T, enc string, plain []byte) {
+		if x, _, err := decryptXAttrs(enc, nodeKR, nodeKR, now); err == nil {
+			checkXAttrs(t, x)
+		}
+		if len(plain) > maxXAttrsSize/2 {
+			return
+		}
+		enc, err := encryptArmored(plain, nil, nodeKR, nodeKR, true, now)
+		require.NoError(t, err)
+		x, verifyErr, err := decryptXAttrs(enc, nodeKR, nodeKR, now)
+		if !utf8.Valid(plain) {
+			assert.Error(t, err)
+			return
+		}
+		require.NoError(t, err)
+		require.NoError(t, verifyErr)
+		checkXAttrs(t, x)
+	})
+}
+
+func FuzzDecodeAPIError(f *testing.F) {
+	for _, seed := range []struct {
+		status     int
+		body       string
+		retryAfter string
+	}{
+		{422, `{"Code":2500,"Error":"A file or folder with that name already exists","Details":{"ConflictLinkID":"abc","ConflictRevisionID":"def"}}`, ""},
+		{422, `{"Code":2501,"Error":"File or folder not found"}`, ""},
+		{429, `{"Code":85131,"Error":"Too many recent API requests"}`, "17"},
+		{429, ``, "Wed, 21 Oct 2015 07:28:00 GMT"},
+		{429, ``, "99999999999999999999999"},
+		{401, `{"Code":401,"Error":"Invalid access token"}`, ""},
+		{400, `{"Code":2064,"Error":"protontest: invalid x-pm-appversion \"x\""}`, ""},
+		{500, `<html>Internal Server Error</html>`, ""},
+		{200, `{"Code":"1000"}`, ""},
+		{0, `{"Error":null,"Details":[1,2,3]}`, "-5"},
+	} {
+		f.Add(seed.status, []byte(seed.body), seed.retryAfter)
+	}
+	f.Fuzz(func(t *testing.T, status int, body []byte, retryAfterValue string) {
+		resp := &http.Response{
+			StatusCode: status,
+			Status:     fmt.Sprintf("%d %s", status, http.StatusText(status)),
+			Header:     http.Header{"Retry-After": {retryAfterValue}},
+			Body:       io.NopCloser(bytes.NewReader(body)),
+		}
+		err := errorHandler(resp)
+		var apiErr *api.Error
+		require.ErrorAs(t, err, &apiErr)
+		assert.Equal(t, status, apiErr.Status)
+		assert.NotEmpty(t, apiErr.Message)
+		assert.NotEmpty(t, err.Error())
+		assert.Equal(t, apiErr.Code, apiErrorCode(err))
+
+		wait := retryAfter(resp)
+		assert.Positive(t, wait)
+		assert.LessOrEqual(t, wait, maxRetryAfter)
+	})
+}
+
+func FuzzDecryptForkPayload(f *testing.F) {
+	key := bytes.Repeat([]byte{7}, 32)
+	for _, seed := range []struct{ plain, aad string }{
+		{`{"type":"default","keyPassword":"the-key-password"}`, forkPayloadAAD},
+		{`{"keyPassword":""}`, forkPayloadAAD},
+		{`{"keyPassword":null}`, forkPayloadAAD},
+		{`{"type":"default"}`, forkPayloadAAD},
+		{`not json`, forkPayloadAAD},
+		{`{"keyPassword":"wrong additional data"}`, "other"},
+	} {
+		f.Add(sealForkPayload(f, key, seed.plain, seed.aad))
+	}
+	f.Add("")
+	f.Add("not base64!")
+	f.Add(base64.StdEncoding.EncodeToString(make([]byte, 12)))
+	f.Fuzz(func(t *testing.T, payload string) {
+		keyPassword, err := decryptForkPayload(payload, key)
+		if err != nil {
+			assert.Empty(t, keyPassword)
+			return
+		}
+		// Success only for a payload sealed with key: open it here
+		// independently.
+		blob, err := base64.StdEncoding.DecodeString(payload)
+		require.NoError(t, err)
+		block, err := aes.NewCipher(key)
+		require.NoError(t, err)
+		gcm, err := cipher.NewGCM(block)
+		require.NoError(t, err)
+		require.GreaterOrEqual(t, len(blob), gcm.NonceSize())
+		plain, err := gcm.Open(nil, blob[:gcm.NonceSize()], blob[gcm.NonceSize():], []byte(forkPayloadAAD))
+		require.NoError(t, err, "accepted a payload which doesn't authenticate")
+		var p struct{ KeyPassword *string }
+		require.NoError(t, json.Unmarshal(plain, &p))
+		require.NotNil(t, p.KeyPassword)
+		assert.Equal(t, *p.KeyPassword, keyPassword)
+	})
+}

backend/protondrive/protondrive_internal_test.go

@@ -1,97 +1,678 @@
 package protondrive
 
 import (
+	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
-	"regexp"
+	"io"
+	"maps"
+	"strings"
 	"testing"
+	"time"
 
-	"github.com/rclone/go-proton-api"
+	"github.com/pquerna/otp/totp"
+	"github.com/rclone/rclone/backend/protondrive/api"
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/object"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-var protonDriveAppVersionPattern = regexp.MustCompile(`(?i)^external-drive(-[a-z_]+)+@[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?-((stable|beta|RC|alpha)(([.-]?\d+)*)?)?([.-]?dev)?(\+.*)?$`)
+func TestTranslateError(t *testing.T) {
+	apiErr := func(code int) error {
+		return fmt.Errorf("call: %w", &api.Error{Status: 422, Code: code, Message: "proton says no"})
+	}
+	notFound := fmt.Errorf("get node: %w", errNotFound)
+	exists := fmt.Errorf("create: %w", errExists)
+	other := errors.New("other")
 
-func TestProtonDriveAppVersionFromRcloneVersion(t *testing.T) {
-	testCases := []struct {
-		name          string
-		rcloneVersion string
-		want          string
+	for _, test := range []struct {
+		name  string
+		err   error
+		op    errorOp
+		want  error // compared with ==, if set
+		check func(t *testing.T, got error)
 	}{
-		{
-			name:          "release",
-			rcloneVersion: "v1.73.5",
-			want:          "external-drive-rclone@1.73.5-stable",
-		},
-		{
-			name:          "dev build",
-			rcloneVersion: "v1.74.0-DEV",
-			want:          "external-drive-rclone@1.74.0-dev",
-		},
-		{
-			name:          "beta build with extra metadata",
-			rcloneVersion: "v1.74.0-beta.9519.990f33f2a.fix-protondrive-sdk-2026",
-			want:          "external-drive-rclone@1.74.0-beta.9519+990f33f2a.fix-protondrive-sdk-2026",
-		},
-		{
-			name:          "beta build with unsanitized branch name",
-			rcloneVersion: "v1.74.0-beta.9519.990f33f2a.fix/protondrive-sdk-2026",
-			want:          "external-drive-rclone@1.74.0-beta.9519+990f33f2a.fix-protondrive-sdk-2026",
-		},
-		{
-			name:          "invalid version falls back to stable",
-			rcloneVersion: "not-a-version",
-			want:          "external-drive-rclone@1.0.0-stable",
-		},
-	}
-
-	for _, testCase := range testCases {
-		t.Run(testCase.name, func(t *testing.T) {
-			got := protonDriveAppVersionFromRcloneVersion(testCase.rcloneVersion)
-
-			if got != testCase.want {
-				t.Fatalf("unexpected app version: got %q, want %q", got, testCase.want)
-			}
-			if !protonDriveAppVersionPattern.MatchString(got) {
-				t.Fatalf("app version %q does not match Proton pattern", got)
-			}
-		})
-	}
-}
-
-func TestShouldRetry(t *testing.T) {
-	ctx := context.Background()
-	cancelledCtx, cancel := context.WithCancel(ctx)
-	cancel()
-
-	apiErr := func(status int, code proton.Code) error {
-		return &proton.APIError{Status: status, Code: code, Message: "test"}
-	}
-
-	for _, tc := range []struct {
-		name      string
-		ctx       context.Context
-		err       error
-		wantRetry bool
-	}{
-		{"nil error", ctx, nil, false},
-		{"cancelled context", cancelledCtx, errors.New("some error"), false},
-		{"permanent validation error Code=200501 Status=422 (not retried)", ctx, apiErr(422, 200501), false},
-		{"transient storage block error Code=200501 Status=500 (retried)", ctx, apiErr(500, 200501), true},
-		{"server error Status=500", ctx, apiErr(500, 0), true},
-		{"server error Status=502", ctx, apiErr(502, 0), true},
-		{"server error Status=504", ctx, apiErr(504, 0), true},
-		{"server error Status=503 (handled by SDK, not retried here)", ctx, apiErr(503, 0), false},
-		{"rate limit Status=429 (handled by SDK, not retried here)", ctx, apiErr(429, 0), false},
-		{"client error Status=400", ctx, apiErr(400, 0), false},
-		{"client error Status=404", ctx, apiErr(404, 0), false},
-		{"wrapped API error retried via errors.As", ctx, fmt.Errorf("wrapped: %w", apiErr(500, 0)), true},
-		{"non-API error falls back to fserrors.ShouldRetry", ctx, errors.New("plain error"), false},
+		{name: "nil", err: nil, op: opObject, want: nil},
+		{name: "not found object", err: notFound, op: opObject, want: fs.ErrorObjectNotFound},
+		{name: "not found dir", err: notFound, op: opDir, want: fs.ErrorDirNotFound},
+		{name: "code 2501 object", err: apiErr(api.CodeNotExists), op: opObject, want: fs.ErrorObjectNotFound},
+		{name: "code 2501 dir", err: apiErr(api.CodeNotExists), op: opDir, want: fs.ErrorDirNotFound},
+		{name: "exists mkdir", err: exists, op: opMkdir, want: nil},
+		{name: "exists move", err: exists, op: opMove, want: fs.ErrorCantMove},
+		{name: "exists copy", err: exists, op: opCopy, want: fs.ErrorCantCopy},
+		{name: "exists dirmove", err: exists, op: opDirMove, want: fs.ErrorDirExists},
+		{name: "exists object", err: exists, op: opObject, check: func(t *testing.T, got error) {
+			assert.ErrorIs(t, got, errExists)
+		}},
+		{name: "other unchanged", err: other, op: opObject, want: other},
 	} {
-		t.Run(tc.name, func(t *testing.T) {
-			gotRetry, _ := shouldRetry(tc.ctx, tc.err)
-			assert.Equal(t, tc.wantRetry, gotRetry)
+		t.Run(test.name, func(t *testing.T) {
+			got := translateError(test.err, test.op)
+			if test.check != nil {
+				test.check(t, got)
+				return
+			}
+			assert.Equal(t, test.want, got)
+		})
+	}
+
+	for _, code := range []int{
+		api.CodeInsufficientQuota, api.CodeInsufficientSpace, api.CodeMaxFileSizeForFreeUser,
+		api.CodeMaxPublicEditModeForFreeUser, api.CodeTooManyChildren, api.CodeNestingTooDeep,
+	} {
+		t.Run(fmt.Sprintf("code %d no retry", code), func(t *testing.T) {
+			got := translateError(apiErr(code), opObject)
+			assert.True(t, fserrors.IsNoRetryError(got))
+			assert.False(t, fserrors.IsFatalError(got))
+			assert.Contains(t, got.Error(), "proton says no")
+		})
+	}
+	for _, code := range []int{api.CodeAppVersionMissing, api.CodeAppVersionBad} {
+		t.Run(fmt.Sprintf("code %d fatal", code), func(t *testing.T) {
+			got := translateError(apiErr(code), opObject)
+			assert.True(t, fserrors.IsFatalError(got))
+			assert.Contains(t, got.Error(), "upgrade rclone")
+			assert.Contains(t, got.Error(), "proton says no")
+		})
+	}
+	t.Run("code 2000 no retry", func(t *testing.T) {
+		got := translateError(apiErr(api.CodeInvalidRequirements), opMove)
+		assert.True(t, fserrors.IsNoRetryError(got))
+		assert.False(t, fserrors.IsFatalError(got))
+		assert.NotContains(t, got.Error(), "upgrade rclone")
+		assert.Contains(t, got.Error(), "proton says no")
+	})
+	t.Run("code 2028 not fatal", func(t *testing.T) {
+		got := translateError(apiErr(api.CodeFeatureBlocked), opObject)
+		assert.False(t, fserrors.IsFatalError(got))
+		assert.False(t, fserrors.IsNoRetryError(got))
+		assert.NotContains(t, got.Error(), "upgrade rclone")
+		assert.Contains(t, got.Error(), "proton says no")
+	})
+	for _, err := range []error{errIntegrity, errUploadMismatch, errUploadSuperseded} {
+		t.Run(err.Error()+" no retry", func(t *testing.T) {
+			got := translateError(fmt.Errorf("op: %w", err), opObject)
+			assert.True(t, fserrors.IsNoRetryError(got))
+			assert.ErrorIs(t, got, err)
+		})
+	}
+	for _, err := range []error{errUploadOutcomeUnknown, errUploadUnconfirmed} {
+		t.Run(err.Error()+" no low level retry", func(t *testing.T) {
+			got := translateError(fmt.Errorf("op: %w", err), opObject)
+			assert.True(t, fserrors.IsNoLowLevelRetryError(got))
+			assert.False(t, fserrors.IsNoRetryError(got))
+			assert.ErrorIs(t, got, err)
 		})
 	}
 }
+
+// sessionConfig returns the config keys of a v1.76 remote holding a new
+// saved session of u, pointed at srv.
+func sessionConfig(srv *protontest.Server, u *protontest.User) configmap.Simple {
+	sess := srv.NewSession(u)
+	return configmap.Simple{
+		"type":                   "protondrive",
+		"username":               "",
+		"password":               "",
+		"mailbox_password":       "",
+		"2fa":                    "",
+		"otp_secret_key":         "",
+		"client_uid":             sess.UID,
+		"client_access_token":    sess.AccessToken,
+		"client_refresh_token":   sess.RefreshToken,
+		"client_salted_key_pass": base64.StdEncoding.EncodeToString([]byte(u.KeyPassword())),
+		"encoding":               "Slash,LeftSpace,RightSpace,InvalidUtf8,Dot",
+		"original_file_size":     "true",
+		"app_version":            "",
+		"replace_existing_draft": "false",
+		"enable_caching":         "true",
+		// Not a v1.76 key, but the only way to reach the fake.
+		"endpoint": srv.URL(),
+	}
+}
+
+// newTestFs returns an Fs for the config m, shut down when the test ends.
+func newTestFs(t *testing.T, m configmap.Mapper) *Fs {
+	t.Helper()
+	f, err := NewFs(context.Background(), "test", "", m)
+	require.NoError(t, err)
+	t.Cleanup(func() { _ = f.(*Fs).Shutdown(context.Background()) })
+	return f.(*Fs)
+}
+
+// authRequests returns the auth requests made to srv.
+func authRequests(srv *protontest.Server) []string {
+	var out []string
+	for route := range srv.Requests() {
+		if strings.Contains(route, "/auth") {
+			out = append(out, route)
+		}
+	}
+	return out
+}
+
+func TestOptionsBackwardsCompatible(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	m := sessionConfig(srv, u)
+	want := m["client_access_token"]
+	f := newTestFs(t, m)
+
+	_, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+	assert.Empty(t, authRequests(srv), "a saved session must be used without signing in")
+	assert.Equal(t, want, m["client_access_token"])
+	assert.Len(t, m["draft_client_uid"], 32, "draft_client_uid is generated and saved")
+	assert.True(t, f.opt.ReportOriginalSize)
+	assert.True(t, f.opt.EnableCaching)
+}
+
+// unsavedMap is a config which saves nothing, as one from environment
+// variables or a connection string.
+type unsavedMap struct{ configmap.Simple }
+
+// Set does nothing.
+func (unsavedMap) Set(key, value string) {}
+
+func TestDraftClientUIDStableWhenUnsaved(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	uid := func(name string) string {
+		m := sessionConfig(srv, u)
+		f, err := NewFs(ctx, name, "", unsavedMap{m})
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = f.(*Fs).Shutdown(ctx) })
+		got := f.(*Fs).c.opt.DraftClientUID
+		assert.Len(t, got, 32)
+		return got
+	}
+	first := uid("env")
+	assert.Equal(t, first, uid("env"), "a later run of the same remote has another draft client ID")
+	assert.NotEqual(t, first, uid("other"), "another remote of the account has the same draft client ID")
+}
+
+func TestNewFsPasswordFallback(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	m := sessionConfig(srv, u)
+	m["username"] = "user"
+	m["password"] = obscure.MustObscure("pass")
+	oldUID := m["client_uid"]
+	srv.RevokeSession(oldUID)
+
+	f := newTestFs(t, m)
+	_, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+	assert.NotEqual(t, oldUID, m["client_uid"])
+	assert.NotEmpty(t, m["client_access_token"])
+	assert.NotEmpty(t, m["client_refresh_token"])
+	keyPass, err := base64.StdEncoding.DecodeString(m["client_salted_key_pass"])
+	require.NoError(t, err)
+	assert.Equal(t, u.KeyPassword(), string(keyPass))
+	assert.Equal(t, 1, srv.Requests()["POST /core/v4/auth"])
+}
+
+func TestNewFsPasswordFallbackTOTP(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	secret := u.EnableTOTP()
+	m := sessionConfig(srv, u)
+	for _, k := range []string{"client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass"} {
+		m[k] = ""
+	}
+	m["username"] = "user"
+	m["password"] = obscure.MustObscure("pass")
+	m["otp_secret_key"] = obscure.MustObscure(secret)
+
+	newTestFs(t, m)
+	assert.NotEmpty(t, m["client_uid"])
+	assert.Equal(t, 1, srv.Requests()["POST /core/v4/auth/2fa"])
+}
+
+func TestNewFsNoCredentials(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+
+	t.Run("none saved", func(t *testing.T) {
+		m := configmap.Simple{"endpoint": srv.URL()}
+		_, err := NewFs(context.Background(), "myremote", "", m)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
+	})
+	t.Run("revoked", func(t *testing.T) {
+		m := sessionConfig(srv, u)
+		srv.RevokeSession(m["client_uid"])
+		_, err := NewFs(context.Background(), "myremote", "", m)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
+	})
+	t.Run("undecodable key password", func(t *testing.T) {
+		m := sessionConfig(srv, u)
+		m["client_salted_key_pass"] = "not base64!"
+		_, err := NewFs(context.Background(), "myremote", "", m)
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), `rclone config reconnect myremote:`)
+	})
+}
+
+func TestTwoRemotesIsolated(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u1 := srv.AddUser("user1", "pass1")
+	u2 := srv.AddUser("user2", "pass2")
+	m1, m2 := sessionConfig(srv, u1), sessionConfig(srv, u2)
+	uid1, uid2 := m1["client_uid"], m2["client_uid"]
+	f1, f2 := newTestFs(t, m1), newTestFs(t, m2)
+	require.NoError(t, f1.Mkdir(ctx, "one"))
+	require.NoError(t, f2.Mkdir(ctx, "two"))
+
+	// Make both sessions refresh, saving new tokens.
+	access1, access2 := m1["client_access_token"], m2["client_access_token"]
+	srv.ResetRequests()
+	srv.ExpireAccessToken(uid1)
+	srv.ExpireAccessToken(uid2)
+	f1.DirCacheFlush()
+	f2.DirCacheFlush()
+	entries1, err := f1.List(ctx, "")
+	require.NoError(t, err)
+	entries2, err := f2.List(ctx, "")
+	require.NoError(t, err)
+
+	require.Len(t, entries1, 1)
+	assert.Equal(t, "one", entries1[0].Remote())
+	require.Len(t, entries2, 1)
+	assert.Equal(t, "two", entries2[0].Remote())
+	assert.Equal(t, 2, srv.Requests()["POST /auth/v4/refresh"])
+	assert.NotEqual(t, access1, m1["client_access_token"], "remote 1 saved its refreshed token")
+	assert.NotEqual(t, access2, m2["client_access_token"], "remote 2 saved its refreshed token")
+	assert.Equal(t, uid1, m1["client_uid"])
+	assert.Equal(t, uid2, m2["client_uid"])
+	for _, k := range []string{"client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass", "draft_client_uid"} {
+		assert.NotEqual(t, m1[k], m2[k], k)
+	}
+}
+
+// configAnswers drives configure through its states, answering each
+// question by its state from answers, and returns the questions asked.
+func configAnswers(t *testing.T, m configmap.Mapper, signIn signInOptions, answers map[string]string) []*fs.ConfigOut {
+	t.Helper()
+	var asked []*fs.ConfigOut
+	in := fs.ConfigIn{}
+	for range 10 {
+		out, err := configure(context.Background(), "test", m, in, signIn)
+		require.NoError(t, err)
+		if out == nil {
+			return asked
+		}
+		require.Empty(t, out.Error)
+		in = fs.ConfigIn{State: out.State}
+		if out.Option != nil {
+			asked = append(asked, out)
+			answer, ok := answers[out.State]
+			require.True(t, ok, "unexpected question in state %q: %s", out.State, out.Option.Help)
+			in.Result = answer
+		}
+	}
+	t.Fatal("config didn't finish")
+	return nil
+}
+
+// requireSession checks m holds a working session of u on srv.
+func requireSession(t *testing.T, srv *protontest.Server, m configmap.Simple, u *protontest.User) {
+	t.Helper()
+	keyPass, err := base64.StdEncoding.DecodeString(m["client_salted_key_pass"])
+	require.NoError(t, err)
+	assert.Equal(t, u.KeyPassword(), string(keyPass))
+	srv.ResetRequests()
+	newTestFs(t, m)
+	assert.Empty(t, authRequests(srv))
+}
+
+func TestConfigBrowser(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	m := configmap.Simple{"endpoint": srv.URL()}
+	var shown string
+	signIn := signInOptions{
+		pollInterval: 10 * time.Millisecond,
+		pollTimeout:  10 * time.Second,
+		showURL: func(signInURL string) {
+			shown = signInURL
+			require.NoError(t, srv.ApproveFork(signInURL, u))
+		},
+	}
+
+	asked := configAnswers(t, m, signIn, map[string]string{"choose": "true"})
+	require.Len(t, asked, 1)
+	assert.Contains(t, asked[0].Option.Help, "rclone is a third-party application not officially supported by Proton.")
+	assert.Equal(t, true, asked[0].Option.Default)
+	assert.True(t, strings.HasPrefix(shown, forkSignInURL), shown)
+	requireSession(t, srv, m, u)
+}
+
+func TestConfigPassword(t *testing.T) {
+	srv := protontest.New(t)
+
+	t.Run("no 2fa", func(t *testing.T) {
+		u := srv.AddUser("user", "pass")
+		m := configmap.Simple{"endpoint": srv.URL(), "username": "user", "password": obscure.MustObscure("pass")}
+		asked := configAnswers(t, m, signInOptions{}, map[string]string{"choose": "false"})
+		require.Len(t, asked, 1)
+		assert.Equal(t, false, asked[0].Option.Default, "password sign-in is the default when a password is set")
+		requireSession(t, srv, m, u)
+	})
+	t.Run("2fa asked", func(t *testing.T) {
+		u := srv.AddUser("user2fa", "pass")
+		secret := u.EnableTOTP()
+		m := configmap.Simple{"endpoint": srv.URL(), "username": "user2fa", "password": obscure.MustObscure("pass")}
+		code, err := totp.GenerateCode(secret, time.Now())
+		require.NoError(t, err)
+		asked := configAnswers(t, m, signInOptions{}, map[string]string{"choose": "false", "password_2fa": code})
+		require.Len(t, asked, 2)
+		requireSession(t, srv, m, u)
+		assert.Empty(t, m["2fa"], "the 2FA code is not saved")
+	})
+	t.Run("captcha", func(t *testing.T) {
+		u := srv.AddUser("usercaptcha", "pass")
+		srv.RequireCaptcha(u)
+		m := configmap.Simple{"endpoint": srv.URL(), "username": "usercaptcha", "password": obscure.MustObscure("pass")}
+		out, err := configure(context.Background(), "test", m, fs.ConfigIn{State: "password"}, signInOptions{})
+		require.NoError(t, err)
+		require.NotNil(t, out)
+		assert.Contains(t, out.Error, "web browser")
+		assert.Equal(t, "method", out.State, "goes back to the choice of sign-in")
+		assert.Empty(t, m["client_uid"])
+	})
+	t.Run("no password", func(t *testing.T) {
+		m := configmap.Simple{"endpoint": srv.URL()}
+		out, err := configure(context.Background(), "test", m, fs.ConfigIn{State: "password"}, signInOptions{})
+		require.NoError(t, err)
+		require.NotNil(t, out)
+		assert.NotEmpty(t, out.Error)
+	})
+}
+
+func TestConfigStore(t *testing.T) {
+	creds := credentials{UID: "uid", AccessToken: "access", RefreshToken: "refresh", KeyPassword: "key\x00pass"}
+
+	t.Run("round trip", func(t *testing.T) {
+		m := configmap.Simple{}
+		s := &configStore{m: m}
+		s.Save(creds)
+		assert.Equal(t, base64.StdEncoding.EncodeToString([]byte(creds.KeyPassword)), m[clientSaltedKeyPassKey])
+		assert.Equal(t, creds, s.Load())
+		// Another user of the config saves newer tokens.
+		m[clientRefreshTokenKey] = "newer"
+		assert.Equal(t, "newer", s.Load().RefreshToken)
+	})
+	t.Run("overridden", func(t *testing.T) {
+		// Values from the environment or the command line take
+		// precedence over the config file the store saves to.
+		env := configmap.Simple{clientUIDKey: "old uid", clientAccessTokenKey: "old access", clientRefreshTokenKey: "old refresh"}
+		file := configmap.Simple{}
+		m := configmap.New().AddGetter(env, configmap.PriorityNormal).AddGetter(file, configmap.PriorityConfig).AddSetter(file)
+		s := &configStore{m: m}
+		assert.Equal(t, "old refresh", s.Load().RefreshToken)
+		s.Save(creds)
+		assert.Equal(t, creds, s.Load(), "saved credentials must not be shadowed by spent ones")
+	})
+}
+
+// putFile uploads content to f at remote and returns the object.
+func putFile(t *testing.T, f fs.Fs, remote string, content []byte) fs.Object {
+	t.Helper()
+	src := object.NewStaticObjectInfo(remote, time.Now(), int64(len(content)), true, nil, nil)
+	o, err := f.Put(context.Background(), bytes.NewReader(content), src)
+	require.NoError(t, err)
+	return o
+}
+
+// readObject returns the content of o read with options.
+func readObject(t *testing.T, o fs.Object, options ...fs.OpenOption) ([]byte, error) {
+	t.Helper()
+	rc, err := o.Open(context.Background(), options...)
+	if err != nil {
+		return nil, err
+	}
+	defer func() { require.NoError(t, rc.Close()) }()
+	return io.ReadAll(rc)
+}
+
+func TestRmdirRefusesHiddenChildren(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	f := newTestFs(t, sessionConfig(srv, u))
+
+	require.NoError(t, f.Mkdir(ctx, "docs"))
+	docsID, err := f.dirCache.FindDir(ctx, "docs", false)
+	require.NoError(t, err)
+	u.SeedDoc(docsID, "notes")
+	assert.Equal(t, fs.ErrorDirectoryNotEmpty, f.Rmdir(ctx, "docs"))
+
+	// Filled by another client after an empty listing was cached.
+	require.NoError(t, f.Mkdir(ctx, "later"))
+	entries, err := f.List(ctx, "later")
+	require.NoError(t, err)
+	require.Empty(t, entries)
+	laterID, err := f.dirCache.FindDir(ctx, "later", false)
+	require.NoError(t, err)
+	u.SeedFile(laterID, "new.txt", []byte("x"), time.Now())
+	assert.Equal(t, fs.ErrorDirectoryNotEmpty, f.Rmdir(ctx, "later"))
+
+	require.NoError(t, f.Mkdir(ctx, "empty"))
+	require.NoError(t, f.Rmdir(ctx, "empty"))
+	paths := map[string]bool{}
+	for _, d := range u.Dump() {
+		paths[d.Path] = d.State == api.LinkStateActive
+	}
+	assert.True(t, paths["docs/notes"])
+	assert.True(t, paths["later/new.txt"])
+}
+
+func TestConfigKeepsSession(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	ri, err := fs.Find("protondrive")
+	require.NoError(t, err)
+
+	t.Run("default", func(t *testing.T) {
+		m := sessionConfig(srv, u)
+		want := maps.Clone(m)
+		srv.ResetRequests()
+		asked := configAnswers(t, m, signInOptions{}, map[string]string{"reauth": "false"})
+		require.Len(t, asked, 1)
+		assert.Equal(t, false, asked[0].Option.Default)
+		assert.Equal(t, "config_refresh_token", asked[0].Option.Name)
+		assert.Empty(t, srv.Requests())
+		assert.Equal(t, want, m)
+	})
+	t.Run("config update", func(t *testing.T) {
+		// As "rclone config update remote enable_caching=false" runs it.
+		m := sessionConfig(srv, u)
+		srv.ResetRequests()
+		out, err := fs.BackendConfig(context.Background(), "test", m, ri, configmap.Simple{"config_refresh_token": "false"}, fs.ConfigIn{})
+		require.NoError(t, err)
+		assert.Nil(t, out)
+		assert.Empty(t, srv.Requests())
+	})
+}
+
+func TestConfigSignInAgain(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	m := sessionConfig(srv, u)
+	m["username"] = "user"
+	m["password"] = obscure.MustObscure("pass")
+	old := maps.Clone(m)
+
+	configAnswers(t, m, signInOptions{}, map[string]string{"reauth": "true", "choose": "false"})
+	assert.NotEqual(t, old["client_uid"], m["client_uid"])
+	assert.Equal(t, 1, srv.Requests()["DELETE /core/v4/auth"], "the old session is signed out")
+	requireSession(t, srv, m, u)
+
+	// The old session no longer works.
+	old["username"], old["password"] = "", ""
+	_, err := NewFs(context.Background(), "test", "", old)
+	require.Error(t, err)
+}
+
+func TestShutdownKeepsFsUsable(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	f := newTestFs(t, sessionConfig(srv, u))
+	o := putFile(t, f, "dir/a.txt", []byte("hello"))
+
+	require.NoError(t, f.Shutdown(ctx))
+	require.NoError(t, f.Shutdown(ctx))
+	got, err := readObject(t, o)
+	require.NoError(t, err)
+	assert.Equal(t, "hello", string(got))
+	putFile(t, f, "dir/b.txt", []byte("world"))
+	entries, err := f.List(ctx, "dir")
+	require.NoError(t, err)
+	assert.Len(t, entries, 2)
+	o, err = f.NewObject(ctx, "dir/b.txt")
+	require.NoError(t, err)
+	got, err = readObject(t, o)
+	require.NoError(t, err)
+	assert.Equal(t, "world", string(got))
+}
+
+func TestDisconnect(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	m := sessionConfig(srv, u)
+	f := newTestFs(t, m)
+	o := putFile(t, f, "a.txt", []byte("hello"))
+
+	require.NoError(t, f.Disconnect(ctx))
+	assert.Empty(t, m["client_access_token"])
+	assert.Empty(t, m["client_refresh_token"])
+
+	_, err := f.List(ctx, "")
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `rclone config reconnect test:`)
+	_, err = readObject(t, o)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `rclone config reconnect test:`)
+	_, err = f.Put(ctx, bytes.NewReader([]byte("x")), object.NewStaticObjectInfo("b.txt", time.Now(), 1, true, nil, nil))
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), `rclone config reconnect test:`)
+}
+
+func TestSessionExpiredNamesRemote(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	m := sessionConfig(srv, u)
+	f := newTestFs(t, m)
+	f.c.s.adoptWait = 0
+	srv.RevokeSession(m["client_uid"])
+
+	_, err := f.List(ctx, "")
+	require.ErrorIs(t, err, errSessionExpired)
+	assert.Contains(t, err.Error(), `rclone config reconnect test:`)
+	assert.NotContains(t, err.Error(), "remote:")
+}
+
+func TestTrashedBehindTheFsBack(t *testing.T) {
+	ctx := context.Background()
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	f := newTestFs(t, sessionConfig(srv, u))
+	other := newTestFs(t, sessionConfig(srv, u))
+
+	o := putFile(t, f, "a.txt", []byte("hello"))
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	_, err := f.List(ctx, "")
+	require.NoError(t, err)
+	trash := func(remote string) {
+		otherObj, err := other.NewObject(ctx, remote)
+		if err == nil {
+			require.NoError(t, otherObj.Remove(ctx))
+			return
+		}
+		require.NoError(t, other.Purge(ctx, remote))
+	}
+	trash("a.txt")
+	trash("dir")
+
+	src := object.NewStaticObjectInfo("a.txt", time.Now(), 1, true, nil, nil)
+	assert.Equal(t, fs.ErrorObjectNotFound, o.Update(ctx, bytes.NewReader([]byte("x")), src))
+	_, err = f.Move(ctx, o, "b.txt")
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
+	_, err = f.Copy(ctx, o, "c.txt")
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
+	assert.Equal(t, fs.ErrorDirNotFound, f.DirMove(ctx, f, "dir", "dir2"))
+	for _, d := range u.Dump() {
+		assert.NotContains(t, []string{"b.txt", "c.txt", "dir2"}, d.Path)
+	}
+}
+
+func TestOpenRange(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	content := []byte("0123456789abcdefghijklmnopqrstuvwxyz")
+
+	// A file without extended attributes, whose plaintext size only
+	// its content tells.
+	seeded := u.SeedFile(u.RootID(), "tmp", content, time.Now())
+	stored := u.StoredFile(seeded)
+	stored.Revision.XAttr = ""
+	m := sessionConfig(srv, u)
+	cleanup := newTestFs(t, m)
+	o, err := cleanup.NewObject(context.Background(), "tmp")
+	require.NoError(t, err)
+	require.NoError(t, o.Remove(context.Background()))
+	u.SeedStoredFile(u.RootID(), stored)
+
+	for _, original := range []bool{true, false} {
+		t.Run(fmt.Sprint("original_file_size=", original), func(t *testing.T) {
+			m := sessionConfig(srv, u)
+			m["original_file_size"] = fmt.Sprint(original)
+			f := newTestFs(t, m)
+			put := putFile(t, f, fmt.Sprintf("put-%v.txt", original), content)
+			bare, err := f.NewObject(context.Background(), "tmp")
+			require.NoError(t, err)
+			for _, o := range []fs.Object{put, bare} {
+				for _, test := range []struct {
+					opt  fs.OpenOption
+					want string
+				}{
+					{&fs.RangeOption{Start: -1, End: 5}, "vwxyz"},
+					{&fs.RangeOption{Start: 30, End: -1}, "uvwxyz"},
+					{&fs.RangeOption{Start: 30, End: 1000}, "uvwxyz"},
+					{&fs.RangeOption{Start: 2, End: 4}, "234"},
+					{&fs.SeekOption{Offset: 33}, "xyz"},
+					{&fs.RangeOption{Start: -1, End: 1000}, string(content)},
+				} {
+					got, err := readObject(t, o, test.opt)
+					require.NoError(t, err)
+					assert.Equal(t, test.want, string(got), "%s %v", o.Remote(), test.opt)
+				}
+			}
+		})
+	}
+}
+
+// TestPutNestedReadBackLag puts a file into new nested directories
+// while the server takes a moment to show new nodes.
+func TestPutNestedReadBackLag(t *testing.T) {
+	srv := protontest.New(t)
+	u := srv.AddUser("user", "pass")
+	f := newTestFs(t, sessionConfig(srv, u))
+	srv.SetReadLag(2, protontest.LagMissing)
+	o := putFile(t, f, "A1/B1/C1/one", []byte("aaa"))
+	assert.Equal(t, int64(3), o.Size())
+}

backend/protondrive/protondrive_test.go

@@ -1,6 +1,7 @@
 package protondrive_test
 
 import (
+	"flag"
 	"testing"
 
 	"github.com/rclone/rclone/backend/protondrive"
@@ -9,6 +10,13 @@ import (
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
+	// A golden capture runs with the real config, where this test would
+	// write to and delete from the live remote if -run let it through.
+	for _, name := range []string{"capture", "capture-remote"} {
+		if f := flag.Lookup(name); f != nil && f.Value.String() != "" {
+			t.Skipf("-%s is set: not running integration tests during a golden capture", name)
+		}
+	}
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestProtonDrive:",
 		NilObject:  (*protondrive.Object)(nil),

backend/protondrive/protontest/account.go

@@ -0,0 +1,340 @@
+package protontest
+
+import (
+	"encoding/base64"
+	"encoding/hex"
+	"fmt"
+	"net/http"
+
+	"github.com/ProtonMail/go-srp"
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/rclone/rclone/backend/protondrive/api"
+)
+
+// userKey is a user key with the password salt which locks it.
+type userKey struct {
+	id          string
+	salt        string // base64
+	keyPassword string // derived from the mailbox password and salt
+	armored     string // private key locked with keyPassword
+	key         *crypto.Key
+}
+
+// address is an email address of a user.
+type address struct {
+	id    string
+	email string
+	keys  []*addressKey // the primary key is first
+}
+
+// addressKey is an address key, locked with a random passphrase which
+// is stored encrypted to and signed by the primary user key.
+type addressKey struct {
+	id         string
+	passphrase []byte
+	armored    string // private key locked with passphrase
+	token      string // armored passphrase encrypted to the user key
+	signature  string // armored detached signature of passphrase by the user key
+	key        *crypto.Key
+}
+
+// mustCrypto panics with a message naming what failed if err is not nil.
+//
+// Key operations on keys the fake generated itself fail only on a bug.
+func mustCrypto(what string, err error) {
+	if err != nil {
+		panic(fmt.Sprintf("protontest: %s: %v", what, err))
+	}
+}
+
+// generateKey returns a new unlocked Curve25519 key for email, which
+// is fast to generate unlike RSA.
+func generateKey(name, email string) *crypto.Key {
+	key, err := crypto.PGP().KeyGeneration().AddUserId(name, email).GenerationTime(signTime()).New().GenerateKey()
+	mustCrypto("generate key", err)
+	return key
+}
+
+// lockKey returns key armored and locked with passphrase.
+func lockKey(key *crypto.Key, passphrase []byte) string {
+	locked, err := crypto.PGP().LockKey(key, passphrase)
+	mustCrypto("lock key", err)
+	armored, err := locked.Armor()
+	mustCrypto("armor key", err)
+	return armored
+}
+
+// newUserKey returns a new user key for u locked with a key password
+// derived from u's mailbox password and a new salt.
+func newUserKey(u *User) *userKey {
+	rawSalt := randomBytes(16)
+	hashed, err := srp.MailboxPassword([]byte(u.MailboxPassword), rawSalt)
+	mustCrypto("hash mailbox password", err)
+	k := &userKey{
+		id:          randomID(),
+		salt:        base64.StdEncoding.EncodeToString(rawSalt),
+		keyPassword: string(hashed[len(hashed)-31:]),
+		key:         generateKey(u.Username, u.Username),
+	}
+	k.armored = lockKey(k.key, []byte(k.keyPassword))
+	return k
+}
+
+// newAddress returns a new address with one address key whose
+// passphrase is encrypted to userKey.
+func newAddress(email string, userKey *userKey) *address {
+	passphrase := []byte(hex.EncodeToString(randomBytes(32)))
+	key := generateKey(email, email)
+	k := &addressKey{
+		id:         randomID(),
+		passphrase: passphrase,
+		armored:    lockKey(key, passphrase),
+		key:        key,
+	}
+	k.sealPassphrase(userKey)
+	return &address{id: randomID(), email: email, keys: []*addressKey{k}}
+}
+
+// sealPassphrase sets the token and signature of k to its passphrase
+// encrypted to and signed by userKey, as Proton stores it.
+func (k *addressKey) sealPassphrase(userKey *userKey) {
+	eh, err := crypto.PGP().Encryption().Recipient(userKey.key).New()
+	mustCrypto("address key token", err)
+	msg, err := eh.Encrypt(k.passphrase)
+	mustCrypto("address key token", err)
+	token, err := msg.ArmorBytes()
+	mustCrypto("address key token", err)
+	sh, err := crypto.PGP().Sign().SigningKey(userKey.key).Detached().SignTime(signTime()).New()
+	mustCrypto("address key signature", err)
+	sig, err := sh.Sign(k.passphrase, crypto.Armor)
+	mustCrypto("address key signature", err)
+	k.token, k.signature = string(token), string(sig)
+}
+
+// KeyPassword returns the password which unlocks u's primary user key:
+// the last 31 characters of the bcrypt hash of the mailbox password
+// and the key salt, as a client derives it.
+func (u *User) KeyPassword() string {
+	u.srv.mu.Lock()
+	defer u.srv.mu.Unlock()
+	return u.keyPassword
+}
+
+// Email returns u's primary address.
+func (u *User) Email() string {
+	u.srv.mu.Lock()
+	defer u.srv.mu.Unlock()
+	return u.addresses[0].email
+}
+
+// AddAddress adds an address with a new address key to u.
+func (u *User) AddAddress(email string) {
+	u.srv.mu.Lock()
+	primary := u.keys[len(u.keys)-1]
+	u.srv.mu.Unlock()
+	addr := newAddress(email, primary)
+	u.srv.mu.Lock()
+	defer u.srv.mu.Unlock()
+	u.addresses = append(u.addresses, addr)
+}
+
+// AddUserKey gives u a new primary user key with a new salt, so
+// KeySalt and KeyPassword change.
+//
+// The old keys are listed before the new one and stay locked with
+// their old key password, as after a key password change on an account
+// with legacy keys. The address key passphrases are re-encrypted to the
+// new key.
+func (u *User) AddUserKey() {
+	key := newUserKey(u)
+	u.srv.mu.Lock()
+	defer u.srv.mu.Unlock()
+	u.keys = append(u.keys, key)
+	u.KeySalt, u.keyPassword = key.salt, key.keyPassword
+	for _, addr := range u.addresses {
+		for _, k := range addr.keys {
+			k.sealPassphrase(key)
+		}
+	}
+}
+
+// CorruptAddressKeySignature replaces the Signature of every key of u's
+// address email, as a malicious server would: with a valid signature
+// by an unrelated key, or with nothing if empty is true. AddUserKey
+// restores the signatures.
+func (s *Server) CorruptAddressKeySignature(u *User, email string, empty bool) {
+	impostor := generateKey("impostor", "impostor@proton.test")
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	addr := u.addressByEmailLocked(email)
+	if addr == nil {
+		panic("protontest: no address " + email)
+	}
+	for _, k := range addr.keys {
+		if empty {
+			k.signature = ""
+			continue
+		}
+		// Sign the real passphrase, so only the signer is wrong.
+		sh, err := crypto.PGP().Sign().SigningKey(impostor).Detached().SignTime(signTime()).New()
+		mustCrypto("impostor signature", err)
+		sig, err := sh.Sign(k.passphrase, crypto.Armor)
+		mustCrypto("impostor signature", err)
+		k.signature = string(sig)
+	}
+}
+
+// EncryptToAddress returns plaintext as an armored message encrypted to
+// the primary key of u's address email.
+func (u *User) EncryptToAddress(email string, plaintext []byte) string {
+	u.srv.mu.Lock()
+	addr := u.addressByEmailLocked(email)
+	u.srv.mu.Unlock()
+	if addr == nil {
+		panic("protontest: no address " + email)
+	}
+	eh, err := crypto.PGP().Encryption().Recipient(addr.keys[0].key).New()
+	mustCrypto("encrypt to address", err)
+	msg, err := eh.Encrypt(plaintext)
+	mustCrypto("encrypt to address", err)
+	armored, err := msg.ArmorBytes()
+	mustCrypto("encrypt to address", err)
+	return string(armored)
+}
+
+// addressByEmailLocked returns u's address email or nil. srv.mu must
+// be held.
+func (u *User) addressByEmailLocked(email string) *address {
+	for _, addr := range u.addresses {
+		if addr.email == email {
+			return addr
+		}
+	}
+	return nil
+}
+
+// boolInt returns 1 for true and 0 for false, as the API encodes flags.
+func boolInt(b bool) int {
+	if b {
+		return 1
+	}
+	return 0
+}
+
+// registerAccount registers the user, address and public key routes.
+func (s *Server) registerAccount() {
+	s.handleAuth("GET", "/core/v4/users", s.handleUser)
+	s.handleAuth("GET", "/core/v4/addresses", s.handleAddresses)
+	s.handleAuth("GET", "/core/v4/keys/all", s.handlePublicKeys)
+}
+
+// userWithQuota is api.User with the quota fields of Proton's User
+// schema (coreTypes.ts).
+type userWithQuota struct {
+	api.User
+	UsedSpace        int64
+	MaxSpace         int64
+	ProductUsedSpace productUsedSpace
+}
+
+// productUsedSpace is the UserUsage schema: used space per product.
+type productUsedSpace struct {
+	Calendar int64
+	Contact  int64
+	Drive    int64
+	Mail     int64
+	Pass     int64
+	Lumo     int64
+}
+
+// handleUser returns the user, its keys and its quota, all used space
+// being Drive's.
+func (s *Server) handleUser(w http.ResponseWriter, r *http.Request, u *User) {
+	s.mu.Lock()
+	user := userWithQuota{
+		User:     api.User{ID: u.ID, Name: u.Username, Email: u.addresses[0].email},
+		MaxSpace: DefaultMaxSpace,
+	}
+	for i, key := range u.keys {
+		user.Keys = append(user.Keys, api.UserKey{
+			ID:         key.id,
+			Version:    3,
+			PrivateKey: key.armored,
+			Primary:    boolInt(i == len(u.keys)-1),
+			Active:     1,
+		})
+	}
+	if u.drive != nil {
+		user.UsedSpace = u.drive.usedSpace()
+		user.ProductUsedSpace.Drive = user.UsedSpace
+	}
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, &struct {
+		Code int
+		User userWithQuota
+	}{Code: api.CodeOK, User: user})
+}
+
+// handleAddresses returns the user's addresses with their keys.
+func (s *Server) handleAddresses(w http.ResponseWriter, r *http.Request, u *User) {
+	s.mu.Lock()
+	resp := &api.AddressesResponse{Code: api.CodeOK}
+	for i, addr := range u.addresses {
+		out := api.Address{ID: addr.id, Email: addr.email, Status: 1, Type: 1, Priority: i + 1}
+		for j, key := range addr.keys {
+			out.Keys = append(out.Keys, api.AddressKey{
+				ID:         key.id,
+				Version:    3,
+				PrivateKey: key.armored,
+				Token:      key.token,
+				Signature:  key.signature,
+				Primary:    boolInt(j == 0),
+				Active:     1,
+				Flags:      3,
+			})
+		}
+		resp.Addresses = append(resp.Addresses, out)
+	}
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// handlePublicKeys returns the public keys of the address in the Email
+// query parameter, or HTTP 422 code 33102 if no user has it.
+func (s *Server) handlePublicKeys(w http.ResponseWriter, r *http.Request, _ *User) {
+	email := r.URL.Query().Get("Email")
+	if email == "" {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: no Email")
+		return
+	}
+	s.mu.Lock()
+	var addr *address
+	if owner := s.keysAs[email]; owner != nil {
+		addr = owner.addresses[0]
+	}
+	for _, u := range s.users {
+		if addr != nil {
+			break
+		}
+		addr = u.addressByEmailLocked(email)
+	}
+	var keys []*addressKey
+	if addr != nil {
+		keys = addr.keys
+	}
+	s.mu.Unlock()
+	if addr == nil {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeAddressMissing, "protontest: address does not exist")
+		return
+	}
+	resp := &api.PublicKeysResponse{Code: api.CodeOK}
+	for i, key := range keys {
+		pub, err := key.key.GetArmoredPublicKey()
+		if err != nil {
+			writeError(w, http.StatusInternalServerError, 0, "protontest: public key: "+err.Error())
+			return
+		}
+		resp.Address.Keys = append(resp.Address.Keys, api.PublicKey{PublicKey: pub, Primary: boolInt(i == 0), Flags: 3})
+	}
+	writeJSON(w, http.StatusOK, resp)
+}

backend/protondrive/protontest/auth.go

@@ -0,0 +1,555 @@
+package protontest
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/ProtonMail/go-srp"
+	"github.com/pquerna/otp/totp"
+	"github.com/rclone/rclone/backend/protondrive/api"
+)
+
+// signedModulus is a 2048-bit SRP modulus signed by Proton's modulus
+// key, from go-srp's tests.
+const signedModulus = `-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA256
+
+W2z5HBi8RvsfYzZTS7qBaUxxPhsfHJFZpu3Kd6s1JafNrCCH9rfvPLrfuqocxWPgWDH2R8neK7PkNvjxto9TStuY5z7jAzWRvFWN9cQhAKkdWgy0JY6ywVn22+HFpF4cYesHrqFIKUPDMSSIlWjBVmEJZ/MusD44ZT29xcPrOqeZvwtCffKtGAIjLYPZIEbZKnDM1Dm3q2K/xS5h+xdhjnndhsrkwm9U9oyA2wxzSXFL+pdfj2fOdRwuR5nW0J2NFrq3kJjkRmpO/Genq1UW+TEknIWAb6VzJJJA244K/H8cnSx2+nSNZO3bbo6Ys228ruV9A8m6DhxmS+bihN3ttQ==
+-----BEGIN PGP SIGNATURE-----
+Version: ProtonMail
+Comment: https://protonmail.com
+
+wl4EARYIABAFAlwB1j0JEDUFhcTpUY8mAAD8CgEAnsFnF4cF0uSHKkXa1GIa
+GO86yMV4zDZEZcDSJo0fgr8A/AlupGN9EdHlsrZLmTA1vhIx+rOgxdEff28N
+kvNM7qIK
+=q6vu
+-----END PGP SIGNATURE-----`
+
+// srpBits is the bit length of the SRP modulus.
+const srpBits = 2048
+
+// forkClientID is the client ID rclone's sign-in URL must carry.
+const forkClientID = "external-drive"
+
+// User is an account on the fake server.
+type User struct {
+	ID              string
+	Username        string
+	Password        string // login password, checked with SRP
+	MailboxPassword string // derives the key password
+	KeySalt         string // base64 salt of the primary user key's password
+
+	srv     *Server
+	srpSalt []byte
+
+	// Guarded by srv.mu once the user is added.
+	keyPassword string
+	keys        []*userKey // the primary key is last
+	addresses   []*address // the primary address is first
+	drive       *drive     // nil until the user has a volume
+
+	verifierOnce sync.Once
+	verifier     []byte
+	verifierErr  error
+
+	// Guarded by srv.mu.
+	totpSecret   string
+	captcha      bool
+	corruptProof bool
+}
+
+// srpVerifier returns the SRP verifier of u's login password, computing
+// it on first use since it costs a bcrypt hash.
+func (u *User) srpVerifier() ([]byte, error) {
+	u.verifierOnce.Do(func() {
+		var auth *srp.Auth
+		auth, u.verifierErr = srp.NewAuthForVerifier([]byte(u.Password), signedModulus, u.srpSalt)
+		if u.verifierErr == nil {
+			u.verifier, u.verifierErr = auth.GenerateVerifier(srpBits)
+		}
+	})
+	return u.verifier, u.verifierErr
+}
+
+// EnableTOTP turns on TOTP two-factor authentication for u and returns
+// the base32 secret to generate codes from.
+func (u *User) EnableTOTP() (secret string) {
+	key, err := totp.Generate(totp.GenerateOpts{Issuer: "protontest", AccountName: u.Username})
+	if err != nil {
+		panic(fmt.Sprintf("protontest: generate TOTP secret: %v", err))
+	}
+	u.srv.mu.Lock()
+	defer u.srv.mu.Unlock()
+	u.totpSecret = key.Secret()
+	return u.totpSecret
+}
+
+// srpLogin is an SRP challenge issued by POST /core/v4/auth/info.
+type srpLogin struct {
+	user   *User
+	server *srp.Server
+}
+
+// fork is a session fork requested by GET /auth/v4/sessions/forks.
+type fork struct {
+	userCode string
+	key      []byte   // encryption key from the sign-in URL, once approved
+	approved *session // child session, once approved
+}
+
+// Session holds the tokens of a signed in session.
+type Session struct {
+	UID          string
+	AccessToken  string
+	RefreshToken string
+}
+
+// session is the server side state of a Session.
+type session struct {
+	Session
+	user    *User
+	expired bool // access token no longer accepted
+	locked  bool // waiting for the second factor
+}
+
+// randomBytes returns n random bytes.
+func randomBytes(n int) []byte {
+	b := make([]byte, n)
+	_, _ = rand.Read(b)
+	return b
+}
+
+// randomID returns a random hex string.
+func randomID() string {
+	return hex.EncodeToString(randomBytes(16))
+}
+
+// registerAuth registers the session routes.
+func (s *Server) registerAuth() {
+	s.handle("POST", "/auth/v4/refresh", s.handleRefresh)
+	s.handle("POST", "/core/v4/auth/info", s.handleAuthInfo)
+	s.handle("POST", "/core/v4/auth", s.handleLogin)
+	s.handleSession("POST", "/core/v4/auth/2fa", s.handle2FA)
+	s.handleSession("DELETE", "/core/v4/auth", s.handleLogout)
+	s.handleAuth("GET", "/core/v4/keys/salts", s.handleKeySalts)
+	s.handle("GET", "/auth/v4/sessions/forks", s.handleForkInit)
+	s.handle("GET", "/auth/v4/sessions/forks/{selector}", s.handleForkStatus)
+	s.handleAuth("GET", "/tests/auth", func(w http.ResponseWriter, r *http.Request, u *User) {
+		writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "UserID": u.ID})
+	})
+	s.handleAuth("POST", "/tests/echo", func(w http.ResponseWriter, r *http.Request, u *User) {
+		var body struct{ Echo string }
+		if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+			writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: "+err.Error())
+			return
+		}
+		writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "Echo": body.Echo})
+	})
+}
+
+// AddUser creates an account whose mailbox password is its password.
+func (s *Server) AddUser(username, password string) *User {
+	return s.AddTwoPasswordUser(username, password, password)
+}
+
+// AddTwoPasswordUser creates an account which signs in with password
+// and unlocks its keys with mailboxPassword.
+//
+// The account has a user key and one address, username itself if it
+// contains "@" or else username@proton.test, with an address key. Its
+// drive has a volume, a main share and an empty root folder.
+func (s *Server) AddTwoPasswordUser(username, password, mailboxPassword string) *User {
+	u := newUser(s, username, password, mailboxPassword)
+	u.drive = s.newDrive(u.addresses[0])
+	s.addUser(u)
+	return u
+}
+
+// AddUserWithoutDrive creates an account like AddUser but without a
+// volume, as a brand-new Proton account is: its my-files route answers
+// HTTP 422 code 2501 until a client creates the volume.
+func (s *Server) AddUserWithoutDrive(username, password string) *User {
+	u := newUser(s, username, password, password)
+	s.addUser(u)
+	return u
+}
+
+// newUser returns an account with a user key and one address, not yet
+// added to s.
+func newUser(s *Server, username, password, mailboxPassword string) *User {
+	u := &User{
+		ID:              randomID(),
+		Username:        username,
+		Password:        password,
+		MailboxPassword: mailboxPassword,
+		srv:             s,
+		srpSalt:         randomBytes(10),
+	}
+	key := newUserKey(u)
+	u.keys = []*userKey{key}
+	u.KeySalt, u.keyPassword = key.salt, key.keyPassword
+	email := username
+	if !strings.Contains(email, "@") {
+		email += "@proton.test"
+	}
+	u.addresses = []*address{newAddress(email, key)}
+	return u
+}
+
+// addUser adds u to s.
+func (s *Server) addUser(u *User) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.users[u.Username] = u
+}
+
+// RequireCaptcha makes password sign-in for u fail with HTTP 422 code
+// 9001, as when Proton asks for human verification.
+func (s *Server) RequireCaptcha(u *User) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	u.captcha = true
+}
+
+// CorruptServerProof makes password sign-in for u return a server
+// proof with one byte changed, as an impostor server would.
+func (s *Server) CorruptServerProof(u *User) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	u.corruptProof = true
+}
+
+// NewSession signs u in and returns the new session's tokens.
+func (s *Server) NewSession(u *User) Session {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.newSessionLocked(u, false).Session
+}
+
+// newSessionLocked creates a session for u. s.mu must be held.
+func (s *Server) newSessionLocked(u *User, locked bool) *session {
+	sess := &session{
+		Session: Session{UID: randomID(), AccessToken: s.newSecretLocked(SessionToken), RefreshToken: s.newSecretLocked(SessionToken)},
+		user:    u,
+		locked:  locked,
+	}
+	s.sessions[sess.UID] = sess
+	return sess
+}
+
+// ApproveFork completes the session fork whose sign-in URL is
+// signInURL as if u had signed in on Proton's page: the next poll of
+// the fork returns a new session for u and u's key password encrypted
+// with the key from the URL.
+func (s *Server) ApproveFork(signInURL string, u *User) error {
+	parsed, err := url.Parse(signInURL)
+	if err != nil {
+		return fmt.Errorf("protontest: parse sign-in URL: %w", err)
+	}
+	payload, ok := strings.CutPrefix(parsed.Fragment, "payload=")
+	parts := strings.Split(payload, ":")
+	if !ok || len(parts) != 4 || parts[0] != "0" || parts[3] != forkClientID {
+		return fmt.Errorf("protontest: bad sign-in URL payload %q", payload)
+	}
+	key, err := base64.StdEncoding.DecodeString(parts[2])
+	if err != nil || len(key) != 32 {
+		return fmt.Errorf("protontest: bad key in sign-in URL payload %q", payload)
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, f := range s.forks {
+		if f.userCode == parts[1] {
+			f.key = key
+			f.approved = s.newSessionLocked(u, false)
+			return nil
+		}
+	}
+	return fmt.Errorf("protontest: no fork with user code %q", parts[1])
+}
+
+// ExpireAccessToken makes the access token of session uid return 401
+// until the session is refreshed.
+func (s *Server) ExpireAccessToken(uid string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if sess := s.sessions[uid]; sess != nil {
+		sess.expired = true
+	}
+}
+
+// RevokeSession ends session uid, so neither its access token nor its
+// refresh token is accepted.
+func (s *Server) RevokeSession(uid string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	delete(s.sessions, uid)
+}
+
+// handleAuth registers h for an authenticated route. Requests without
+// a valid x-pm-uid and bearer access token get HTTP 401, and requests
+// from a session waiting for its second factor get HTTP 403.
+func (s *Server) handleAuth(method, pattern string, h func(w http.ResponseWriter, r *http.Request, u *User)) {
+	s.handleSession(method, pattern, func(w http.ResponseWriter, r *http.Request, sess *session) {
+		s.mu.Lock()
+		locked := sess.locked
+		s.mu.Unlock()
+		if locked {
+			writeError(w, http.StatusForbidden, 0, "protontest: second factor required")
+			return
+		}
+		h(w, r, sess.user)
+	})
+}
+
+// handleSession registers h for a route which needs a session, which
+// may be waiting for its second factor. Requests without a valid
+// x-pm-uid and bearer access token get HTTP 401.
+func (s *Server) handleSession(method, pattern string, h func(w http.ResponseWriter, r *http.Request, sess *session)) {
+	s.handle(method, pattern, func(w http.ResponseWriter, r *http.Request) {
+		token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+		s.mu.Lock()
+		sess := s.sessions[r.Header.Get("x-pm-uid")]
+		valid := ok && sess != nil && !sess.expired && sess.AccessToken == token
+		s.mu.Unlock()
+		if !valid {
+			writeError(w, http.StatusUnauthorized, 401, "protontest: invalid access token")
+			return
+		}
+		h(w, r, sess)
+	})
+}
+
+// handleRefresh rotates the tokens of the session named by x-pm-uid.
+// Refresh tokens are single use.
+func (s *Server) handleRefresh(w http.ResponseWriter, r *http.Request) {
+	var req api.AuthRefreshRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.ResponseType != "token" || req.GrantType != "refresh_token" {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad refresh request")
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	sess := s.sessions[r.Header.Get("x-pm-uid")]
+	if sess == nil || req.RefreshToken == "" || sess.RefreshToken != req.RefreshToken {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeRefreshTokenInvalid, "protontest: invalid refresh token")
+		return
+	}
+	sess.AccessToken = s.newSecretLocked(SessionToken)
+	sess.RefreshToken = s.newSecretLocked(SessionToken)
+	sess.expired = false
+	writeJSON(w, http.StatusOK, &api.AuthRefreshResponse{
+		Code:         api.CodeOK,
+		UID:          sess.UID,
+		AccessToken:  sess.AccessToken,
+		RefreshToken: sess.RefreshToken,
+	})
+}
+
+// handleAuthInfo issues an SRP challenge for the user named in the
+// request.
+func (s *Server) handleAuthInfo(w http.ResponseWriter, r *http.Request) {
+	var req api.AuthInfoRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Intent != "Proton" {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad auth info request")
+		return
+	}
+	s.mu.Lock()
+	u := s.users[req.Username]
+	s.mu.Unlock()
+	if u == nil {
+		writeError(w, http.StatusUnprocessableEntity, api.CodePasswordWrong, "protontest: unknown user")
+		return
+	}
+	verifier, err := u.srpVerifier()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, 0, "protontest: SRP verifier: "+err.Error())
+		return
+	}
+	server, err := srp.NewServerFromSigned(signedModulus, verifier, srpBits)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, 0, "protontest: SRP server: "+err.Error())
+		return
+	}
+	challenge, err := server.GenerateChallenge()
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, 0, "protontest: SRP challenge: "+err.Error())
+		return
+	}
+	srpSession := randomID()
+	s.mu.Lock()
+	s.srp[srpSession] = &srpLogin{user: u, server: server}
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, &api.AuthInfoResponse{
+		Code:            api.CodeOK,
+		Version:         4,
+		Modulus:         signedModulus,
+		ServerEphemeral: base64.StdEncoding.EncodeToString(challenge),
+		Salt:            base64.StdEncoding.EncodeToString(u.srpSalt),
+		SRPSession:      srpSession,
+		Username:        u.Username,
+	})
+}
+
+// handleLogin checks the SRP proof for a challenge and creates a
+// session, locked until the second factor if the user has TOTP.
+// Challenges are single use.
+func (s *Server) handleLogin(w http.ResponseWriter, r *http.Request) {
+	var req api.AuthRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad auth request")
+		return
+	}
+	ephemeral, err1 := base64.StdEncoding.DecodeString(req.ClientEphemeral)
+	proof, err2 := base64.StdEncoding.DecodeString(req.ClientProof)
+	if err := errors.Join(err1, err2); err != nil {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad auth request: "+err.Error())
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	login := s.srp[req.SRPSession]
+	delete(s.srp, req.SRPSession)
+	if login == nil || login.user.Username != req.Username {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: unknown SRP session")
+		return
+	}
+	u := login.user
+	if u.captcha {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeHumanVerification, "protontest: human verification required")
+		return
+	}
+	serverProof, err := login.server.VerifyProofs(ephemeral, proof)
+	if err != nil {
+		writeError(w, http.StatusUnprocessableEntity, api.CodePasswordWrong, "protontest: incorrect login credentials")
+		return
+	}
+	if u.corruptProof {
+		serverProof[0] ^= 1
+	}
+	resp := &api.AuthResponse{
+		Code:         api.CodeOK,
+		UserID:       u.ID,
+		TokenType:    "Bearer",
+		ServerProof:  base64.StdEncoding.EncodeToString(serverProof),
+		Scopes:       []string{"full"},
+		PasswordMode: api.PasswordModeOne,
+	}
+	if u.MailboxPassword != u.Password {
+		resp.PasswordMode = api.PasswordModeTwo
+	}
+	if u.totpSecret != "" {
+		resp.TwoFA.Enabled = api.TwoFATOTP
+		resp.Scopes = []string{"twofactor"}
+	}
+	sess := s.newSessionLocked(u, u.totpSecret != "")
+	resp.UID, resp.AccessToken, resp.RefreshToken = sess.UID, sess.AccessToken, sess.RefreshToken
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// handle2FA unlocks a session with a TOTP code.
+func (s *Server) handle2FA(w http.ResponseWriter, r *http.Request, sess *session) {
+	var req api.Auth2FARequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad 2FA request")
+		return
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	secret := sess.user.totpSecret
+	if secret == "" || !totp.Validate(req.TwoFactorCode, secret) {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: incorrect 2FA code")
+		return
+	}
+	sess.locked = false
+	writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK, "Scopes": []string{"full"}})
+}
+
+// handleLogout ends the session.
+func (s *Server) handleLogout(w http.ResponseWriter, r *http.Request, sess *session) {
+	s.mu.Lock()
+	delete(s.sessions, sess.UID)
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, map[string]any{"Code": api.CodeOK})
+}
+
+// handleKeySalts returns the password salt of each user key.
+func (s *Server) handleKeySalts(w http.ResponseWriter, r *http.Request, u *User) {
+	resp := &api.KeySaltsResponse{Code: api.CodeOK}
+	s.mu.Lock()
+	for _, key := range u.keys {
+		resp.KeySalts = append(resp.KeySalts, api.KeySalt{ID: key.id, KeySalt: key.salt})
+	}
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// handleForkInit starts a session fork for a client to sign in with.
+func (s *Server) handleForkInit(w http.ResponseWriter, r *http.Request) {
+	f := &fork{userCode: strings.ToUpper(hex.EncodeToString(randomBytes(4)))}
+	s.mu.Lock()
+	selector := s.newSecretLocked(ForkSelector)
+	s.forks[selector] = f
+	s.mu.Unlock()
+	writeJSON(w, http.StatusOK, &api.ForkInitResponse{Code: api.CodeOK, Selector: selector, UserCode: f.userCode})
+}
+
+// handleForkStatus returns the child session of an approved fork, once,
+// or HTTP 422 while the fork is waiting for approval.
+func (s *Server) handleForkStatus(w http.ResponseWriter, r *http.Request) {
+	selector := r.PathValue("selector")
+	s.mu.Lock()
+	f := s.forks[selector]
+	if f == nil || f.approved == nil {
+		s.mu.Unlock()
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: fork not ready")
+		return
+	}
+	delete(s.forks, selector)
+	sess := f.approved.Session
+	user := f.approved.user
+	s.mu.Unlock()
+
+	payload, err := sealForkPayload(f.key, user.KeyPassword())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, 0, "protontest: fork payload: "+err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, &api.ForkStatusResponse{
+		Code:         api.CodeOK,
+		Payload:      payload,
+		UID:          sess.UID,
+		AccessToken:  sess.AccessToken,
+		RefreshToken: sess.RefreshToken,
+	})
+}
+
+// sealForkPayload encrypts keyPassword for a fork as Proton's sign-in
+// page does: base64 of nonce, AES-256-GCM ciphertext and tag, with
+// additional data "fork".
+func sealForkPayload(key []byte, keyPassword string) (string, error) {
+	plain, err := json.Marshal(map[string]string{"keyPassword": keyPassword})
+	if err != nil {
+		return "", err
+	}
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+	gcm, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+	nonce := randomBytes(gcm.NonceSize())
+	return base64.StdEncoding.EncodeToString(gcm.Seal(nonce, nonce, plain, []byte("fork"))), nil
+}

backend/protondrive/protontest/auth_test.go

@@ -0,0 +1,139 @@
+package protontest
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/go-srp"
+	"github.com/pquerna/otp/totp"
+	"github.com/rclone/rclone/backend/protondrive/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// refresh calls POST /auth/v4/refresh for uid with refreshToken.
+func refresh(t *testing.T, s *Server, uid, refreshToken string) (*http.Response, []byte) {
+	t.Helper()
+	body, err := json.Marshal(&api.AuthRefreshRequest{ResponseType: "token", GrantType: "refresh_token", RefreshToken: refreshToken})
+	require.NoError(t, err)
+	req, err := http.NewRequest("POST", s.URL()+"/auth/v4/refresh", bytes.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("x-pm-appversion", testAppVersion)
+	req.Header.Set("x-pm-uid", uid)
+	resp, err := s.Client().Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+	out, err := io.ReadAll(resp.Body)
+	require.NoError(t, err)
+	return resp, out
+}
+
+func authed(t *testing.T, s *Server, sess Session) int {
+	t.Helper()
+	resp, _ := do(t, s, "GET", "/tests/auth", map[string]string{
+		"x-pm-appversion": testAppVersion,
+		"x-pm-uid":        sess.UID,
+		"Authorization":   "Bearer " + sess.AccessToken,
+	})
+	return resp.StatusCode
+}
+
+func TestAuth(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	assert.Equal(t, "pass", u.MailboxPassword)
+	sess := s.NewSession(u)
+
+	assert.Equal(t, http.StatusOK, authed(t, s, sess))
+	assert.Equal(t, http.StatusUnauthorized, authed(t, s, Session{UID: sess.UID, AccessToken: "wrong"}))
+	assert.Equal(t, http.StatusUnauthorized, authed(t, s, Session{UID: "unknown", AccessToken: sess.AccessToken}))
+
+	s.ExpireAccessToken(sess.UID)
+	assert.Equal(t, http.StatusUnauthorized, authed(t, s, sess))
+
+	resp, body := refresh(t, s, sess.UID, sess.RefreshToken)
+	require.Equal(t, http.StatusOK, resp.StatusCode, "body: %s", body)
+	var out api.AuthRefreshResponse
+	require.NoError(t, json.Unmarshal(body, &out))
+	assert.Equal(t, sess.UID, out.UID)
+	next := Session{UID: out.UID, AccessToken: out.AccessToken, RefreshToken: out.RefreshToken}
+	assert.Equal(t, http.StatusOK, authed(t, s, next))
+	assert.Equal(t, http.StatusUnauthorized, authed(t, s, sess), "old access token")
+
+	// Refresh tokens are single use.
+	resp, body = refresh(t, s, sess.UID, sess.RefreshToken)
+	assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
+	assert.Equal(t, api.CodeRefreshTokenInvalid, decodeError(t, body).Code)
+
+	s.RevokeSession(sess.UID)
+	assert.Equal(t, http.StatusUnauthorized, authed(t, s, next))
+	resp, body = refresh(t, s, next.UID, next.RefreshToken)
+	assert.Equal(t, http.StatusUnprocessableEntity, resp.StatusCode)
+	assert.Equal(t, api.CodeRefreshTokenInvalid, decodeError(t, body).Code)
+}
+
+// postJSON sends in as JSON to path with the given extra headers and
+// decodes the response into out, returning the HTTP status.
+func postJSON(t *testing.T, s *Server, path string, header map[string]string, in, out any) int {
+	t.Helper()
+	body, err := json.Marshal(in)
+	require.NoError(t, err)
+	req, err := http.NewRequest("POST", s.URL()+path, bytes.NewReader(body))
+	require.NoError(t, err)
+	req.Header.Set("x-pm-appversion", testAppVersion)
+	for k, v := range header {
+		req.Header.Set(k, v)
+	}
+	resp, err := s.Client().Do(req)
+	require.NoError(t, err)
+	defer func() { _ = resp.Body.Close() }()
+	require.NoError(t, json.NewDecoder(resp.Body).Decode(out))
+	return resp.StatusCode
+}
+
+func TestLoginTOTPLocksSession(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	secret := u.EnableTOTP()
+
+	var info api.AuthInfoResponse
+	require.Equal(t, http.StatusOK, postJSON(t, s, "/core/v4/auth/info", nil, &api.AuthInfoRequest{Intent: "Proton", Username: "user"}, &info))
+	auth, err := srp.NewAuth(info.Version, "user", []byte("pass"), info.Salt, info.Modulus, info.ServerEphemeral)
+	require.NoError(t, err)
+	proofs, err := auth.GenerateProofs(srpBits)
+	require.NoError(t, err)
+	var out api.AuthResponse
+	require.Equal(t, http.StatusOK, postJSON(t, s, "/core/v4/auth", nil, &api.AuthRequest{
+		Username:        "user",
+		ClientEphemeral: base64.StdEncoding.EncodeToString(proofs.ClientEphemeral),
+		ClientProof:     base64.StdEncoding.EncodeToString(proofs.ClientProof),
+		SRPSession:      info.SRPSession,
+	}, &out))
+	assert.Equal(t, base64.StdEncoding.EncodeToString(proofs.ExpectedServerProof), out.ServerProof)
+	assert.Equal(t, api.TwoFATOTP, out.TwoFA.Enabled)
+	sess := Session{UID: out.UID, AccessToken: out.AccessToken, RefreshToken: out.RefreshToken}
+	assert.Equal(t, http.StatusForbidden, authed(t, s, sess))
+
+	headers := map[string]string{"x-pm-uid": sess.UID, "Authorization": "Bearer " + sess.AccessToken}
+	var res api.Error
+	assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/core/v4/auth/2fa", headers, &api.Auth2FARequest{TwoFactorCode: "000000x"}, &res))
+	assert.Equal(t, http.StatusForbidden, authed(t, s, sess))
+
+	code, err := totp.GenerateCode(secret, time.Now())
+	require.NoError(t, err)
+	assert.Equal(t, http.StatusOK, postJSON(t, s, "/core/v4/auth/2fa", headers, &api.Auth2FARequest{TwoFactorCode: code}, &res))
+	assert.Equal(t, http.StatusOK, authed(t, s, sess))
+
+	// SRP challenges are single use.
+	assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/core/v4/auth", nil, &api.AuthRequest{
+		Username:        "user",
+		ClientEphemeral: base64.StdEncoding.EncodeToString(proofs.ClientEphemeral),
+		ClientProof:     base64.StdEncoding.EncodeToString(proofs.ClientProof),
+		SRPSession:      info.SRPSession,
+	}, &res))
+}

backend/protondrive/protontest/cmd/protonfake/main.go

@@ -0,0 +1,57 @@
+// Command protonfake serves a fake Proton Drive API for integration
+// tests.
+//
+// It prints the rclone config of a signed in user with a Drive volume
+// as key=value lines on stdout, then serves until interrupted.
+//
+//	go run ./backend/protondrive/protontest/cmd/protonfake -addr 127.0.0.1:0
+package main
+
+import (
+	"encoding/base64"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/rclone/rclone/backend/protondrive/protontest"
+)
+
+func main() {
+	addr := flag.String("addr", "127.0.0.1:0", "address to listen on")
+	flag.Parse()
+	stop := make(chan struct{})
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
+	go func() {
+		<-sig
+		close(stop)
+	}()
+	if err := run(*addr, os.Stdout, stop); err != nil {
+		fmt.Fprintln(os.Stderr, "protonfake:", err)
+		os.Exit(1)
+	}
+}
+
+// run serves a fake server on addr until stop is closed, after writing
+// its endpoint and a fresh session's config to out.
+func run(addr string, out io.Writer, stop <-chan struct{}) error {
+	srv, err := protontest.NewStandalone(addr)
+	if err != nil {
+		return err
+	}
+	defer srv.Close()
+	user := srv.AddUser("protonfake", "protonfake")
+	sess := srv.NewSession(user)
+	// The config holds the salted key password base64 encoded.
+	keyPass := base64.StdEncoding.EncodeToString([]byte(user.KeyPassword()))
+	_, err = fmt.Fprintf(out, "endpoint=%s\nclient_uid=%s\nclient_access_token=%s\nclient_refresh_token=%s\nclient_salted_key_pass=%s\ntype=protondrive\n",
+		srv.URL(), sess.UID, sess.AccessToken, sess.RefreshToken, keyPass)
+	if err != nil {
+		return fmt.Errorf("write config: %w", err)
+	}
+	<-stop
+	return nil
+}

backend/protondrive/protontest/cmd/protonfake/main_test.go

@@ -0,0 +1,65 @@
+package main
+
+import (
+	"bufio"
+	"encoding/base64"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRun(t *testing.T) {
+	pr, pw := io.Pipe()
+	stop := make(chan struct{})
+	done := make(chan error, 1)
+	go func() {
+		err := run("127.0.0.1:0", pw, stop)
+		_ = pw.Close()
+		done <- err
+	}()
+
+	wantKeys := []string{"endpoint", "client_uid", "client_access_token", "client_refresh_token", "client_salted_key_pass", "type"}
+	values := make(map[string]string)
+	lines := bufio.NewScanner(pr)
+	for _, key := range wantKeys {
+		require.True(t, lines.Scan(), "missing line for %s", key)
+		k, v, ok := strings.Cut(lines.Text(), "=")
+		require.True(t, ok, "line %q is not key=value", lines.Text())
+		require.Equal(t, key, k)
+		require.NotEmpty(t, v, key)
+		values[k] = v
+	}
+	assert.Equal(t, "protondrive", values["type"])
+	assert.False(t, strings.HasSuffix(values["endpoint"], ":0"), "endpoint %q must carry the bound port", values["endpoint"])
+	keyPass, err := base64.StdEncoding.DecodeString(values["client_salted_key_pass"])
+	require.NoError(t, err)
+	assert.Len(t, keyPass, 31, "a key password is the last 31 characters of a bcrypt hash")
+
+	req, err := http.NewRequest("GET", values["endpoint"]+"/core/v4/users", nil)
+	require.NoError(t, err)
+	req.Header.Set("x-pm-appversion", "external-drive-rclone@1.0.0-stable")
+	req.Header.Set("x-pm-uid", values["client_uid"])
+	req.Header.Set("Authorization", "Bearer "+values["client_access_token"])
+	resp, err := http.DefaultClient.Do(req)
+	require.NoError(t, err)
+	_ = resp.Body.Close()
+	assert.Equal(t, http.StatusOK, resp.StatusCode)
+
+	close(stop)
+	extra := lines.Scan()
+	assert.False(t, extra, "unexpected output %q", lines.Text())
+	require.NoError(t, <-done)
+	_, err = http.Get(values["endpoint"] + "/tests/ping")
+	assert.Error(t, err, "server still answering after stop")
+}
+
+func TestRunBadAddress(t *testing.T) {
+	var out strings.Builder
+	err := run("not an address", &out, make(chan struct{}))
+	assert.Error(t, err)
+	assert.Empty(t, out.String())
+}

backend/protondrive/protontest/crypto.go

@@ -0,0 +1,339 @@
+package protontest
+
+// The Drive cryptography of the fake, written from Proton's Drive SDK
+// (client/js/src/crypto/driveCrypto.ts and the nodes, shares and upload
+// crypto services) independently of the protondrive backend, so a bug
+// in one is unlikely to be mirrored in the other.
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
+	"github.com/ProtonMail/gopenpgp/v3/constants"
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/ProtonMail/gopenpgp/v3/profile"
+)
+
+// driveKeyName is the user ID of every node and share key, as the SDK
+// generates them.
+const driveKeyName = "Drive key"
+
+// aeadPGP returns a handle whose keys advertise SEIPDv2 and whose
+// session-key encryption uses AES-256 GCM, as the SDK's file content
+// does (openpgp.js aeadProtect with Proton's GCM preference).
+//
+// Everything that is not file content goes through encryptArmored,
+// which never produces SEIPDv2 whatever the recipient advertises.
+func aeadPGP() *crypto.PGPHandle {
+	p := profile.Default()
+	p.AeadEncryption = &packet.AEADConfig{DefaultMode: packet.AEADModeGCM}
+	return crypto.PGPWithProfile(p)
+}
+
+// signTime returns when the fake generates keys and makes signatures: a
+// day before the local clock, so that what it writes is valid at the
+// server clock whatever skew within a day a test sets.
+func signTime() int64 {
+	return time.Now().Add(-24 * time.Hour).Unix()
+}
+
+// signTimeOf returns when the fake signs with signer: signTime, or
+// signer's creation time if later, as for a key a test supplied which
+// was generated less than a day ago.
+func signTimeOf(signer *crypto.Key) int64 {
+	return max(signTime(), signer.GetEntity().PrimaryKey.CreationTime.Unix())
+}
+
+// newPassphrase returns base64 of 32 random bytes, the SDK's
+// generatePassphrase.
+func newPassphrase() []byte {
+	return []byte(base64.StdEncoding.EncodeToString(randomBytes(32)))
+}
+
+// lockedKey is a generated key with its passphrase and armored locked
+// form.
+type lockedKey struct {
+	key        *crypto.Key
+	passphrase []byte
+	armored    string
+}
+
+// newDriveKey generates a node or share key locked with a new
+// passphrase. aead makes the key advertise SEIPDv2, as file node keys
+// do; folder and share keys do not.
+func newDriveKey(aead bool) lockedKey {
+	pgp := crypto.PGP()
+	if aead {
+		pgp = aeadPGP()
+	}
+	key, err := pgp.KeyGeneration().AddUserId(driveKeyName, "").GenerationTime(signTime()).New().GenerateKey()
+	mustCrypto("generate drive key", err)
+	passphrase := newPassphrase()
+	return lockedKey{key: key, passphrase: passphrase, armored: lockKey(key, passphrase)}
+}
+
+// sealedPassphrase is a node or share passphrase as the server stores it.
+type sealedPassphrase struct {
+	armored   string // encrypted to the parent key
+	signature string // armored detached signature of the plaintext
+}
+
+// sealPassphrase encrypts passphrase to parent without AEAD and signs it
+// with signer in a detached signature, the SDK's encryptPassphrase.
+func sealPassphrase(passphrase []byte, parent, signer *crypto.Key) sealedPassphrase {
+	return sealedPassphrase{
+		armored:   encryptArmored(passphrase, parent, nil, false),
+		signature: signArmored(passphrase, signer),
+	}
+}
+
+// encryptArmored encrypts data to key without AEAD, signed inline by
+// signer if not nil, compressed if compress is set.
+//
+// Encrypting to recipients picks SEIPDv2 whenever every recipient key
+// advertises it, as file node keys do, so the session key is generated
+// and wrapped separately: the SDK's ignoreSEIPDv2FeatureFlag.
+func encryptArmored(data []byte, key, signer *crypto.Key, compress bool) string {
+	sk, err := crypto.GenerateSessionKeyAlgo(constants.AES256)
+	mustCrypto("session key", err)
+	kh, err := crypto.PGP().Encryption().Recipient(key).New()
+	mustCrypto("encrypt session key", err)
+	keyPacket, err := kh.EncryptSessionKey(sk)
+	mustCrypto("encrypt session key", err)
+	b := crypto.PGP().Encryption().SessionKey(sk)
+	if signer != nil {
+		b = b.SigningKey(signer).SignTime(signTimeOf(signer))
+	}
+	if compress {
+		b = b.Compress()
+	}
+	eh, err := b.New()
+	mustCrypto("encrypt", err)
+	msg, err := eh.Encrypt(data)
+	mustCrypto("encrypt", err)
+	armored, err := crypto.NewPGPMessage(append(keyPacket, msg.DataPacket...)).Armor()
+	mustCrypto("armor message", err)
+	return armored
+}
+
+// signArmored returns an armored detached signature of data by signer.
+func signArmored(data []byte, signer *crypto.Key) string {
+	sh, err := crypto.PGP().Sign().SigningKey(signer).Detached().SignTime(signTimeOf(signer)).New()
+	mustCrypto("sign", err)
+	sig, err := sh.Sign(data, crypto.Armor)
+	mustCrypto("sign", err)
+	return string(sig)
+}
+
+// signBinary returns a binary detached signature of data by signer.
+func signBinary(data []byte, signer *crypto.Key) []byte {
+	sh, err := crypto.PGP().Sign().SigningKey(signer).Detached().SignTime(signTimeOf(signer)).New()
+	mustCrypto("sign", err)
+	sig, err := sh.Sign(data, crypto.Bytes)
+	mustCrypto("sign", err)
+	return sig
+}
+
+// newHashKey returns a new folder hash key and its armored form,
+// encrypted to and signed by nodeKey, the SDK's generateHashKey.
+func newHashKey(nodeKey *crypto.Key) (hashKey []byte, armored string) {
+	hashKey = newPassphrase()
+	return hashKey, encryptArmored(hashKey, nodeKey, nodeKey, false)
+}
+
+// lookupHash returns the hex HMAC-SHA256 of name keyed with the parent
+// folder's hash key, the SDK's generateLookupHash.
+func lookupHash(parentHashKey []byte, name string) string {
+	mac := hmac.New(sha256.New, parentHashKey)
+	mac.Write([]byte(name))
+	return hex.EncodeToString(mac.Sum(nil))
+}
+
+// contentKey is a file's content session key with its key packet.
+type contentKey struct {
+	sk        *crypto.SessionKey
+	packet    string // base64 PKESK to the node key
+	signature string // armored detached signature of the session key by the node key
+	aead      bool
+}
+
+// newContentKey generates a content session key for nodeKey, the SDK's
+// generateContentKey. With aead the key packet is a v6 PKESK and blocks
+// are SEIPDv2; without, the legacy v3 PKESK and SEIPDv1.
+func newContentKey(nodeKey *crypto.Key, aead bool) contentKey {
+	token := randomBytes(32)
+	sk := crypto.NewSessionKeyFromTokenWithAead(token, constants.AES256, aead)
+	eh, err := crypto.PGP().Encryption().Recipient(nodeKey).New()
+	mustCrypto("content key packet", err)
+	keyPacket, err := eh.EncryptSessionKey(sk)
+	mustCrypto("content key packet", err)
+	return contentKey{
+		sk:        sk,
+		packet:    base64.StdEncoding.EncodeToString(keyPacket),
+		signature: signArmored(sk.Key, nodeKey),
+		aead:      aead,
+	}
+}
+
+// pgp returns the handle for encrypting with ck.
+func (ck contentKey) pgp() *crypto.PGPHandle {
+	if ck.aead {
+		return aeadPGP()
+	}
+	return crypto.PGP()
+}
+
+// encryptBlock encrypts one block of plaintext with the content key and
+// returns the data packet and the armored block signature: a detached
+// signature by signer encrypted to nodeKey with the content key, the
+// SDK's encryptBlock.
+func (ck contentKey) encryptBlock(plain []byte, nodeKey, signer *crypto.Key) (data []byte, encSignature string) {
+	eh, err := ck.pgp().Encryption().SessionKey(ck.sk).New()
+	mustCrypto("encrypt block", err)
+	msg, err := eh.Encrypt(plain)
+	mustCrypto("encrypt block", err)
+
+	sigHandle, err := ck.pgp().Encryption().Recipient(nodeKey).SessionKey(ck.sk).New()
+	mustCrypto("encrypt block signature", err)
+	sigMsg, err := sigHandle.Encrypt(signBinary(plain, signer))
+	mustCrypto("encrypt block signature", err)
+	encSignature, err = sigMsg.Armor()
+	mustCrypto("armor block signature", err)
+	return msg.DataPacket, encSignature
+}
+
+// keyRing returns a key ring of keys, which must not be empty.
+func keyRing(keys ...*crypto.Key) *crypto.KeyRing {
+	kr, err := crypto.NewKeyRing(nil)
+	mustCrypto("key ring", err)
+	for _, k := range keys {
+		mustCrypto("key ring", kr.AddKey(k))
+	}
+	return kr
+}
+
+// errSignature wraps every signature verification failure.
+var errSignature = errors.New("protontest: bad signature")
+
+// decryptArmored decrypts an armored message with key. If verify is not
+// nil the embedded signature must be valid for one of its keys at time
+// at, or now if at is zero.
+func decryptArmored(armored string, key *crypto.Key, verify *crypto.KeyRing, at time.Time) ([]byte, error) {
+	b := crypto.PGP().Decryption().DecryptionKey(key)
+	if verify != nil {
+		b = b.VerificationKeys(verify)
+		if !at.IsZero() {
+			b = b.VerifyTime(at.Unix())
+		}
+	}
+	dh, err := b.New()
+	if err != nil {
+		return nil, err
+	}
+	res, err := dh.Decrypt([]byte(armored), crypto.Armor)
+	if err != nil {
+		return nil, err
+	}
+	if verify != nil {
+		if err := res.SignatureError(); err != nil {
+			return res.Bytes(), fmt.Errorf("%w: %v", errSignature, err)
+		}
+	}
+	return res.Bytes(), nil
+}
+
+// verifyDetached checks a detached signature, armored or binary per
+// encoding, of data against verify at time at, or now if at is zero.
+func verifyDetached(data, sig []byte, encoding int8, verify *crypto.KeyRing, at time.Time) error {
+	b := crypto.PGP().Verify().VerificationKeys(verify)
+	if !at.IsZero() {
+		b = b.VerifyTime(at.Unix())
+	}
+	vh, err := b.New()
+	if err != nil {
+		return err
+	}
+	res, err := vh.VerifyDetached(data, sig, encoding)
+	if err != nil {
+		return fmt.Errorf("%w: %v", errSignature, err)
+	}
+	if err := res.SignatureError(); err != nil {
+		return fmt.Errorf("%w: %v", errSignature, err)
+	}
+	return nil
+}
+
+// unlockFunc unlocks an armored private key with a passphrase.
+type unlockFunc func(armoredKey string, passphrase []byte) (*crypto.Key, error)
+
+// unlockDriveKey decrypts a node or share key: the passphrase with
+// parent, its detached signature against verify, then the key itself
+// with unlock. A bad signature is returned alongside the unlocked key.
+// Signatures are checked at time at, or now if it is zero.
+func unlockDriveKey(armoredKey string, sealed sealedPassphrase, parent *crypto.Key, verify *crypto.KeyRing, unlock unlockFunc, at time.Time) (*crypto.Key, error) {
+	passphrase, err := decryptArmored(sealed.armored, parent, nil, at)
+	if err != nil {
+		return nil, fmt.Errorf("decrypt passphrase: %w", err)
+	}
+	key, err := unlock(armoredKey, passphrase)
+	if err != nil {
+		return nil, fmt.Errorf("unlock key: %w", err)
+	}
+	if err := verifyDetached(passphrase, []byte(sealed.signature), crypto.Armor, verify, at); err != nil {
+		return key, fmt.Errorf("passphrase: %w", err)
+	}
+	return key, nil
+}
+
+// decryptContentKey decrypts a base64 content key packet with nodeKey
+// and checks its signature against nodeKey and verify at time at.
+func decryptContentKey(packet64, signature string, nodeKey *crypto.Key, verify *crypto.KeyRing, at time.Time) (*crypto.SessionKey, error) {
+	keyPacket, err := base64.StdEncoding.DecodeString(packet64)
+	if err != nil {
+		return nil, fmt.Errorf("content key packet: %w", err)
+	}
+	dh, err := crypto.PGP().Decryption().DecryptionKey(nodeKey).New()
+	if err != nil {
+		return nil, err
+	}
+	sk, err := dh.DecryptSessionKey(keyPacket)
+	if err != nil {
+		return nil, fmt.Errorf("content key packet: %w", err)
+	}
+	kr := keyRing(nodeKey)
+	for _, k := range verify.GetKeys() {
+		mustCrypto("key ring", kr.AddKey(k))
+	}
+	if err := verifyDetached(sk.Key, []byte(signature), crypto.Armor, kr, at); err != nil {
+		return sk, fmt.Errorf("content key packet: %w", err)
+	}
+	return sk, nil
+}
+
+// decryptBlock decrypts a block's data packet with sk and checks its
+// encrypted signature, decrypted with nodeKey, against verify at time
+// at.
+func decryptBlock(data []byte, encSignature string, sk *crypto.SessionKey, nodeKey *crypto.Key, verify *crypto.KeyRing, at time.Time) ([]byte, error) {
+	dh, err := crypto.PGP().Decryption().SessionKey(sk).New()
+	if err != nil {
+		return nil, err
+	}
+	res, err := dh.Decrypt(data, crypto.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("decrypt block: %w", err)
+	}
+	sig, err := decryptArmored(encSignature, nodeKey, nil, at)
+	if err != nil {
+		return nil, fmt.Errorf("decrypt block signature: %w", err)
+	}
+	if err := verifyDetached(res.Bytes(), sig, crypto.Bytes, verify, at); err != nil {
+		return res.Bytes(), fmt.Errorf("block: %w", err)
+	}
+	return res.Bytes(), nil
+}

backend/protondrive/protontest/drive_test.go

@@ -0,0 +1,716 @@
+package protontest
+
+import (
+	"bytes"
+	"crypto/sha1"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/ProtonMail/go-crypto/openpgp/armor"
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
+	"github.com/ProtonMail/gopenpgp/v3/crypto"
+	"github.com/rclone/rclone/backend/protondrive/api"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// sessionHeader returns the headers of an authenticated request.
+func sessionHeader(sess Session) map[string]string {
+	return map[string]string{
+		"x-pm-appversion": testAppVersion,
+		"x-pm-uid":        sess.UID,
+		"Authorization":   "Bearer " + sess.AccessToken,
+	}
+}
+
+// getJSON calls GET path as sess, decodes the response into out if it
+// is not nil and returns the HTTP status and body.
+func getJSON(t *testing.T, s *Server, sess Session, path string, out any) (int, []byte) {
+	t.Helper()
+	resp, body := do(t, s, "GET", path, sessionHeader(sess))
+	if out != nil && resp.StatusCode == http.StatusOK {
+		require.NoError(t, json.Unmarshal(body, out), "body: %s", body)
+	}
+	return resp.StatusCode, body
+}
+
+// patterned returns content of n bytes which differs per block.
+func patterned(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = byte(i*7 + i/blockSize)
+	}
+	return b
+}
+
+func sha1Hex(b []byte) string {
+	sum := sha1.Sum(b)
+	return hex.EncodeToString(sum[:])
+}
+
+func TestSeedAndDump(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	modTime := time.Date(2024, 5, 6, 7, 8, 9, 123e6, time.FixedZone("x", 3600))
+
+	docs := u.SeedFolder(u.RootID(), "Documents")
+	sub := u.SeedFolder(docs, "Sub")
+	empty := u.SeedFile(u.RootID(), "empty.txt", nil, modTime)
+	small := u.SeedFile(docs, "small.bin", []byte("hello"), modTime)
+	exact := u.SeedFile(sub, "exact.bin", patterned(blockSize), modTime)
+	big := u.SeedFile(sub, "big.bin", patterned(2*blockSize+123), modTime)
+	before := time.Now().Truncate(time.Second)
+	legacy := u.SeedLegacyFile(docs, "legacy.txt", []byte("old format"))
+	doc := u.SeedDoc(u.RootID(), "Notes")
+
+	type want struct {
+		id, path string
+		folder   bool
+		content  []byte
+		hasTime  bool
+	}
+	wants := []want{
+		{docs, "Documents", true, nil, false},
+		{sub, "Documents/Sub", true, nil, false},
+		{big, "Documents/Sub/big.bin", false, patterned(2*blockSize + 123), true},
+		{exact, "Documents/Sub/exact.bin", false, patterned(blockSize), true},
+		{legacy, "Documents/legacy.txt", false, []byte("old format"), true},
+		{small, "Documents/small.bin", false, []byte("hello"), true},
+		{doc, "Notes", false, nil, false},
+		{empty, "empty.txt", false, nil, true},
+	}
+	dump := u.Dump()
+	require.Len(t, dump, len(wants))
+	for i, w := range wants {
+		n := dump[i]
+		assert.NoError(t, n.Err, w.path)
+		assert.Equal(t, w.id, n.ID, w.path)
+		assert.Equal(t, w.path, n.Path)
+		assert.Equal(t, w.folder, n.IsFolder, w.path)
+		assert.Equal(t, linkStateActive, n.State, w.path)
+		assert.True(t, bytes.Equal(w.content, n.Content), "%s: content", w.path)
+		switch {
+		case !w.hasTime:
+			assert.True(t, n.ModTime.IsZero(), w.path)
+			assert.Empty(t, n.SHA1, w.path)
+		case w.id == legacy:
+			assert.False(t, n.ModTime.Before(before), w.path)
+			assert.False(t, n.ModTime.After(time.Now()), w.path)
+			assert.Equal(t, sha1Hex(w.content), n.SHA1, w.path)
+		default:
+			assert.True(t, modTime.Equal(n.ModTime), "%s: modtime %v", w.path, n.ModTime)
+			assert.Equal(t, sha1Hex(w.content), n.SHA1, w.path)
+		}
+	}
+	assert.Empty(t, u.Drafts())
+}
+
+// packetVersion returns the tag and version byte of the first packet
+// of a binary OpenPGP message.
+func packetVersion(t *testing.T, msg []byte) (tag packet.Packet, version byte) {
+	t.Helper()
+	p, err := packet.NewReader(bytes.NewReader(msg)).Next()
+	require.NoError(t, err)
+	switch p := p.(type) {
+	case *packet.EncryptedKey:
+		return p, byte(p.Version)
+	case *packet.SymmetricallyEncrypted:
+		return p, byte(p.Version)
+	}
+	t.Fatalf("unexpected packet %T", p)
+	return nil, 0
+}
+
+// dataPacketVersion returns the SEIPD version of an armored message.
+func dataPacketVersion(t *testing.T, armored string) byte {
+	t.Helper()
+	block, err := armor.Decode(strings.NewReader(armored))
+	require.NoError(t, err)
+	packets := packet.NewReader(block.Body)
+	for {
+		p, err := packets.Next()
+		require.NoError(t, err)
+		if p, ok := p.(*packet.SymmetricallyEncrypted); ok {
+			return byte(p.Version)
+		}
+	}
+}
+
+func TestSeedFormats(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	current := u.SeedFile(u.RootID(), "current", []byte("new"), time.Now())
+	legacy := u.SeedLegacyFile(u.RootID(), "legacy", []byte("old"))
+	folder := u.SeedFolder(u.RootID(), "folder")
+	v := u.view()
+
+	for _, tc := range []struct {
+		id         string
+		pkesk      byte
+		seipd      byte
+		aeadKey    bool
+		mediaType  string
+		blockCount int
+	}{
+		{current, 6, 2, true, "application/octet-stream", 1},
+		{legacy, 3, 1, false, "application/octet-stream", 1},
+	} {
+		l := v.d.links[tc.id]
+		keyPacket, err := base64.StdEncoding.DecodeString(l.contentKeyPacket)
+		require.NoError(t, err)
+		_, version := packetVersion(t, keyPacket)
+		assert.Equal(t, tc.pkesk, version, "content key packet version")
+		rev := l.activeRevision()
+		require.Len(t, rev.blocks, tc.blockCount)
+		_, version = packetVersion(t, rev.blocks[0].data)
+		assert.Equal(t, tc.seipd, version, "block SEIPD version")
+		assert.Equal(t, tc.seipd, dataPacketVersion(t, rev.blocks[0].encSignature), "block signature SEIPD version")
+		assert.Equal(t, tc.mediaType, l.mediaType)
+
+		key, err := crypto.NewKeyFromArmored(l.nodeKey)
+		require.NoError(t, err)
+		sig, err := key.GetEntity().PrimarySelfSignature(time.Now(), nil)
+		require.NoError(t, err)
+		assert.Equal(t, tc.aeadKey, sig.SEIPDv2, "node key advertises SEIPDv2")
+
+		// Everything but content avoids AEAD whatever the node key
+		// advertises.
+		assert.Equal(t, byte(1), dataPacketVersion(t, l.name), "name")
+		assert.Equal(t, byte(1), dataPacketVersion(t, l.nodePass.armored), "passphrase")
+		assert.Equal(t, byte(1), dataPacketVersion(t, rev.xattr), "extended attributes")
+	}
+
+	l := v.d.links[folder]
+	key, err := crypto.NewKeyFromArmored(l.nodeKey)
+	require.NoError(t, err)
+	sig, err := key.GetEntity().PrimarySelfSignature(time.Now(), nil)
+	require.NoError(t, err)
+	assert.False(t, sig.SEIPDv2, "folder key advertises SEIPDv2")
+	assert.Equal(t, byte(1), dataPacketVersion(t, l.hashKey), "hash key")
+}
+
+func TestCorrupt(t *testing.T) {
+	for _, what := range []string{"name", "name-signature", "passphrase-signature", "block-hash", "manifest", "swap-blocks", "truncate-blocks", "block-rollback", "foreign-block"} {
+		t.Run(what, func(t *testing.T) {
+			s := New(t)
+			u := s.AddUser("user", "pass")
+			victim := u.SeedFile(u.RootID(), "victim", patterned(blockSize+10), time.Now())
+			if what == "block-rollback" {
+				u.SeedRevision(victim, patterned(blockSize+20), time.Now())
+			}
+			bystander := u.SeedFile(u.RootID(), "bystander", []byte("fine"), time.Now())
+			u.Corrupt(victim, what)
+			for _, n := range u.Dump() {
+				switch n.ID {
+				case victim:
+					assert.Error(t, n.Err)
+				case bystander:
+					assert.NoError(t, n.Err)
+				}
+			}
+		})
+	}
+}
+
+func TestBootstrapRoutes(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+
+	var myFiles myFilesResponse
+	status, body := getJSON(t, s, sess, "/drive/v2/shares/my-files", &myFiles)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	assert.Equal(t, u.VolumeID(), myFiles.Volume.VolumeID)
+	assert.Equal(t, u.ShareID(), myFiles.Share.ShareID)
+	assert.Equal(t, u.RootID(), myFiles.Link.Link.LinkID)
+	assert.Equal(t, u.Email(), myFiles.Share.CreatorEmail)
+	assert.Nil(t, myFiles.Link.Link.ParentLinkID)
+
+	var volume getVolumeResponse
+	status, body = getJSON(t, s, sess, "/drive/volumes/"+u.VolumeID(), &volume)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	assert.Equal(t, u.ShareID(), volume.Volume.Share.ShareID)
+	assert.Equal(t, u.RootID(), volume.Volume.Share.LinkID)
+
+	var share shareResponse
+	status, body = getJSON(t, s, sess, "/drive/shares/"+u.ShareID(), &share)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	assert.Equal(t, u.RootID(), share.LinkID)
+	assert.Equal(t, myFiles.Share.Key, share.Key)
+	require.NotNil(t, share.AddressID)
+	assert.Equal(t, myFiles.Share.AddressID, *share.AddressID)
+
+	// Another user's IDs are not found.
+	other := s.AddUser("other", "pass")
+	status, body = getJSON(t, s, sess, "/drive/volumes/"+other.VolumeID(), nil)
+	assert.Equal(t, http.StatusUnprocessableEntity, status)
+	assert.Equal(t, api.CodeNotExists, decodeError(t, body).Code)
+	status, _ = getJSON(t, s, sess, "/drive/shares/"+other.ShareID(), nil)
+	assert.Equal(t, http.StatusUnprocessableEntity, status)
+}
+
+// keysOf returns the keys of a JSON object.
+func keysOf(t *testing.T, v any) []string {
+	t.Helper()
+	m, ok := v.(map[string]any)
+	require.True(t, ok, "not an object: %v", v)
+	var keys []string
+	for k := range m {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+// TestWireFieldNames checks the link metadata response carries the
+// fields the SDK's nodes/apiService.ts reads, by name.
+func TestWireFieldNames(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	folder := u.SeedFolder(u.RootID(), "folder")
+	file := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
+
+	var resp struct{ Links []map[string]any }
+	status := postJSON(t, s, "/drive/v2/volumes/"+u.VolumeID()+"/links", sessionHeader(sess), &linksRequest{LinkIDs: []string{folder, file}}, &resp)
+	require.Equal(t, http.StatusOK, status)
+	require.Len(t, resp.Links, 2)
+	for _, l := range resp.Links {
+		assert.Subset(t, keysOf(t, l), []string{"Link", "Folder", "File", "Sharing", "Membership"})
+		assert.Subset(t, keysOf(t, l["Link"]), []string{
+			"LinkID", "Type", "ParentLinkID", "State", "CreateTime", "ModifyTime", "TrashTime",
+			"Name", "NameHash", "NodeKey", "NodePassphrase", "NodePassphraseSignature",
+			"SignatureEmail", "NameSignatureEmail", "OwnedBy", "ThirdParty", "Sdk",
+		})
+		if l["File"] == nil {
+			assert.Subset(t, keysOf(t, l["Folder"]), []string{"NodeHashKey", "XAttr", "IsImported"})
+			continue
+		}
+		f := l["File"]
+		assert.Subset(t, keysOf(t, f), []string{"ActiveRevision", "TotalEncryptedSize", "ContentKeyPacket", "MediaType", "ContentKeyPacketSignature"})
+		assert.Subset(t, keysOf(t, f.(map[string]any)["ActiveRevision"]), []string{
+			"RevisionID", "CreateTime", "EncryptedSize", "SignatureEmail", "XAttr", "IsImported",
+			"Thumbnails", "ThirdParty", "Sdk", "ManifestSignature",
+		})
+	}
+}
+
+func TestChildrenPaging(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	s.SetChildrenPageSize(100)
+	parent := u.SeedFolder(u.RootID(), "parent")
+	want := make(map[string]bool)
+	for i := range 250 {
+		want[u.SeedFolder(parent, fmt.Sprintf("child %03d", i))] = true
+	}
+	trashed := u.SeedFolder(parent, "trashed")
+	s.mu.Lock()
+	u.drive.links[trashed].state = linkStateTrashed
+	s.mu.Unlock()
+	s.ResetRequests()
+
+	got := make(map[string]bool)
+	anchor, pages := "", 0
+	for {
+		path := "/drive/v2/volumes/" + u.VolumeID() + "/folders/" + parent + "/children"
+		if anchor != "" {
+			path += "?AnchorID=" + anchor
+		}
+		var page childrenResponse
+		status, body := getJSON(t, s, sess, path, &page)
+		require.Equal(t, http.StatusOK, status, "body: %s", body)
+		pages++
+		for _, id := range page.LinkIDs {
+			assert.False(t, got[id], "duplicate %s", id)
+			got[id] = true
+		}
+		if !page.More {
+			assert.Nil(t, page.AnchorID)
+			break
+		}
+		require.NotNil(t, page.AnchorID)
+		require.Len(t, page.LinkIDs, 100)
+		anchor = *page.AnchorID
+	}
+	assert.Equal(t, 3, pages)
+	assert.Equal(t, want, got)
+	assert.Equal(t, 3, s.Requests()["GET /drive/v2/volumes/{volumeID}/folders/{linkID}/children"])
+}
+
+func TestLinksBatchLimit(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	a := u.SeedFolder(u.RootID(), "a")
+	b := u.SeedFile(u.RootID(), "b", []byte("b"), time.Now())
+	path := "/drive/v2/volumes/" + u.VolumeID() + "/links"
+
+	ids := []string{a, b}
+	for len(ids) < maxLinksPerRequest {
+		ids = append(ids, randomID())
+	}
+	var resp linksResponse
+	require.Equal(t, http.StatusOK, postJSON(t, s, path, sessionHeader(sess), &linksRequest{LinkIDs: ids}, &resp))
+	require.Len(t, resp.Links, 2, "unknown IDs are left out")
+	assert.Equal(t, a, resp.Links[0].Link.LinkID)
+	assert.NotNil(t, resp.Links[0].Folder)
+	assert.Equal(t, b, resp.Links[1].Link.LinkID)
+	require.NotNil(t, resp.Links[1].File)
+	assert.NotNil(t, resp.Links[1].File.ActiveRevision)
+
+	var apiErr api.Error
+	ids = append(ids, randomID())
+	assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, path, sessionHeader(sess), &linksRequest{LinkIDs: ids}, &apiErr))
+	assert.Equal(t, api.CodeInvalidValue, apiErr.Code)
+}
+
+// getBlock fetches a block URL with the given storage token, sent
+// unless empty.
+func getBlock(t *testing.T, s *Server, bareURL, token string) (int, []byte) {
+	t.Helper()
+	header := map[string]string{"x-pm-appversion": testAppVersion}
+	if token != "" {
+		header["pm-storage-token"] = token
+	}
+	require.True(t, strings.HasPrefix(bareURL, s.URL()+"/"), "block URL %s is not on the test server", bareURL)
+	resp, body := do(t, s, "GET", strings.TrimPrefix(bareURL, s.URL()), header)
+	return resp.StatusCode, body
+}
+
+func TestBlockDownloadNeedsToken(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	content := patterned(blockSize + 100)
+	file := u.SeedFile(u.RootID(), "file", content, time.Now())
+
+	var links linksResponse
+	require.Equal(t, http.StatusOK, postJSON(t, s, "/drive/v2/volumes/"+u.VolumeID()+"/links", sessionHeader(sess), &linksRequest{LinkIDs: []string{file}}, &links))
+	revisionPath := "/drive/v2/volumes/" + u.VolumeID() + "/files/" + file + "/revisions/" + links.Links[0].File.ActiveRevision.RevisionID
+
+	var rev revisionResponse
+	status, body := getJSON(t, s, sess, revisionPath+"?NoBlockUrls=true", &rev)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	require.Len(t, rev.Revision.Blocks, 2)
+	for _, b := range rev.Revision.Blocks {
+		assert.Nil(t, b.Token)
+		assert.Nil(t, b.BareURL)
+	}
+	require.NotNil(t, rev.Revision.ManifestSignature)
+
+	status, body = getJSON(t, s, sess, revisionPath+"?PageSize=1&FromBlockIndex=2", &rev)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	require.Len(t, rev.Revision.Blocks, 1)
+	b := rev.Revision.Blocks[0]
+	assert.Equal(t, 2, b.Index)
+	require.NotNil(t, b.Token)
+	require.NotNil(t, b.BareURL)
+
+	status, _ = getBlock(t, s, *b.BareURL, "")
+	assert.Equal(t, http.StatusForbidden, status)
+	status, _ = getBlock(t, s, *b.BareURL, "wrong")
+	assert.Equal(t, http.StatusForbidden, status)
+	status, data := getBlock(t, s, *b.BareURL, *b.Token)
+	require.Equal(t, http.StatusOK, status)
+	sum := sha256.Sum256(data)
+	assert.Equal(t, b.Hash, base64.StdEncoding.EncodeToString(sum[:]))
+	status, _ = getBlock(t, s, s.URL()+"/storage/blocks/unknown", *b.Token)
+	assert.Equal(t, http.StatusNotFound, status)
+}
+
+func TestQuota(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	u.SeedFile(u.RootID(), "a", patterned(1000), time.Now())
+	u.SeedFile(u.RootID(), "b", patterned(10), time.Now())
+
+	var want int64
+	v := u.view()
+	for _, l := range v.d.links {
+		for _, r := range l.revisions {
+			want += r.size()
+		}
+	}
+	require.NotZero(t, want)
+
+	var resp struct {
+		User struct {
+			UsedSpace        int64
+			MaxSpace         int64
+			ProductUsedSpace struct{ Drive int64 }
+		}
+	}
+	status, body := getJSON(t, s, sess, "/core/v4/users", &resp)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	assert.Equal(t, want, resp.User.UsedSpace)
+	assert.Equal(t, want, resp.User.ProductUsedSpace.Drive)
+	assert.Equal(t, DefaultMaxSpace, resp.User.MaxSpace)
+}
+
+// volumeBootstrap returns a create volume request for the primary
+// address of u made as the SDK's generateVolumeBootstrap does.
+func volumeBootstrap(u *User) *createVolumeRequest {
+	addr := u.addresses[0]
+	addrKey := addr.keys[0].key
+	share := newDriveKey(false)
+	sharePass := sealPassphrase(share.passphrase, addrKey, addrKey)
+	root := newDriveKey(false)
+	rootPass := sealPassphrase(root.passphrase, share.key, addrKey)
+	_, hashKey := newHashKey(root.key)
+	return &createVolumeRequest{
+		AddressID:                 addr.id,
+		AddressKeyID:              &addr.keys[0].id,
+		ShareKey:                  share.armored,
+		SharePassphrase:           sharePass.armored,
+		SharePassphraseSignature:  sharePass.signature,
+		FolderName:                encryptArmored([]byte("root"), share.key, addrKey, false),
+		FolderKey:                 root.armored,
+		FolderPassphrase:          rootPass.armored,
+		FolderPassphraseSignature: rootPass.signature,
+		FolderHashKey:             hashKey,
+	}
+}
+
+func TestAccountWithoutDrive(t *testing.T) {
+	s := New(t)
+	u := s.AddUserWithoutDrive("new", "pass")
+	sess := s.NewSession(u)
+
+	status, body := getJSON(t, s, sess, "/drive/v2/shares/my-files", nil)
+	assert.Equal(t, http.StatusUnprocessableEntity, status)
+	assert.Equal(t, api.CodeNotExists, decodeError(t, body).Code)
+	assert.Equal(t, "Volume not found", decodeError(t, body).Message)
+
+	impostor := generateKey("impostor", "impostor@proton.test")
+	for name, spoil := range map[string]func(req *createVolumeRequest){
+		"share passphrase to another key": func(req *createVolumeRequest) {
+			req.SharePassphrase = encryptArmored([]byte("x"), impostor, nil, false)
+		},
+		"share passphrase signed by another key": func(req *createVolumeRequest) {
+			other := volumeBootstrap(u)
+			req.SharePassphraseSignature = other.SharePassphraseSignature
+		},
+		"folder name missing": func(req *createVolumeRequest) {
+			req.FolderName = ""
+		},
+		"hash key missing": func(req *createVolumeRequest) {
+			req.FolderHashKey = ""
+		},
+		"unknown address": func(req *createVolumeRequest) {
+			req.AddressID = "unknown"
+		},
+	} {
+		req := volumeBootstrap(u)
+		spoil(req)
+		var apiErr api.Error
+		assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/drive/volumes", sessionHeader(sess), req, &apiErr), name)
+		assert.Equal(t, api.CodeInvalidValue, apiErr.Code, name)
+	}
+
+	var created getVolumeResponse
+	status = postJSON(t, s, "/drive/volumes", sessionHeader(sess), volumeBootstrap(u), &created)
+	require.Equal(t, http.StatusOK, status)
+	assert.Equal(t, u.VolumeID(), created.Volume.ID)
+	assert.Equal(t, u.ShareID(), created.Volume.Share.ShareID)
+	assert.Equal(t, u.RootID(), created.Volume.Share.LinkID)
+
+	var apiErr api.Error
+	assert.Equal(t, http.StatusUnprocessableEntity, postJSON(t, s, "/drive/volumes", sessionHeader(sess), volumeBootstrap(u), &apiErr))
+	assert.Equal(t, api.CodeAlreadyExists, apiErr.Code)
+
+	var myFiles myFilesResponse
+	status, body = getJSON(t, s, sess, "/drive/v2/shares/my-files", &myFiles)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	assert.Equal(t, u.RootID(), myFiles.Link.Link.LinkID)
+
+	assert.Empty(t, u.Dump())
+	id := u.SeedFile(u.RootID(), "after", []byte("works"), time.Now())
+	dump := u.Dump()
+	require.Len(t, dump, 1)
+	assert.Equal(t, id, dump[0].ID)
+	assert.Equal(t, "after", dump[0].Path)
+	assert.NoError(t, dump[0].Err)
+}
+
+// TestBlockBodyIsReadable checks a downloaded block is the exact data
+// packet that decrypts with the content key.
+func TestBlockBodyIsReadable(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	file := u.SeedFile(u.RootID(), "file", []byte("block body"), time.Now())
+	v := u.view()
+	l := v.d.links[file]
+	key, err := v.unlock(file)
+	require.NoError(t, err)
+	sk, err := decryptContentKey(l.contentKeyPacket, l.contentKeyPacketSignature, key, keyRing(key), time.Time{})
+	require.NoError(t, err)
+
+	var rev revisionResponse
+	status, body := getJSON(t, s, sess, "/drive/v2/volumes/"+u.VolumeID()+"/files/"+file+"/revisions/"+l.activeRevision().id, &rev)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	b := rev.Revision.Blocks[0]
+	status, data := getBlock(t, s, *b.BareURL, *b.Token)
+	require.Equal(t, http.StatusOK, status)
+	plain, err := decryptBlock(data, *b.EncSignature, sk, key, keyRing(u.addresses[0].keys[0].key), time.Time{})
+	require.NoError(t, err)
+	assert.Equal(t, "block body", string(plain))
+}
+
+func TestSeedRevision(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	ids := map[string]string{
+		"current": u.SeedFile(u.RootID(), "current", []byte("old"), time.Now()),
+		"legacy":  u.SeedLegacyFile(u.RootID(), "legacy", []byte("old")),
+	}
+	content := patterned(blockSize + 5)
+	for name, id := range ids {
+		old := u.view().d.links[id].activeRevision().id
+		rev := u.SeedRevision(id, content, time.Now())
+		l := u.view().d.links[id]
+		assert.Equal(t, rev, l.activeRevision().id, name)
+		require.Len(t, l.revisions, 2, name)
+		assert.Equal(t, old, l.revisions[0].id, name)
+		assert.Equal(t, revisionStateObsolete, l.revisions[0].state, name)
+	}
+	dump := u.Dump()
+	require.Len(t, dump, 2)
+	for _, d := range dump {
+		assert.NoError(t, d.Err, d.Path)
+		assert.True(t, bytes.Equal(content, d.Content), d.Path)
+	}
+}
+
+func TestSeedForgedFolder(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	forged := u.SeedForgedFolder(u.RootID(), "forged")
+	nested := u.SeedAnonymousFolder(forged, "nested")
+	file := u.SeedAnonymousFile(nested, "file", []byte("forged content"), time.Now())
+	for _, n := range u.Dump() {
+		switch n.ID {
+		case forged:
+			assert.Error(t, n.Err, "the forged key's signature verifies")
+		case nested, file:
+			assert.NoError(t, n.Err, n.Path)
+		}
+		if n.ID == file {
+			assert.Equal(t, "forged/nested/file", n.Path)
+			assert.Equal(t, "forged content", string(n.Content))
+		}
+	}
+}
+
+func TestForgeShareAndRoot(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	other := s.AddUser("other", "pass")
+	u.ForgeShareCreator(other, other.Email())
+	u.ForgeRootSigner(other, other.Email())
+	v := u.view()
+	assert.Equal(t, other.Email(), v.d.creatorEmail)
+	_, err := v.unlockShare()
+	assert.NoError(t, err, "the share verifies against the other user's key")
+	root := v.d.links[v.d.rootID]
+	assert.Equal(t, other.Email(), root.signatureEmail)
+	_, err = v.unlock(v.d.rootID)
+	assert.NoError(t, err, "the root verifies against the other user's key")
+	id := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
+	for _, n := range u.Dump() {
+		if n.ID == id {
+			assert.NoError(t, n.Err)
+		}
+	}
+}
+
+func TestServeKeysAs(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	other := s.AddUser("other", "pass")
+	sess := s.NewSession(u)
+	s.ServeKeysAs("USER@proton.test", other)
+	var resp api.PublicKeysResponse
+	status, body := getJSON(t, s, sess, "/core/v4/keys/all?Email=USER@proton.test", &resp)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	require.NotEmpty(t, resp.Address.Keys)
+	want, err := other.addresses[0].keys[0].key.GetArmoredPublicKey()
+	require.NoError(t, err)
+	assert.Equal(t, want, resp.Address.Keys[0].PublicKey)
+}
+
+func TestNodeSecrets(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	id := u.SeedFile(u.RootID(), "file", []byte("secret"), time.Now())
+	passphrase, sessionKey := u.NodeSecrets(id)
+
+	v := u.view()
+	l := v.d.links[id]
+	key, err := crypto.NewPrivateKeyFromArmored(l.nodeKey, passphrase)
+	require.NoError(t, err, "the passphrase unlocks the node key")
+	sk, err := decryptContentKey(l.contentKeyPacket, l.contentKeyPacketSignature, key, keyRing(key), time.Time{})
+	require.NoError(t, err)
+	assert.Equal(t, sk.Key, sessionKey)
+}
+
+func TestIssuedSecrets(t *testing.T) {
+	s := New(t)
+	u := s.AddUser("user", "pass")
+	sess := s.NewSession(u)
+	file := u.SeedFile(u.RootID(), "file", []byte("x"), time.Now())
+	assert.Equal(t, []string{sess.AccessToken, sess.RefreshToken}, s.IssuedSecrets())
+
+	var rev revisionResponse
+	status, body := getJSON(t, s, sess, "/drive/v2/volumes/"+u.VolumeID()+"/files/"+file+"/revisions/"+u.view().d.links[file].activeRevision().id, &rev)
+	require.Equal(t, http.StatusOK, status, "body: %s", body)
+	token := *rev.Revision.Blocks[0].Token
+	assert.Equal(t, []string{token}, s.IssuedSecrets(StorageToken))
+	assert.Equal(t, []string{sess.AccessToken, sess.RefreshToken}, s.IssuedSecrets(SessionToken))
+	assert.Equal(t, []string{sess.AccessToken, sess.RefreshToken, token}, s.IssuedSecrets())
+	assert.Empty(t, s.IssuedSecrets(ForkSelector))
+}
+
+func TestStoredFileRoundTrip(t *testing.T) {
+	s := New(t)
+	addrPass := newPassphrase()
+	root := newDriveKey(false)
+	_, hashKey := newHashKey(root.key)
+	keys := FixedKeys{
+		AddressKey:           lockKey(generateKey("fixed", "fixed@proton.test"), addrPass),
+		AddressKeyPassphrase: string(addrPass),
+		RootKey:              root.armored,
+		RootKeyPassphrase:    string(root.passphrase),
+		RootHashKey:          hashKey,
+	}
+	// Two users holding the same keys can read each other's stored files.
+	one := s.AddUserWithKeys("one", "pass", keys)
+	two := s.AddUserWithKeys("two", "pass", keys)
+	content := patterned(blockSize + 5)
+	id := one.SeedFile(one.RootID(), "f", content, time.Now())
+
+	f := one.StoredFile(id)
+	assert.Equal(t, "one@proton.test", f.Link.SignatureEmail)
+	require.Len(t, f.Revision.Blocks, 2)
+	for _, b := range f.Revision.Blocks {
+		sum := sha256.Sum256(b.Data)
+		assert.Equal(t, sum[:], b.Hash)
+	}
+	two.SeedStoredFile(two.RootID(), f)
+	for _, u := range []*User{one, two} {
+		dump := u.Dump()
+		require.Len(t, dump, 1)
+		assert.NoError(t, dump[0].Err)
+		assert.Equal(t, "f", dump[0].Path)
+		assert.True(t, bytes.Equal(content, dump[0].Content))
+	}
+}

backend/protondrive/protontest/mutate.go

@@ -0,0 +1,760 @@
+package protontest
+
+import (
+	"bytes"
+	"encoding/hex"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"slices"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/ProtonMail/go-crypto/openpgp/armor"
+	"github.com/ProtonMail/go-crypto/openpgp/packet"
+	"github.com/ProtonMail/go-crypto/openpgp/s2k"
+	"github.com/rclone/rclone/backend/protondrive/api"
+)
+
+// OpenPGP packet tags checked in client-written values.
+const (
+	tagPKESK        = 1
+	tagSignature    = 2
+	tagSecretKey    = 5
+	tagSecretSubkey = 7
+	tagSEIPD        = 18
+)
+
+// s2kUsageSHA1 is the S2K usage octet of a secret key locked with CFB
+// and a SHA-1 check, as ordinary (not AEAD) key locking writes.
+const s2kUsageSHA1 = 254
+
+// Wire types of the mutating routes, named after and shaped like the
+// schemas in the SDK's driveTypes.ts.
+
+type createFolderRequest struct {
+	ParentLinkID            string
+	NodeKey                 string
+	NodeHashKey             string
+	NodePassphrase          string
+	NodePassphraseSignature string
+	SignatureEmail          *string
+	Name                    string
+	Hash                    string
+	XAttr                   *string
+}
+
+type folderIDJSON struct {
+	ID string
+}
+
+type createFolderResponse struct {
+	Folder folderIDJSON
+	Code   int
+}
+
+type renameRequest struct {
+	Name               string
+	Hash               *string
+	NameSignatureEmail *string
+	OriginalHash       *string
+}
+
+type moveRequest struct {
+	Name                    string
+	NodePassphrase          string
+	Hash                    string
+	ParentLinkID            string
+	OriginalHash            string
+	NameSignatureEmail      *string
+	ContentHash             *string
+	NodePassphraseSignature *string
+	SignatureEmail          *string
+}
+
+type copyRequest struct {
+	Name                    string
+	NodePassphrase          string
+	Hash                    string
+	TargetVolumeID          string
+	TargetParentLinkID      string
+	NameSignatureEmail      *string
+	NodePassphraseSignature *string
+	SignatureEmail          *string
+}
+
+type copyResponse struct {
+	LinkID string
+	Code   int
+}
+
+type codeJSON struct {
+	Code  int
+	Error string `json:",omitempty"`
+}
+
+type linkResultJSON struct {
+	LinkID   string
+	Response codeJSON
+}
+
+type multiResponse struct {
+	Code      int
+	Responses []linkResultJSON
+}
+
+type checkHashesRequest struct {
+	Hashes    []string
+	ClientUID []string
+}
+
+type pendingHashJSON struct {
+	Hash       string
+	RevisionID string
+	LinkID     string
+	ClientUID  *string
+}
+
+type availableHashesResponse struct {
+	AvailableHashes []string
+	PendingHashes   []pendingHashJSON
+	Code            int
+}
+
+type conflictDetailsJSON struct {
+	ConflictLinkID          string
+	ConflictRevisionID      *string
+	ConflictDraftRevisionID *string
+	ConflictDraftClientUID  *string
+}
+
+type conflictResponse struct {
+	Code    int
+	Error   string
+	Details conflictDetailsJSON
+}
+
+// registerMutations registers the routes which change a drive.
+func (s *Server) registerMutations() {
+	s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/folders", s.handleCreateFolder)
+	s.handleDrive("PUT", "/drive/v2/volumes/{volumeID}/links/{linkID}/rename", s.handleRename)
+	s.handleDrive("PUT", "/drive/v2/volumes/{volumeID}/links/{linkID}/move", s.handleMove)
+	s.handleDrive("POST", "/drive/volumes/{volumeID}/links/{linkID}/copy", s.handleCopy)
+	s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/trash_multiple", s.handleTrash)
+	s.handleDrive("DELETE", "/drive/volumes/{volumeID}/trash", s.handleEmptyTrash)
+	s.handleDrive("POST", "/drive/v2/volumes/{volumeID}/links/{linkID}/checkAvailableHashes", s.handleCheckAvailableHashes)
+}
+
+// decodeRequest decodes the JSON body of r into v, or answers HTTP 400
+// and returns false.
+func decodeRequest(w http.ResponseWriter, r *http.Request, v any) bool {
+	if err := json.NewDecoder(r.Body).Decode(v); err != nil {
+		writeError(w, http.StatusBadRequest, api.CodeInvalidValue, "protontest: bad request body: "+err.Error())
+		return false
+	}
+	return true
+}
+
+// writeInvalid answers HTTP 422 code 2001 for a request failing check.
+func writeInvalid(w http.ResponseWriter, err error) {
+	writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, "protontest: "+err.Error())
+}
+
+// writeConflict answers HTTP 422 code 2500 for a request whose name
+// hash is taken by l.
+func writeConflict(w http.ResponseWriter, l *link) {
+	details := conflictDetailsJSON{ConflictLinkID: l.id}
+	for _, r := range l.revisions {
+		switch r.state {
+		case revisionStateActive:
+			details.ConflictRevisionID = nullable(r.id)
+		case revisionStateDraft:
+			details.ConflictDraftRevisionID = nullable(r.id)
+			details.ConflictDraftClientUID = nullable(r.clientUID)
+		}
+	}
+	writeJSON(w, http.StatusUnprocessableEntity, &conflictResponse{
+		Code:    api.CodeAlreadyExists,
+		Error:   "A file or folder with that name already exists",
+		Details: details,
+	})
+}
+
+// activeLink returns the active link id, or nil.
+func (d *drive) activeLink(id string) *link {
+	if l := d.links[id]; l != nil && l.state == linkStateActive {
+		return l
+	}
+	return nil
+}
+
+// activeFolder returns the active folder id, or nil.
+func (d *drive) activeFolder(id string) *link {
+	if l := d.activeLink(id); l != nil && l.typ == linkTypeFolder {
+		return l
+	}
+	return nil
+}
+
+// conflict returns the link other than except in folder parentID
+// which is not trashed and has the name hash hash, or nil.
+func (d *drive) conflict(parentID, hash, except string) *link {
+	for _, l := range d.links {
+		if l.parentID == parentID && l.nameHash == hash && l.state != linkStateTrashed && l.id != except {
+			return l
+		}
+	}
+	return nil
+}
+
+// within reports whether link id is ancestorID or below it.
+func (d *drive) within(id, ancestorID string) bool {
+	for l := d.links[id]; l != nil; l = d.links[l.parentID] {
+		if l.id == ancestorID {
+			return true
+		}
+	}
+	return false
+}
+
+// inTrash reports whether l or one of its ancestors is trashed.
+func (d *drive) inTrash(l *link) bool {
+	for ; l != nil; l = d.links[l.parentID] {
+		if l.state == linkStateTrashed {
+			return true
+		}
+	}
+	return false
+}
+
+// ownsEmailLocked reports whether email is one of u's addresses.
+// srv.mu must be held.
+func (u *User) ownsEmailLocked(email string) bool {
+	return u.addressByEmailLocked(email) != nil
+}
+
+// checkHash returns an error unless hash is a lower-case hex SHA-256.
+func checkHash(hash string) error {
+	if len(hash) != 64 || strings.ToLower(hash) != hash {
+		return fmt.Errorf("hash %q is not lower-case hex SHA-256", hash)
+	}
+	if _, err := hex.DecodeString(hash); err != nil {
+		return fmt.Errorf("hash %q is not lower-case hex SHA-256", hash)
+	}
+	return nil
+}
+
+// checkSigner returns an error unless email is one of u's addresses.
+func checkSigner(u *User, field string, email *string) error {
+	if email == nil || !u.ownsEmailLocked(*email) {
+		return fmt.Errorf("%s must be an address of the user", field)
+	}
+	return nil
+}
+
+// checkMessages checks every named armored message with checkMessage.
+func checkMessages(messages map[string]string) error {
+	for field, armored := range messages {
+		if err := checkMessage(armored); err != nil {
+			return fmt.Errorf("%s: %w", field, err)
+		}
+	}
+	return nil
+}
+
+// checkMessage returns an error unless armored is an OpenPGP message of
+// v3 PKESKs followed by one SEIPDv1 packet.
+//
+// Official clients write names, passphrases, hash keys and extended
+// attributes in that format and not all of them read SEIPDv2 or v6
+// PKESKs there, whatever the recipient key accepts.
+func checkMessage(armored string) error {
+	if armored == "" {
+		return errors.New("missing")
+	}
+	block, err := armor.Decode(strings.NewReader(armored))
+	if err != nil {
+		return fmt.Errorf("not armored: %w", err)
+	}
+	if block.Type != "PGP MESSAGE" {
+		return fmt.Errorf("armor type %q, want PGP MESSAGE", block.Type)
+	}
+	packets := packet.NewOpaqueReader(block.Body)
+	keyPackets, dataPackets := 0, 0
+	for {
+		op, err := packets.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("malformed packet: %w", err)
+		}
+		if dataPackets > 0 {
+			return errors.New("packet after the encrypted data")
+		}
+		version := -1
+		if len(op.Contents) > 0 {
+			version = int(op.Contents[0])
+		}
+		switch op.Tag {
+		case tagPKESK:
+			if version != 3 {
+				return fmt.Errorf("v%d PKESK, want v3", version)
+			}
+			keyPackets++
+		case tagSEIPD:
+			if version != 1 {
+				return fmt.Errorf("SEIPDv%d, want SEIPDv1", version)
+			}
+			dataPackets++
+		default:
+			return fmt.Errorf("unexpected packet with tag %d", op.Tag)
+		}
+	}
+	if keyPackets == 0 || dataPackets == 0 {
+		return errors.New("want PKESKs and an encrypted data packet")
+	}
+	return nil
+}
+
+// checkFolderKey returns an error unless armored is a v4 secret key
+// whose signatures don't advertise AEAD, with every secret key packet
+// locked with CFB and an iterated and salted S2K, as official clients
+// generate folder keys. The official web app can't use a node key
+// locked with Argon2 or AEAD.
+func checkFolderKey(armored string) error {
+	return checkNodeKey(armored, false)
+}
+
+// checkFileKey returns an error unless armored is a v4 secret key
+// locked as checkFolderKey requires. Unlike a folder key, a file key
+// advertises AEAD, for its content.
+func checkFileKey(armored string) error {
+	return checkNodeKey(armored, true)
+}
+
+// checkNodeKey does the work of checkFolderKey and checkFileKey; aead
+// allows the key to advertise AEAD.
+func checkNodeKey(armored string, aead bool) error {
+	block, err := armor.Decode(strings.NewReader(armored))
+	if err != nil {
+		return fmt.Errorf("not armored: %w", err)
+	}
+	if block.Type != "PGP PRIVATE KEY BLOCK" {
+		return fmt.Errorf("armor type %q, want PGP PRIVATE KEY BLOCK", block.Type)
+	}
+	packets := packet.NewOpaqueReader(block.Body)
+	secretKeys := 0
+	for {
+		op, err := packets.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("malformed packet: %w", err)
+		}
+		switch op.Tag {
+		case tagSecretKey, tagSecretSubkey:
+			if err := checkSecretKeyPacket(op); err != nil {
+				return err
+			}
+			secretKeys++
+		case tagSignature:
+			p, err := op.Parse()
+			sig, ok := p.(*packet.Signature)
+			if err != nil || !ok {
+				return fmt.Errorf("malformed signature: %v", err)
+			}
+			if !aead && (sig.SEIPDv2 || len(sig.PreferredCipherSuites) > 0) {
+				return errors.New("key advertises AEAD (SEIPDv2)")
+			}
+		}
+	}
+	if secretKeys == 0 {
+		return errors.New("no secret key")
+	}
+	return nil
+}
+
+// checkSecretKeyPacket checks one secret key or subkey packet for
+// checkNodeKey.
+func checkSecretKeyPacket(op *packet.OpaquePacket) error {
+	if len(op.Contents) == 0 || op.Contents[0] != 4 {
+		return errors.New("not a v4 key")
+	}
+	p, err := op.Parse()
+	pk, ok := p.(*packet.PrivateKey)
+	if err != nil || !ok {
+		return fmt.Errorf("malformed secret key: %v", err)
+	}
+	// The packet starts with the public key body, which a v4 key hashes
+	// after 0x99 and a two-octet length.
+	var pub bytes.Buffer
+	if err := pk.PublicKey.SerializeForHash(&pub); err != nil {
+		return fmt.Errorf("malformed secret key: %w", err)
+	}
+	rest := op.Contents[pub.Len()-3:]
+	if len(rest) < 3 {
+		return errors.New("malformed secret key: truncated")
+	}
+	switch usage := rest[0]; usage {
+	case s2kUsageSHA1:
+	case 0:
+		return errors.New("secret key is not locked")
+	case 253:
+		return errors.New("secret key is locked with AEAD")
+	default:
+		return fmt.Errorf("secret key S2K usage %d, want %d", usage, s2kUsageSHA1)
+	}
+	if mode := s2k.Mode(rest[2]); mode != s2k.IteratedSaltedS2K {
+		return fmt.Errorf("secret key S2K mode %d, want iterated and salted", mode)
+	}
+	return nil
+}
+
+// keySignature returns the node passphrase signature and its signer
+// that l has once moved or copied, given the request's sig and email.
+//
+// As the SDK's moveNode and copyNode, a request gives them if and only
+// if l's key is anonymous: an anonymous key is then signed by the
+// user, and a signed one keeps its signature, as the passphrase is
+// unchanged.
+func keySignature(u *User, l *link, sig, email *string) (string, string, error) {
+	if l.signatureEmail != "" {
+		if (sig != nil && *sig != "") || (email != nil && *email != "") {
+			return "", "", errors.New("NodePassphraseSignature and SignatureEmail must not be given for a signed node")
+		}
+		return l.nodePass.signature, l.signatureEmail, nil
+	}
+	if sig == nil || *sig == "" {
+		return "", "", errors.New("NodePassphraseSignature is required for an anonymous node")
+	}
+	if err := checkSigner(u, "SignatureEmail", email); err != nil {
+		return "", "", err
+	}
+	return *sig, *email, nil
+}
+
+// handleCreateFolder creates a folder from key material the client
+// generated.
+func (s *Server) handleCreateFolder(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var req createFolderRequest
+	if !decodeRequest(w, r, &req) {
+		return
+	}
+	parent := d.activeFolder(req.ParentLinkID)
+	if parent == nil {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
+		return
+	}
+	messages := map[string]string{
+		"Name":           req.Name,
+		"NodePassphrase": req.NodePassphrase,
+		"NodeHashKey":    req.NodeHashKey,
+	}
+	if req.XAttr != nil {
+		messages["XAttr"] = *req.XAttr
+	}
+	err := errors.Join(
+		checkHash(req.Hash),
+		checkSigner(u, "SignatureEmail", req.SignatureEmail),
+		checkMessages(messages),
+	)
+	if err == nil && req.NodePassphraseSignature == "" {
+		err = errors.New("NodePassphraseSignature is required")
+	}
+	if err == nil {
+		err = annotateField("NodeKey", checkFolderKey(req.NodeKey))
+	}
+	if err != nil {
+		writeInvalid(w, err)
+		return
+	}
+	if other := d.conflict(parent.id, req.Hash, ""); other != nil {
+		writeConflict(w, other)
+		return
+	}
+	now := time.Now().Unix()
+	l := &link{
+		id:                 randomID(),
+		parentID:           parent.id,
+		typ:                linkTypeFolder,
+		state:              linkStateActive,
+		createTime:         now,
+		modifyTime:         now,
+		name:               req.Name,
+		nameHash:           req.Hash,
+		nodeKey:            req.NodeKey,
+		nodePass:           sealedPassphrase{armored: req.NodePassphrase, signature: req.NodePassphraseSignature},
+		signatureEmail:     *req.SignatureEmail,
+		nameSignatureEmail: *req.SignatureEmail,
+		hashKey:            req.NodeHashKey,
+	}
+	if req.XAttr != nil {
+		l.xattr = *req.XAttr
+	}
+	d.addLink(l)
+	s.lagLocked(l.id, nil)
+	writeJSON(w, http.StatusOK, &createFolderResponse{Folder: folderIDJSON{ID: l.id}, Code: api.CodeOK})
+}
+
+// annotateField returns err prefixed with field, or nil.
+func annotateField(field string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("%s: %w", field, err)
+}
+
+// handleRename renames a link within its folder.
+func (s *Server) handleRename(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var req renameRequest
+	if !decodeRequest(w, r, &req) {
+		return
+	}
+	l := d.activeLink(r.PathValue("linkID"))
+	switch {
+	case l == nil:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File or folder not found")
+		return
+	case l.parentID == "":
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: the root can't be renamed")
+		return
+	case req.OriginalHash == nil || *req.OriginalHash != l.nameHash:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: OriginalHash doesn't match: the node changed")
+		return
+	}
+	hash := ""
+	if req.Hash != nil {
+		hash = *req.Hash
+	}
+	err := errors.Join(
+		checkHash(hash),
+		checkSigner(u, "NameSignatureEmail", req.NameSignatureEmail),
+		checkMessages(map[string]string{"Name": req.Name}),
+	)
+	if err != nil {
+		writeInvalid(w, err)
+		return
+	}
+	if other := d.conflict(l.parentID, hash, l.id); other != nil {
+		writeConflict(w, other)
+		return
+	}
+	l.name, l.nameHash, l.nameSignatureEmail = req.Name, hash, *req.NameSignatureEmail
+	l.modifyTime = time.Now().Unix()
+	writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
+}
+
+// handleMove moves a link to another folder, possibly renaming it.
+func (s *Server) handleMove(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var req moveRequest
+	if !decodeRequest(w, r, &req) {
+		return
+	}
+	l := d.activeLink(r.PathValue("linkID"))
+	parent := d.activeFolder(req.ParentLinkID)
+	switch {
+	case l == nil:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File or folder not found")
+		return
+	case l.parentID == "":
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: the root can't be moved")
+		return
+	case parent == nil:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
+		return
+	case d.within(parent.id, l.id):
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: a folder can't be moved into itself")
+		return
+	case req.OriginalHash != l.nameHash:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: OriginalHash doesn't match: the node changed")
+		return
+	}
+	err := errors.Join(
+		checkHash(req.Hash),
+		checkSigner(u, "NameSignatureEmail", req.NameSignatureEmail),
+		checkMessages(map[string]string{"Name": req.Name, "NodePassphrase": req.NodePassphrase}),
+	)
+	if err != nil {
+		writeInvalid(w, err)
+		return
+	}
+	sig, signer, err := keySignature(u, l, req.NodePassphraseSignature, req.SignatureEmail)
+	if err != nil {
+		writeInvalid(w, err)
+		return
+	}
+	if other := d.conflict(parent.id, req.Hash, l.id); other != nil {
+		writeConflict(w, other)
+		return
+	}
+	l.parentID, l.name, l.nameHash, l.nameSignatureEmail = parent.id, req.Name, req.Hash, *req.NameSignatureEmail
+	l.nodePass = sealedPassphrase{armored: req.NodePassphrase, signature: sig}
+	l.signatureEmail = signer
+	l.modifyTime = time.Now().Unix()
+	writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
+}
+
+// handleCopy copies a link, and a folder's active descendants, into a
+// folder. Only the name and passphrase of the copy come from the
+// client; its key, content and descendants are the server's copies.
+func (s *Server) handleCopy(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var req copyRequest
+	if !decodeRequest(w, r, &req) {
+		return
+	}
+	l := d.activeLink(r.PathValue("linkID"))
+	parent := d.activeFolder(req.TargetParentLinkID)
+	switch {
+	case l == nil:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "File or folder not found")
+		return
+	case l.parentID == "":
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: the root can't be copied")
+		return
+	case req.TargetVolumeID != d.volumeID:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Volume not found")
+		return
+	case parent == nil:
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Parent folder not found")
+		return
+	case d.within(parent.id, l.id):
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidRequirements, "protontest: a folder can't be copied into itself")
+		return
+	}
+	err := errors.Join(
+		checkHash(req.Hash),
+		checkSigner(u, "NameSignatureEmail", req.NameSignatureEmail),
+		checkMessages(map[string]string{"Name": req.Name, "NodePassphrase": req.NodePassphrase}),
+	)
+	if err != nil {
+		writeInvalid(w, err)
+		return
+	}
+	sig, signer, err := keySignature(u, l, req.NodePassphraseSignature, req.SignatureEmail)
+	if err != nil {
+		writeInvalid(w, err)
+		return
+	}
+	if other := d.conflict(parent.id, req.Hash, ""); other != nil {
+		writeConflict(w, other)
+		return
+	}
+	c := d.copyLink(l, parent.id, time.Now().Unix())
+	c.name, c.nameHash, c.nameSignatureEmail = req.Name, req.Hash, *req.NameSignatureEmail
+	c.nodePass = sealedPassphrase{armored: req.NodePassphrase, signature: sig}
+	c.signatureEmail = signer
+	s.lagLocked(c.id, nil)
+	writeJSON(w, http.StatusOK, &copyResponse{LinkID: c.id, Code: api.CodeOK})
+}
+
+// copyLink stores and returns a copy of l in folder parentID with its
+// active revision and, for a folder, copies of its active descendants.
+// Blocks are shared, as they are immutable.
+func (d *drive) copyLink(l *link, parentID string, now int64) *link {
+	c := *l
+	c.id, c.parentID = randomID(), parentID
+	c.state, c.createTime, c.modifyTime, c.trashTime = linkStateActive, now, now, 0
+	c.revisions = nil
+	if r := l.activeRevision(); r != nil {
+		rc := *r
+		rc.id = randomID()
+		rc.blocks = slices.Clone(r.blocks)
+		c.revisions = []*revision{&rc}
+	}
+	var children []*link
+	for _, child := range d.links {
+		if child.parentID == l.id && child.state == linkStateActive {
+			children = append(children, child)
+		}
+	}
+	sort.Slice(children, func(i, j int) bool { return children[i].seq < children[j].seq })
+	d.addLink(&c)
+	for _, child := range children {
+		d.copyLink(child, c.id, now)
+	}
+	return &c
+}
+
+// handleTrash trashes up to 100 links, answering for each.
+func (s *Server) handleTrash(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var req linksRequest
+	if !decodeRequest(w, r, &req) {
+		return
+	}
+	if len(req.LinkIDs) > maxLinksPerRequest {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeInvalidValue, fmt.Sprintf("protontest: %d link IDs, at most %d allowed", len(req.LinkIDs), maxLinksPerRequest))
+		return
+	}
+	now := time.Now().Unix()
+	resp := &multiResponse{Code: api.CodeOKMany, Responses: []linkResultJSON{}}
+	for _, id := range req.LinkIDs {
+		result := linkResultJSON{LinkID: id, Response: codeJSON{Code: api.CodeOK}}
+		l := d.activeLink(id)
+		switch {
+		case l == nil:
+			result.Response = codeJSON{Code: api.CodeNotExists, Error: "File or folder not found"}
+		case l.parentID == "":
+			result.Response = codeJSON{Code: api.CodeInvalidRequirements, Error: "protontest: the root can't be trashed"}
+		default:
+			l.state, l.trashTime, l.modifyTime = linkStateTrashed, now, now
+		}
+		resp.Responses = append(resp.Responses, result)
+	}
+	writeJSON(w, http.StatusOK, resp)
+}
+
+// handleEmptyTrash deletes every trashed link with its descendants.
+func (s *Server) handleEmptyTrash(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var doomed []string
+	for id, l := range d.links {
+		if d.inTrash(l) {
+			doomed = append(doomed, id)
+		}
+	}
+	for _, id := range doomed {
+		delete(d.links, id)
+	}
+	writeJSON(w, http.StatusOK, &codeJSON{Code: api.CodeOK})
+}
+
+// handleCheckAvailableHashes reports which name hashes are free in a
+// folder, and the drafts holding the others unless they belong to one
+// of the given ClientUIDs.
+func (s *Server) handleCheckAvailableHashes(w http.ResponseWriter, r *http.Request, u *User, d *drive) {
+	var req checkHashesRequest
+	if !decodeRequest(w, r, &req) {
+		return
+	}
+	parent := d.activeFolder(r.PathValue("linkID"))
+	if parent == nil {
+		writeError(w, http.StatusUnprocessableEntity, api.CodeNotExists, "Folder not found")
+		return
+	}
+	resp := &availableHashesResponse{AvailableHashes: []string{}, PendingHashes: []pendingHashJSON{}, Code: api.CodeOK}
+	for _, hash := range req.Hashes {
+		other := d.conflict(parent.id, hash, "")
+		if other == nil {
+			resp.AvailableHashes = append(resp.AvailableHashes, hash)
+			continue
+		}
+		if other.state != linkStateDraft {
+			continue
+		}
+		for _, rev := range other.revisions {
+			if rev.state == revisionStateDraft && !slices.Contains(req.ClientUID, rev.clientUID) {
+				resp.PendingHashes = append(resp.PendingHashes, pendingHashJSON{
+					Hash:       hash,
+					RevisionID: rev.id,
+					LinkID:     other.id,
+					ClientUID:  nullable(rev.clientUID),
+				})
+			}
+		}
+	}
+	writeJSON(w, http.StatusOK, resp)
+}