Version v1.76.0

The rc server decoded the whole POST body before looking up the method
and checking whether it needed authorisation, then echoed the decoded
input back in the 403 or 404 error. A client that was not allowed to
call a method could still make rclone read, decode and re-encode its
request.

The method lookup and authorisation check now happen before the body
is read, and these errors no longer echo the request input.

.gitattributes

@@ -8,6 +8,7 @@ lib/http/templates/** text eol=lf
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true
+cmd/bisync/testdata/*/golden/** linguist-generated=true
 
 # Don't fiddle with the line endings of test data
 **/testdata/** -text

.github/workflows/build.yml

@@ -37,7 +37,7 @@ jobs:
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -48,14 +48,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -64,14 +64,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -81,14 +81,14 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '~1.27.2'
+            go: '~1.27.3'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
           - job_name: go1.26
             os: ubuntu-latest
-            go: '~1.26.9'
+            go: '~1.26.10'
             quicktest: true
             racequicktest: true
 
@@ -177,9 +177,12 @@ jobs:
           # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
           # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
           unset HOMEBREW_NO_INSTALL_FROM_API
-          brew untap --force homebrew/core
-          brew untap --force homebrew/cask
-          brew untap --force aws/tap
+          # brew untap fails if the tap isn't installed, which varies between runner images
+          for tap in homebrew/core homebrew/cask aws/tap; do
+            if brew tap | grep -qx "$tap"; then
+              brew untap --force "$tap"
+            fi
+          done
           brew update
           brew install --cask macfuse
           brew install git-annex git-annex-remote-rclone
@@ -284,7 +287,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.27.2'
+          go-version: '~1.27.3'
           check-latest: true
           cache: false
           
@@ -399,7 +402,7 @@ jobs:
         id: setup-go
         uses: actions/setup-go@v7
         with:
-          go-version: '~1.27.2'
+          go-version: '~1.27.3'
           # Caching is handled explicitly below to share the module cache
           # with the other jobs - see the build job for the rationale.
           cache: false

.gitignore

@@ -20,3 +20,4 @@ __pycache__
 .DS_Store
 resource_windows_*.syso
 .devcontainer
+/cmd/bisync/testdata/bisync_vscode_debuggers_launch.json

README.md

@@ -28,12 +28,14 @@ directories to and from different cloud storage providers.
 - Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
 - Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
 - ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
+- Autonomys Auto Drive [:page_facing_up:](https://rclone.org/s3/#autonomys)
 - Bizfly Cloud Simple Storage [:page_facing_up:](https://rclone.org/s3/#bizflycloud)
 - Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
 - Box [:page_facing_up:](https://rclone.org/box/)
 - Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
 - China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
 - Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
+- Clever Cloud Cellar [:page_facing_up:](https://rclone.org/s3/#clever-cloud-cellar)
 - Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
 - Cloudinary [:page_facing_up:](https://rclone.org/cloudinary/)
 - Cubbit DS3 [:page_facing_up:](https://rclone.org/s3/#Cubbit)
@@ -71,8 +73,11 @@ directories to and from different cloud storage providers.
 - Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
 - IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
 - Impossible Cloud [:page_facing_up:](https://rclone.org/s3/#impossible-cloud)
+- Infomaniak Swiss Backup (S3) [:page_facing_up:](https://rclone.org/s3/#infomaniak)
+- Infomaniak Swiss Backup (Swift) [:page_facing_up:](https://rclone.org/swift/)
 - Intercolo Object Storage [:page_facing_up:](https://rclone.org/s3/#intercolo)
 - IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
+- kDrive [:page_facing_up:](https://rclone.org/kdrive/)
 - Koofr [:page_facing_up:](https://rclone.org/koofr/)
 - Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
 - Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
@@ -88,6 +93,7 @@ directories to and from different cloud storage providers.
 - Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
 - Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
 - Minio [:page_facing_up:](https://rclone.org/s3/#minio)
+- Neon Object Storage [:page_facing_up:](https://rclone.org/s3/#neon)
 - Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
 - Blomp Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
 - OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)

RELEASE.md

@@ -62,10 +62,14 @@ This file describes how to make the various kinds of releases
 Early in the next release cycle update the dependencies.
 
 - Active Pins
-  - `github.com/gdamore/tcell/v2` v2.9.0 - awaiting fix for [tcell/issues/1124](https://github.com/gdamore/tcell/issues/1124)
-    - See: [#9539](https://github.com/rclone/rclone/issues/9539)
+  - `github.com/coreos/go-systemd/v22` v22.6.0 - v22.7.0 fails to compile on netbsd
+  - `github.com/hanwen/go-fuse/v2` v2.11.0 - v2.12.0 fails to compile on 32
+    bit linux, awaiting fix for
+    [go-fuse/issues/628](https://github.com/hanwen/go-fuse/issues/628)
 - Review Active Pins in go.mod and remove if possible
 - `make updatedirect`
+- The update ignores the pins so put them back, eg
+  `go get github.com/coreos/go-systemd/v22@v22.6.0` then `go mod tidy`
 - `make GOTAGS=cmount`
 - `make compiletest`
 - Fix anything which doesn't compile at this point and commit changes here

backend/all/all.go

@@ -39,6 +39,7 @@ import (
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/internxt"
 	_ "github.com/rclone/rclone/backend/jottacloud"
+	_ "github.com/rclone/rclone/backend/kdrive"
 	_ "github.com/rclone/rclone/backend/koofr"
 	_ "github.com/rclone/rclone/backend/linkbox"
 	_ "github.com/rclone/rclone/backend/local"

backend/archive/archive.go

@@ -194,8 +194,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 		}
 		// Path to the archive
 		archiveRemote := remote[:len(foundArchive.remote)]
-		// Remote is archive leaf name
-		foundArchive.remote = path.Base(archiveRemote)
+		// Remote is archive leaf name. This must understand remote
+		// names as the archive may be in the root, e.g. "remote:file.zip"
+		_, foundArchive.remote, err = fspath.Split(archiveRemote)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse archive path %q: %w", archiveRemote, err)
+		}
 		foundArchive.prefix = ""
 		// Point remote to archive file
 		remote = archiveRemote

backend/archive/archive_internal_test.go

@@ -17,6 +17,7 @@ import (
 	"strings"
 	"testing"
 
+	_ "github.com/rclone/rclone/backend/alias"
 	"github.com/rclone/rclone/backend/archive/archiver"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
@@ -323,6 +324,50 @@ func TestArchiveUncleanRoot(t *testing.T) {
 	}
 }
 
+// TestArchiveInRemoteRoot checks that an archive in the root of a
+// remote, with no "/" between the remote name and the archive name, can
+// be read, as can a path within it.
+func TestArchiveInRemoteRoot(t *testing.T) {
+	fstest.Initialise()
+	ctx := context.Background()
+
+	var buf bytes.Buffer
+	zw := zip.NewWriter(&buf)
+	w, err := zw.Create("sub/a.txt")
+	require.NoError(t, err)
+	_, err = w.Write([]byte("data"))
+	require.NoError(t, err)
+	require.NoError(t, zw.Close())
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "test.zip"), buf.Bytes(), 0600))
+	t.Chdir(dir)
+
+	for _, test := range []struct {
+		name   string
+		remote string
+	}{
+		{name: "plain", remote: ":local:test.zip"},
+		{name: "slash in remote name", remote: ":alias,remote='" + dir + "':test.zip"},
+	} {
+		for _, sub := range []struct {
+			path string
+			want string
+		}{
+			{path: "", want: "sub"},
+			{path: "/sub", want: "a.txt"},
+		} {
+			t.Run(test.name+sub.path, func(t *testing.T) {
+				f, err := cache.Get(ctx, ":archive:"+test.remote+sub.path)
+				require.NoError(t, err)
+				entries, err := f.List(ctx, "")
+				require.NoError(t, err)
+				require.Len(t, entries, 1)
+				assert.Equal(t, sub.want, entries[0].Remote())
+			})
+		}
+	}
+}
+
 // escapingObject is an object whose remote is not where it was asked for.
 type escapingObject struct {
 	fs.Object

backend/azurefiles/azurefiles.go

@@ -79,7 +79,7 @@ This is required and is the name of the share to access.
 			Help: `Upload chunk size.
 
 Note that this is stored in memory and there may be up to
-"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
+"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
 in memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
@@ -95,7 +95,7 @@ links and these uploads do not fully utilize your bandwidth, then
 increasing this may help to speed up the transfers.
 
 Note that chunks are stored in memory and there may be up to
-"--transfers" * "--azurefile-upload-concurrency" chunks stored at once
+"--transfers" * "--azurefiles-upload-concurrency" chunks stored at once
 in memory.`,
 			Default:  16,
 			Advanced: true,

backend/b2/b2.go

@@ -277,7 +277,7 @@ See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket
 			Name: "sse_customer_key",
 			Help: `To use SSE-C, you may provide the secret encryption key encoded in a UTF-8 compatible string to encrypt/decrypt your data
 
-Alternatively you can provide --sse-customer-key-base64.`,
+Alternatively you can provide --b2-sse-customer-key-base64.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -288,7 +288,7 @@ Alternatively you can provide --sse-customer-key-base64.`,
 			Name: "sse_customer_key_base64",
 			Help: `To use SSE-C, you may provide the secret encryption key encoded in Base64 format to encrypt/decrypt your data
 
-Alternatively you can provide --sse-customer-key.`,
+Alternatively you can provide --b2-sse-customer-key.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",

backend/box/box.go

@@ -27,7 +27,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/golang-jwt/jwt/v4"
+	"github.com/golang-jwt/jwt/v5"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
@@ -77,7 +77,10 @@ var (
 )
 
 type boxCustomClaims struct {
-	jwt.StandardClaims
+	jwt.RegisteredClaims
+	// Audience overrides RegisteredClaims.Audience so it is sent as
+	// a single string rather than an array of strings
+	Audience   string `json:"aud,omitempty"`
 	BoxSubType string `json:"box_sub_type,omitempty"`
 }
 
@@ -223,7 +226,7 @@ func getBoxConfig(m configmap.Mapper) (boxConfig *api.ConfigJSON, err error) {
 
 	if configFileCredentials == "" {
 		configFile, _ := m.Get("box_config_file")
-		configFileBytes, err = os.ReadFile(configFile)
+		configFileBytes, err = os.ReadFile(env.ShellExpand(configFile))
 		if err != nil {
 			return nil, fmt.Errorf("box: failed to read Box config: %w", err)
 		}
@@ -243,15 +246,13 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	}
 
 	claims = &boxCustomClaims{
-		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
-		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
-		StandardClaims: jwt.StandardClaims{
-			Id:        val,
+		RegisteredClaims: jwt.RegisteredClaims{
+			ID:        val,
 			Issuer:    boxConfig.BoxAppSettings.ClientID,
 			Subject:   boxConfig.EnterpriseID,
-			Audience:  tokenURL,
-			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
+			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
 		},
+		Audience:   tokenURL,
 		BoxSubType: boxSubType,
 	}
 	return claims, nil

backend/box/box_internal_test.go

@@ -0,0 +1,118 @@
+package box
+
+import (
+	"crypto"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/sha256"
+	"encoding/base64"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/box/api"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/lib/jwtutil"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// decodeJWTPart decodes one base64url encoded part of a JWT into a map
+func decodeJWTPart(t *testing.T, part string) map[string]any {
+	data, err := base64.RawURLEncoding.DecodeString(part)
+	require.NoError(t, err)
+	var result map[string]any
+	require.NoError(t, json.Unmarshal(data, &result))
+	return result
+}
+
+// Test the JWT assertion sent to the token endpoint is in the format Box expects
+func TestJWTAssertion(t *testing.T) {
+	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+	boxConfig := &api.ConfigJSON{
+		BoxAppSettings: api.AppSettings{
+			ClientID:     "client-id",
+			ClientSecret: "client-secret",
+			AppAuth: api.AppAuth{
+				PublicKeyID: "public-key-id",
+			},
+		},
+		EnterpriseID: "enterprise-id",
+	}
+
+	var form map[string][]string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "POST", r.Method)
+		assert.NoError(t, r.ParseForm())
+		form = r.PostForm
+		_, _ = w.Write([]byte(`{"access_token":"access-token","token_type":"bearer","expires_in":3600}`))
+	}))
+	defer ts.Close()
+
+	claims, err := getClaims(boxConfig, "enterprise")
+	require.NoError(t, err)
+	m := configmap.Simple{}
+	start := time.Now()
+	err = jwtutil.Config("box", "test", ts.URL, *claims, getSigningHeaders(boxConfig), getQueryParams(boxConfig), privateKey, m, ts.Client())
+	require.NoError(t, err)
+
+	// Check the token was stored
+	assert.Contains(t, m["token"], `"access_token":"access-token"`)
+
+	// Check the form parameters
+	assert.Equal(t, []string{"urn:ietf:params:oauth:grant-type:jwt-bearer"}, form["grant_type"])
+	assert.Equal(t, []string{"client-id"}, form["client_id"])
+	assert.Equal(t, []string{"client-secret"}, form["client_secret"])
+	require.Len(t, form["assertion"], 1)
+	parts := strings.Split(form["assertion"][0], ".")
+	require.Len(t, parts, 3)
+
+	// Check the signature independently of the JWT library
+	signature, err := base64.RawURLEncoding.DecodeString(parts[2])
+	require.NoError(t, err)
+	hash := sha256.Sum256([]byte(parts[0] + "." + parts[1]))
+	require.NoError(t, rsa.VerifyPKCS1v15(&privateKey.PublicKey, crypto.SHA256, hash[:], signature))
+
+	// Check the header
+	assert.Equal(t, map[string]any{
+		"alg": "RS256",
+		"typ": "JWT",
+		"kid": "public-key-id",
+	}, decodeJWTPart(t, parts[0]))
+
+	// Check the claims - aud must be a string, not an array
+	payload := decodeJWTPart(t, parts[1])
+	exp, ok := payload["exp"].(float64)
+	require.True(t, ok, "exp should be a number")
+	assert.InDelta(t, float64(start.Add(45*time.Second).Unix()), exp, 5)
+	jti, ok := payload["jti"].(string)
+	require.True(t, ok, "jti should be a string")
+	assert.Len(t, jti, 40)
+	delete(payload, "exp")
+	delete(payload, "jti")
+	assert.Equal(t, map[string]any{
+		"aud":          tokenURL,
+		"iss":          "client-id",
+		"sub":          "enterprise-id",
+		"box_sub_type": "enterprise",
+	}, payload)
+}
+
+// Test box_config_file has ~ and environment variables expanded
+func TestGetBoxConfigShellExpand(t *testing.T) {
+	dir := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.json"), []byte(`{"enterpriseID":"enterprise-id"}`), 0600))
+	t.Setenv("RCLONE_TEST_BOX_CONFIG_DIR", dir)
+	m := configmap.Simple{
+		"box_config_file": "$RCLONE_TEST_BOX_CONFIG_DIR/config.json",
+	}
+	boxConfig, err := getBoxConfig(m)
+	require.NoError(t, err)
+	assert.Equal(t, "enterprise-id", boxConfig.EnterpriseID)
+}

backend/chunker/chunker.go

@@ -1502,11 +1502,21 @@ func (c *chunkingReader) rollback(ctx context.Context, metaObject fs.Object) {
 func (f *Fs) removeOldChunks(ctx context.Context, remote string) {
 	oldFsObject, err := f.NewObject(ctx, remote)
 	if err == nil {
-		oldObject := oldFsObject.(*Object)
-		for _, chunk := range oldObject.chunks {
-			if err := chunk.Remove(ctx); err != nil {
-				fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
-			}
+		removeStaleChunks(ctx, oldFsObject.(*Object), nil)
+	}
+}
+
+func removeStaleChunks(ctx context.Context, oldObject *Object, currentChunks []fs.Object) {
+	currentRemotes := make(map[string]struct{}, len(currentChunks))
+	for _, chunk := range currentChunks {
+		currentRemotes[chunk.Remote()] = struct{}{}
+	}
+	for _, chunk := range oldObject.chunks {
+		if _, found := currentRemotes[chunk.Remote()]; found {
+			continue
+		}
+		if err := chunk.Remove(ctx); err != nil {
+			fs.Errorf(chunk, "Failed to remove old chunk: %v", err)
 		}
 	}
 }
@@ -1687,12 +1697,20 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		// metadata format which might involve unsupported chunk types.
 		return nil, fmt.Errorf("can't %s this file: %w", opName, err)
 	}
+	// Snapshot destination chunks before its metadata is replaced.
+	var oldObject *Object
+	if oldFsObject, err := f.NewObject(ctx, remote); err == nil {
+		oldObject = oldFsObject.(*Object)
+	}
 	if !o.isComposite() {
 		fs.Debugf(o, "%s non-chunked object...", opName)
 		oResult, err := do(ctx, o.mainChunk(), remote) // chain operation to a single wrapped chunk
 		if err != nil {
 			return nil, err
 		}
+		if oldObject != nil {
+			removeStaleChunks(ctx, oldObject, nil)
+		}
 		return f.newObject("", oResult, nil), nil
 	}
 
@@ -1758,6 +1776,9 @@ func (f *Fs) copyOrMove(ctx context.Context, o *Object, remote string, do copyMo
 		silentlyRemove(ctx, newObj)
 		return nil, err
 	}
+	if oldObject != nil {
+		removeStaleChunks(ctx, oldObject, newChunks)
+	}
 	return newObj, nil
 }
 

backend/chunker/chunker_internal_test.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	_ "github.com/rclone/rclone/backend/memory"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/fspath"
@@ -877,6 +878,104 @@ func testChunkerServerSideMove(t *testing.T, f *Fs) {
 	_ = operations.Purge(ctx, f.base, dir)
 }
 
+func testChunkerServerSideOverwrite(t *testing.T, moveFs *Fs) {
+	ctx := context.Background()
+	const dir = "serveroverwrite"
+	remote := fmt.Sprintf(":chunker,remote=':memory:%s',chunk_size=3B:", dir)
+	fsResult, err := fs.NewFs(ctx, remote)
+	require.NoError(t, err)
+	copyFs, ok := fsResult.(*Fs)
+	require.True(t, ok)
+	oldMoveChunkSize := moveFs.opt.ChunkSize
+	moveFs.opt.ChunkSize = fs.SizeSuffix(3)
+	defer func() {
+		moveFs.opt.ChunkSize = oldMoveChunkSize
+		_ = operations.Purge(ctx, copyFs.base, "")
+		_ = operations.Purge(ctx, moveFs.base, dir)
+	}()
+
+	type operation struct {
+		name    string
+		fs      *Fs
+		enabled bool
+		do      copyMoveFn
+	}
+	operations := []operation{
+		{name: "Copy", fs: copyFs, enabled: copyFs.Features().Copy != nil, do: copyFs.Copy},
+		{name: "Move", fs: moveFs, enabled: moveFs.Features().Move != nil, do: moveFs.Move},
+	}
+	testCases := []struct {
+		name           string
+		sourceContents string
+		targetContents string
+	}{
+		{name: "SmallOverComposite", sourceContents: "hi", targetContents: "abcdef"},
+		{name: "FewerChunks", sourceContents: "abcd", targetContents: "abcdefghi"},
+	}
+	transactionModes := []struct {
+		name        string
+		useNoRename bool
+	}{
+		{name: "Rename"},
+		{name: "NoRename", useNoRename: true},
+	}
+
+	for _, operation := range operations {
+		t.Run(operation.name, func(t *testing.T) {
+			if !operation.enabled {
+				t.Skipf("wrapped remote does not support server-side %s", strings.ToLower(operation.name))
+			}
+			for _, transactionMode := range transactionModes {
+				t.Run(transactionMode.name, func(t *testing.T) {
+					f := operation.fs
+					if transactionMode.useNoRename && !f.useMeta {
+						t.Skip("Can't test norename transactions without metadata")
+					}
+					oldUseNoRename := f.useNoRename
+					f.useNoRename = transactionMode.useNoRename
+					defer func() { f.useNoRename = oldUseNoRename }()
+					for _, testCase := range testCases {
+						t.Run(testCase.name, func(t *testing.T) {
+							prefix := path.Join(dir, strings.ToLower(operation.name), strings.ToLower(transactionMode.name), strings.ToLower(testCase.name))
+							source := testPutFile(ctx, t, f, prefix+"-source", testCase.sourceContents, "source", true)
+							target := testPutFile(ctx, t, f, prefix+"-target", testCase.targetContents, "target", true)
+							targetObject := target.(*Object)
+							require.NotEmpty(t, targetObject.chunks)
+
+							oldChunks := append([]fs.Object(nil), targetObject.chunks...)
+							result, err := operation.do(ctx, source, target.Remote())
+							require.NoError(t, err)
+
+							reader, err := result.Open(ctx)
+							require.NoError(t, err)
+							contents, err := io.ReadAll(reader)
+							require.NoError(t, err)
+							require.NoError(t, reader.Close())
+							assert.Equal(t, testCase.sourceContents, string(contents))
+
+							resultObject := result.(*Object)
+							currentChunks := make(map[string]struct{}, len(resultObject.chunks))
+							for _, chunk := range resultObject.chunks {
+								currentChunks[chunk.Remote()] = struct{}{}
+							}
+							removed := 0
+							for _, chunk := range oldChunks {
+								if _, found := currentChunks[chunk.Remote()]; found {
+									continue
+								}
+								_, err := f.base.NewObject(ctx, chunk.Remote())
+								assert.Equal(t, fs.ErrorObjectNotFound, err)
+								removed++
+							}
+							assert.Positive(t, removed)
+						})
+					}
+				})
+			}
+		})
+	}
+}
+
 // Test that md5all creates metadata even for small files
 func testMD5AllSlow(t *testing.T, f *Fs) {
 	ctx := context.Background()
@@ -950,6 +1049,9 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("ChunkerServerSideMove", func(t *testing.T) {
 		testChunkerServerSideMove(t, f)
 	})
+	t.Run("ChunkerServerSideOverwrite", func(t *testing.T) {
+		testChunkerServerSideOverwrite(t, f)
+	})
 	t.Run("MD5AllSlow", func(t *testing.T) {
 		testMD5AllSlow(t, f)
 	})

backend/crypt/crypt.go

@@ -1197,7 +1197,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 		return "", nil
 	}
 	// if this is wrapping a local object then we work out the hash
-	if srcObj.Fs().Features().IsLocal {
+	if srcObj != nil && srcObj.Fs().Features().IsLocal {
 		// Read the data and encrypt it to calculate the hash
 		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
 		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)

backend/crypt/crypt_internal_test.go

@@ -93,6 +93,16 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
 }
 
+// Test ObjectInfo.Hash doesn't panic when the source is an
+// fs.OverrideRemote which wraps an fs.ObjectInfo that isn't an fs.Object
+func TestObjectInfoHashOverrideRemoteNotObject(t *testing.T) {
+	src := object.NewStaticObjectInfo("file", time.Now(), 3, true, nil, nil)
+	oi := (&Fs{}).newObjectInfo(fs.NewOverrideRemote(src, "new_remote"), nonce{})
+	gotHash, err := oi.Hash(context.Background(), hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, "", gotHash)
+}
+
 func testComputeHash(t *testing.T, f *Fs) {
 	var (
 		contents = random.String(100)

backend/drime/drime.go

@@ -321,10 +321,13 @@ func (f *Fs) getItem(ctx context.Context, id string, dirID string, leaf string)
 		}
 		return false
 	})
+	if err != nil {
+		return nil, err
+	}
 	if !found {
 		return nil, fs.ErrorObjectNotFound
 	}
-	return info, err
+	return info, nil
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -546,6 +549,11 @@ OUTER:
 		if err != nil {
 			return found, fmt.Errorf("couldn't list files: %w", err)
 		}
+		// Drime won't page past a fixed number of entries (20,000 for
+		// rclone) and serves the last page it allows again instead
+		if result.CurrentPage != page {
+			return found, errors.New("couldn't list files: directory has more entries than drime will list")
+		}
 		for _, item := range result.Data {
 			if item.Type == api.ItemTypeFolder {
 				if filesOnly {

backend/drime/drime_internal_test.go

@@ -0,0 +1,158 @@
+package drime
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/drime/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	testPerPage = 5  // entries per page the test server returns
+	testLimit   = 20 // entries the test server will page through
+)
+
+// listingServer serves /drive/file-entries from entries the way Drime
+// does.
+//
+// It returns testPerPage entries per page and won't page past testLimit
+// entries. Asking for a later page returns the last page it allows
+// again, with last_page one beyond it.
+//
+// It counts the requests made in requests.
+func listingServer(t *testing.T, entries []api.Item, requests *atomic.Int64) *httptest.Server {
+	t.Helper()
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests.Add(1)
+		assert.Equal(t, "/drive/file-entries", r.URL.Path)
+		params := r.URL.Query()
+		page, err := strconv.Atoi(params.Get("page"))
+		require.NoError(t, err)
+		maxPage := testLimit / testPerPage
+		currentPage := min(page, maxPage)
+		start := min((currentPage-1)*testPerPage, len(entries))
+		end := min(start+testPerPage, len(entries))
+		lastPage := currentPage
+		if page > maxPage || end < len(entries) {
+			lastPage = currentPage + 1
+		}
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(map[string]any{
+			"current_page": currentPage,
+			"last_page":    lastPage,
+			"per_page":     testPerPage,
+			"total":        (currentPage + 1) * testPerPage,
+			"data":         entries[start:end],
+		}))
+	}))
+}
+
+// makeEntries makes n file entries
+func makeEntries(n int) []api.Item {
+	entries := make([]api.Item, n)
+	for i := range entries {
+		entries[i] = api.Item{
+			ID:   json.Number(strconv.Itoa(1000 + i)),
+			Name: fmt.Sprintf("file%05d", i),
+			Type: "text",
+		}
+	}
+	return entries
+}
+
+// testFs makes an Fs which uses server as its API
+func testFs(t *testing.T, server *httptest.Server) *Fs {
+	t.Helper()
+	f := &Fs{
+		opt: Options{ListChunk: 1000},
+		srv: rest.NewClient(server.Client()).SetRoot(server.URL),
+	}
+	f.pacer = fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
+	f.srv.SetErrorHandler(errorHandler)
+	return f
+}
+
+// listNames lists a directory with listAll and returns the names found
+func listNames(ctx context.Context, f *Fs) (names []string, err error) {
+	_, err = f.listAll(ctx, "1", false, false, "", func(item *api.Item) bool {
+		names = append(names, item.Name)
+		return false
+	})
+	return names, err
+}
+
+// TestListAll checks that listAll returns every entry exactly once
+func TestListAll(t *testing.T) {
+	for _, test := range []struct {
+		name    string
+		entries int
+	}{
+		{name: "Empty", entries: 0},
+		{name: "OnePage", entries: 3},
+		{name: "ExactPages", entries: 10},
+		{name: "PartialLastPage", entries: 13},
+		{name: "AtLimit", entries: testLimit},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+			var requests atomic.Int64
+			entries := makeEntries(test.entries)
+			server := listingServer(t, entries, &requests)
+			defer server.Close()
+			f := testFs(t, server)
+
+			names, err := listNames(ctx, f)
+			require.NoError(t, err)
+			var want []string
+			for _, entry := range entries {
+				want = append(want, entry.Name)
+			}
+			assert.ElementsMatch(t, want, names)
+		})
+	}
+}
+
+// TestListAllTooManyEntries checks that listAll returns an error for a
+// directory with more entries than the server will list, rather than
+// looping forever.
+func TestListAllTooManyEntries(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	var requests atomic.Int64
+	server := listingServer(t, makeEntries(2*testLimit), &requests)
+	defer server.Close()
+	f := testFs(t, server)
+
+	_, err := listNames(ctx, f)
+	require.Error(t, err)
+	assert.NoError(t, ctx.Err(), "listing didn't finish")
+	assert.LessOrEqual(t, requests.Load(), int64(testLimit/testPerPage+1))
+}
+
+// TestGetItemListError checks that getItem returns listing errors
+// rather than fs.ErrorObjectNotFound.
+func TestGetItemListError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, `{"message":"access denied"}`, http.StatusForbidden)
+	}))
+	defer server.Close()
+	f := testFs(t, server)
+
+	_, err := f.getItem(context.Background(), "1000", "1", "file00000")
+	require.Error(t, err)
+	assert.NotErrorIs(t, err, fs.ErrorObjectNotFound)
+	assert.Contains(t, err.Error(), "access denied")
+}

backend/drive/drive.go

@@ -568,7 +568,7 @@ recommended usage is using the flag form --drive-size-as-quota when
 doing rclone ls/lsl/lsf/lsjson/etc only.
 
 If you do use this flag for syncing (not recommended) then you will
-need to use --ignore size also.`,
+need to use --ignore-size also.`,
 			Advanced: true,
 			Hide:     fs.OptionHideConfigurator,
 		}, {
@@ -2298,15 +2298,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
 		for len(overflow) > 0 {
-			mu.Lock()
-			// only fill half of the channel to prevent entries being put into overflow again
-			l := min(len(overflow), listRInputBuffer/2)
-			wg.Add(l)
-			for _, d := range overflow[:l] {
-				in <- d
+			if !refillListRInput(&mu, &in, &overflow, &wg) {
+				return
 			}
-			overflow = overflow[l:]
-			mu.Unlock()
 
 			// wait again for the completion of all directories
 			wg.Wait()
@@ -2354,6 +2348,25 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	return nil
 }
 
+// refillListRInput moves queued directory jobs back onto in.
+// It returns false when a listing error has already closed in.
+// Sending on that nil channel would block forever while holding mu.
+func refillListRInput(mu *sync.Mutex, in *chan listREntry, overflow *[]listREntry, wg *sync.WaitGroup) bool {
+	mu.Lock()
+	defer mu.Unlock()
+	if *in == nil {
+		return false
+	}
+	// only fill half of the channel to prevent entries being put into overflow again
+	l := min(len(*overflow), listRInputBuffer/2)
+	wg.Add(l)
+	for _, d := range (*overflow)[:l] {
+		*in <- d
+	}
+	*overflow = (*overflow)[l:]
+	return true
+}
+
 const shortcutSeparator = '\t'
 
 // joinID adds an actual drive ID to the shortcut ID it came from

backend/drive/drive_internal_test.go

@@ -916,3 +916,51 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+func TestRefillListRInputStopsWhenInputClosed(t *testing.T) {
+	mu := stdsync.Mutex{}
+	wg := stdsync.WaitGroup{}
+	in := make(chan listREntry, listRInputBuffer)
+	overflow := []listREntry{{id: "folder", path: "folder"}}
+
+	mu.Lock()
+	close(in)
+	in = nil
+	mu.Unlock()
+
+	done := make(chan bool, 1)
+	go func() {
+		done <- refillListRInput(&mu, &in, &overflow, &wg)
+	}()
+	select {
+	case ok := <-done:
+		require.False(t, ok, "refill continued after the input channel was closed")
+	case <-time.After(time.Second):
+		t.Fatal("refill blocked after the input channel was closed")
+	}
+	require.Len(t, overflow, 1)
+
+	finished := make(chan struct{})
+	go func() {
+		wg.Wait()
+		close(finished)
+	}()
+	select {
+	case <-finished:
+	case <-time.After(time.Second):
+		t.Fatal("refill incremented the wait group after the input channel was closed")
+	}
+}
+
+func TestRefillListRInputSendsQueuedDirectory(t *testing.T) {
+	mu := stdsync.Mutex{}
+	wg := stdsync.WaitGroup{}
+	in := make(chan listREntry, listRInputBuffer)
+	overflow := []listREntry{{id: "folder", path: "folder"}}
+
+	require.True(t, refillListRInput(&mu, &in, &overflow, &wg), "refill stopped while the input channel was open")
+	require.Empty(t, overflow)
+	got := <-in
+	require.Equal(t, listREntry{id: "folder", path: "folder"}, got)
+	wg.Done()
+}

backend/ftp/ftp.go

@@ -584,8 +584,9 @@ func (f *Fs) getFtpConnection(ctx context.Context) (c *ftp.ServerConn, err error
 //
 // It nils the pointed to connection out so it can't be reused
 //
-// if err is not nil then it checks the connection is alive using a
-// NOOP request
+// if err is not nil then it discards the connection unless the error
+// carries an FTP reply code, in which case it checks the connection
+// is alive using a NOOP request
 func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	if f.opt.Concurrency > 0 {
 		defer f.tokens.Put()
@@ -599,14 +600,17 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	}
 	*pc = nil
 	if err != nil {
-		// If not a regular FTP error code then check the connection
-		if tpErr := textprotoError(err); tpErr != nil {
-			nopErr := c.NoOp()
-			if nopErr != nil {
-				fs.Debugf(f, "Connection failed, closing: %v", nopErr)
-				_ = c.Quit()
-				return
-			}
+		// An error without an FTP reply code means the control
+		// connection can no longer be trusted, so discard it.
+		if textprotoError(err) == nil {
+			fs.Debugf(f, "Discarding connection after non-FTP error: %v", err)
+			_ = c.Quit()
+			return
+		}
+		if nopErr := c.NoOp(); nopErr != nil {
+			fs.Debugf(f, "Connection failed, closing: %v", nopErr)
+			_ = c.Quit()
+			return
 		}
 	}
 	f.poolMu.Lock()
@@ -1265,15 +1269,18 @@ func (o *Object) Storable() bool {
 
 // ftpReadCloser implements io.ReadCloser for FTP objects.
 type ftpReadCloser struct {
-	rc  io.ReadCloser
-	c   *ftp.ServerConn
-	f   *Fs
-	err error // errors found during read
+	rc     io.ReadCloser
+	c      *ftp.ServerConn
+	f      *Fs
+	err    error // errors found during read
+	closed bool  // set by Close so a second call is a no-op
+	read   int64 // bytes returned by Read
 }
 
 // Read bytes into p
 func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 	n, err = f.rc.Read(p)
+	f.read += int64(n)
 	if err != nil && err != io.EOF {
 		f.err = err // store any errors for Close to examine
 	}
@@ -1282,6 +1289,10 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
+	if f.closed {
+		return nil
+	}
+	f.closed = true
 	var err error
 	errchan := make(chan error, 1)
 	go func() {
@@ -1305,6 +1316,7 @@ func (f *ftpReadCloser) Close() error {
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
+		f.c = nil // never pool a connection after Quit
 		f.f.putFtpConnection(nil, nil)
 	} else {
 		f.f.putFtpConnection(&f.c, nil)
@@ -1316,6 +1328,12 @@ func (f *ftpReadCloser) Close() error {
 		switch errX.Code {
 		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend, ftp.StatusRequestedFileActionOK:
 			err = nil
+		case ftp.StatusCanNotOpenDataConnection:
+			// Egnyte answers 425 when a RETR is closed before the first
+			// read; a transfer cut after data was read stays an error
+			if f.read == 0 {
+				err = nil
+			}
 		}
 	}
 	return err

backend/ftp/ftp_internal_test.go

@@ -1,12 +1,19 @@
 package ftp
 
 import (
+	"bufio"
 	"context"
+	"errors"
 	"fmt"
+	"io"
+	"net"
+	"net/textproto"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
@@ -144,3 +151,159 @@ func (f *Fs) InternalTest(t *testing.T) {
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)
+
+// fakeFTPServer is a minimal FTP control-connection server which answers
+// every command with 200 OK.
+type fakeFTPServer struct {
+	ln   net.Listener
+	mu   sync.Mutex
+	conn net.Conn
+}
+
+func startFakeFTPServer(t *testing.T) *fakeFTPServer {
+	t.Helper()
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+	s := &fakeFTPServer{ln: ln}
+	go s.serve()
+	t.Cleanup(func() { s.close() })
+	return s
+}
+
+func (s *fakeFTPServer) serve() {
+	conn, err := s.ln.Accept()
+	if err != nil {
+		return
+	}
+	s.mu.Lock()
+	s.conn = conn
+	s.mu.Unlock()
+	defer func() { _ = conn.Close() }()
+	_, _ = fmt.Fprint(conn, "220 fake FTP server\r\n")
+	scanner := bufio.NewScanner(conn)
+	for scanner.Scan() {
+		_, _ = fmt.Fprint(conn, "200 OK\r\n")
+	}
+}
+
+// killConnection closes the server side of the control connection,
+// simulating a server that dropped an idle connection.
+func (s *fakeFTPServer) killConnection() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.conn != nil {
+		_ = s.conn.Close()
+	}
+}
+
+func (s *fakeFTPServer) close() {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	if s.conn != nil {
+		_ = s.conn.Close()
+	}
+	_ = s.ln.Close()
+}
+
+// TestPutFtpConnectionDiscardsDeadConnection checks that a connection
+// returned with a raw network error is closed and never re-pooled, while
+// a connection returned with a plain FTP reply-code error is probed with
+// NOOP and pooled while the probe succeeds.
+func TestPutFtpConnectionDiscardsDeadConnection(t *testing.T) {
+	t.Run("network error discards", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, errors.New("unexpected EOF"))
+		assert.Nil(t, c)
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("FTP reply error with live connection pools", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
+		assert.Nil(t, c)
+		require.Len(t, f.pool, 1)
+		assert.NoError(t, f.pool[0].NoOp())
+		_ = f.pool[0].Quit()
+	})
+
+	t.Run("FTP reply error with dead connection discards", func(t *testing.T) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		server.killConnection()
+		// Let the FIN propagate so the NOOP probe observes the dead connection.
+		time.Sleep(100 * time.Millisecond)
+		f := &Fs{opt: Options{}}
+		f.putFtpConnection(&c, &textproto.Error{Code: 550, Msg: "not found"})
+		assert.Nil(t, c)
+		assert.Empty(t, f.pool)
+	})
+}
+
+// fakeDataReader stands in for the RETR data stream of ftpReadCloser.
+type fakeDataReader struct {
+	data     []byte
+	closeErr error
+}
+
+func (r *fakeDataReader) Read(p []byte) (int, error) {
+	if len(r.data) == 0 {
+		return 0, io.EOF
+	}
+	n := copy(p, r.data)
+	r.data = r.data[n:]
+	return n, nil
+}
+
+func (r *fakeDataReader) Close() error { return r.closeErr }
+
+// TestFtpReadCloserClose checks that an early-closed RETR answered with 425
+// is masked only when nothing was read, that a connection which got QUIT is
+// never pooled, and that a second Close does not touch the pool.
+func TestFtpReadCloserClose(t *testing.T) {
+	err425 := &textproto.Error{Code: ftp.StatusCanNotOpenDataConnection, Msg: "Can't open data connection."}
+	newReader := func(t *testing.T, data []byte, closeErr error) (*ftpReadCloser, *Fs) {
+		server := startFakeFTPServer(t)
+		c, err := ftp.Dial(server.ln.Addr().String())
+		require.NoError(t, err)
+		f := &Fs{opt: Options{}}
+		return &ftpReadCloser{rc: &fakeDataReader{data: data, closeErr: closeErr}, c: c, f: f}, f
+	}
+
+	t.Run("425 before any read is masked and the connection is not pooled", func(t *testing.T) {
+		rc, f := newReader(t, nil, err425)
+		assert.NoError(t, rc.Close())
+		assert.Empty(t, f.pool)
+		assert.NoError(t, rc.Close())
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("425 after data was read stays an error", func(t *testing.T) {
+		rc, f := newReader(t, []byte("partial"), err425)
+		buf := make([]byte, 4)
+		n, err := rc.Read(buf)
+		require.NoError(t, err)
+		require.Equal(t, 4, n)
+		err = rc.Close()
+		require.Error(t, err)
+		assert.Equal(t, ftp.StatusCanNotOpenDataConnection, textprotoError(err).Code)
+		assert.Empty(t, f.pool)
+	})
+
+	t.Run("clean close pools the connection once", func(t *testing.T) {
+		rc, f := newReader(t, []byte("data"), nil)
+		_, err := io.ReadAll(rc)
+		require.NoError(t, err)
+		assert.NoError(t, rc.Close())
+		require.Len(t, f.pool, 1)
+		assert.NoError(t, rc.Close())
+		assert.Len(t, f.pool, 1)
+		_ = f.pool[0].Quit()
+	})
+}

backend/hasher/hasher_internal_test.go

@@ -122,6 +122,49 @@ func (f *Fs) testUpdateStoresHash(t *testing.T) {
 	_ = operations.Purge(ctx, f, dirName)
 }
 
+func (f *Fs) testSetModTimeKeepsHash(t *testing.T) {
+	ctx := context.Background()
+
+	const dirName = "setmodtime_hash_1"
+	const fileName = dirName + "/file_setmodtime_1"
+	const longTime = fs.ModTimeNotSupported
+	hashType := f.keepHashes.GetOne()
+
+	// upload a file to hasher via Put
+	src := putFile(ctx, t, f, fileName, "setmodtime content")
+	dst, err := f.NewObject(ctx, fileName)
+	require.NoError(t, err)
+	require.NotNil(t, dst)
+	_ = src
+
+	// verify hash was stored after Put
+	if f.opt.MaxAge <= 0 {
+		t.Skip("hash cache is disabled")
+	}
+	hash1, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+	require.NoError(t, err)
+	require.NotEmpty(t, hash1)
+
+	// change only the modification time, as operations does when it
+	// syncs the modtime of an otherwise unchanged destination file
+	newTime := fstest.Time("2002-03-04T05:06:07.499999999Z")
+	require.NoError(t, dst.SetModTime(ctx, newTime))
+
+	// the cached hash must survive a pure modtime change
+	hash2, err := f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+	assert.NoError(t, err)
+	assert.Equal(t, hash1, hash2, "hash should be kept after SetModTime")
+
+	// and it must be found via the fingerprint-checked lookup too
+	dst2, err := f.NewObject(ctx, fileName)
+	require.NoError(t, err)
+	hashVal, err := dst2.Hash(ctx, hashType)
+	assert.NoError(t, err)
+	assert.Equal(t, hash1, hashVal, "hash should be returned after SetModTime")
+
+	_ = operations.Purge(ctx, f, dirName)
+}
+
 // InternalTest dispatches all internal tests
 func (f *Fs) InternalTest(t *testing.T) {
 	if !kv.Supported() {
@@ -129,6 +172,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	}
 	t.Run("UploadFromCrypt", f.testUploadFromCrypt)
 	t.Run("UpdateStoresHash", f.testUpdateStoresHash)
+	t.Run("SetModTimeKeepsHash", f.testSetModTimeKeepsHash)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/hasher/kv.go

@@ -206,6 +206,34 @@ func (op *kvPut) Do(ctx context.Context, b kv.Bucket) (err error) {
 	return err
 }
 
+// kvUpdateFingerprint: rewrite the fingerprint of an existing record
+type kvUpdateFingerprint struct {
+	key   string
+	oldFp string
+	newFp string
+	age   time.Duration
+}
+
+func (op *kvUpdateFingerprint) Do(ctx context.Context, b kv.Bucket) error {
+	data := b.Get([]byte(op.key))
+	if len(data) == 0 {
+		return nil
+	}
+	var r hashRecord
+	if err := r.decode(op.key, data); err != nil {
+		return nil
+	}
+	if r.Fp != op.oldFp || time.Since(r.Created) > op.age {
+		return nil
+	}
+	r.Fp = op.newFp
+	data, err := r.encode(op.key)
+	if err != nil {
+		return err
+	}
+	return b.Put([]byte(op.key), data)
+}
+
 // kvDump: dump the database.
 // Note: long dump can cause concurrent operations to fail.
 type kvDump struct {

backend/hasher/object.go

@@ -56,6 +56,23 @@ func (o *Object) putHashes(ctx context.Context, rawHashes hashMap) error {
 	return o.f.putRawHashes(ctx, key, fp, hashes)
 }
 
+// updateFingerprint rewrites the fingerprint of the cached hash
+// record for remote from oldFp to newFp, keeping the hashes. It is
+// used when only the modification time of an object changes, so the
+// cached hashes remain valid under the new fingerprint.
+func (f *Fs) updateFingerprint(ctx context.Context, remote, oldFp, newFp string) error {
+	if oldFp == "" || newFp == "" || oldFp == newFp || f.opt.MaxAge <= 0 {
+		return nil
+	}
+	key := path.Join(f.Fs.Root(), remote)
+	return f.db.Do(true, &kvUpdateFingerprint{
+		key:   key,
+		oldFp: oldFp,
+		newFp: newFp,
+		age:   time.Duration(f.opt.MaxAge),
+	})
+}
+
 // set hashes for a path without any validation
 func (f *Fs) putRawHashes(ctx context.Context, key, fp string, hashes operations.HashSums) error {
 	return f.db.Do(true, &kvPut{
@@ -196,14 +213,27 @@ func (o *Object) Remove(ctx context.Context) error {
 }
 
 // SetModTime sets the modification time of the file.
-// Also prunes the cache entry when modtime changes so that
-// touching a file will trigger checksum recalculation even
-// on backends that don't provide modTime with fingerprint.
+//
+// When the fingerprint includes the modtime, the contents have not
+// changed, so the cached hashes are carried over to the fingerprint
+// for the new modtime. Otherwise the cache entry is pruned when the
+// modtime changes so that touching a file will trigger checksum
+// recalculation even on backends that don't provide modTime with
+// fingerprint.
 func (o *Object) SetModTime(ctx context.Context, mtime time.Time) error {
-	if mtime != o.Object.ModTime(ctx) {
-		_ = o.f.pruneHash(o.Remote())
+	if mtime == o.Object.ModTime(ctx) {
+		return o.Object.SetModTime(ctx, mtime)
 	}
-	return o.Object.SetModTime(ctx, mtime)
+	if !o.f.fpTime {
+		_ = o.f.pruneHash(o.Remote())
+		return o.Object.SetModTime(ctx, mtime)
+	}
+	oldFp := o.fingerprint(ctx)
+	if err := o.Object.SetModTime(ctx, mtime); err != nil {
+		return err
+	}
+	_ = o.f.updateFingerprint(ctx, o.Remote(), oldFp, o.fingerprint(ctx))
+	return nil
 }
 
 // Open opens the file for read.

backend/http/http.go

@@ -63,7 +63,8 @@ The headers are only sent to the host in the configured URL. If the
 server redirects to another host (including a subdomain or a different
 port) the headers are not sent to it, or to any further hop in that
 redirect chain. When headers are set, a redirect from https to http is
-refused as it would send them in cleartext.`,
+refused as it would send them in cleartext, even with
+--allow-https-downgrade.`,
 			Default:  fs.CommaSepList{},
 			Advanced: true,
 		}, {
@@ -100,6 +101,30 @@ that directory listings are much quicker, but rclone won't have the times or
 sizes of any files, and some files that don't exist may be in the listing.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_head_size",
+			Help: `Set the reported file size for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unknown file size.
+Set this option to a non-negative value to report that size for every file,
+including 0. Leave it unset (the default, shown as "off") to keep reporting
+an unknown size.
+
+The configured size is authoritative. If it does not match the actual file
+size, transfers may fail their size check and retry unless --ignore-size is
+used. It can also make multi-thread transfers request incorrect ranges and
+make mounted files unreadable past the reported size.`,
+			Default:  fs.SizeSuffix(-1),
+			Advanced: true,
+		}, {
+			Name: "no_head_time",
+			Help: `Set a default modification time for --http-no-head.
+
+When HEAD requests are disabled, rclone normally reports an unset modification
+time. Set this option to report the specified time for every file instead.
+Changing the configured time later makes every file appear modified to a sync.`,
+			Default:  fs.Time{},
+			Advanced: true,
 		}, {
 			Name:    "no_escape",
 			Help:    "Do not escape URL metacharacters in path names.",
@@ -145,11 +170,13 @@ var systemMetadataInfo = map[string]fs.MetadataHelp{
 
 // Options defines the configuration for this backend
 type Options struct {
-	Endpoint string          `config:"url"`
-	NoSlash  bool            `config:"no_slash"`
-	NoHead   bool            `config:"no_head"`
-	Headers  fs.CommaSepList `config:"headers"`
-	NoEscape bool            `config:"no_escape"`
+	Endpoint   string          `config:"url"`
+	NoSlash    bool            `config:"no_slash"`
+	NoHead     bool            `config:"no_head"`
+	NoHeadSize fs.SizeSuffix   `config:"no_head_size"`
+	NoHeadTime fs.Time         `config:"no_head_time"`
+	Headers    fs.CommaSepList `config:"headers"`
+	NoEscape   bool            `config:"no_escape"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -289,8 +316,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 	}
 
 	client := fshttp.NewClient(ctx)
-	// Without configured headers keep the default policy which
-	// public mirrors that redirect from https to http rely on.
+	// Without configured headers keep the default policy so
+	// --allow-https-downgrade can follow public mirrors that
+	// redirect from https to http.
 	if len(opt.Headers) > 0 {
 		client.CheckRedirect = checkRedirect(opt)
 	}
@@ -322,7 +350,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
 // the host specified in the config file.
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
-	opt := new(Options)
+	opt := &Options{
+		NoHeadSize: fs.SizeSuffix(-1),
+	}
 	err := configstruct.Set(m, opt)
 	if err != nil {
 		return nil, err
@@ -717,8 +747,11 @@ func (o *Object) url() string {
 // head sends a HEAD request to update info fields in the Object
 func (o *Object) head(ctx context.Context) error {
 	if o.fs.opt.NoHead {
-		o.size = -1
+		o.size = int64(o.fs.opt.NoHeadSize)
 		o.modTime = timeUnset
+		if o.fs.opt.NoHeadTime.IsSet() {
+			o.modTime = time.Time(o.fs.opt.NoHeadTime)
+		}
 		o.contentType = fs.MimeType(ctx, o)
 		return nil
 	}

backend/http/http_internal_test.go

@@ -314,6 +314,54 @@ func TestOpen(t *testing.T) {
 	}
 }
 
+func TestNoHeadMetadataDefaults(t *testing.T) {
+	tests := []struct {
+		name        string
+		size        string
+		modTime     string
+		wantSize    int64
+		wantModTime time.Time
+	}{
+		{
+			name:        "defaults",
+			wantSize:    -1,
+			wantModTime: timeUnset,
+		}, {
+			name:        "zero size",
+			size:        "0B",
+			wantSize:    0,
+			wantModTime: timeUnset,
+		}, {
+			name:        "configured metadata",
+			size:        "100B",
+			modTime:     "2024-08-05T07:36:07Z",
+			wantSize:    100,
+			wantModTime: time.Date(2024, 8, 5, 7, 36, 7, 0, time.UTC),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			m := prepareServer(t)
+			m.Set("no_head", "true")
+			if test.size != "" {
+				m.Set("no_head_size", test.size)
+			}
+			if test.modTime != "" {
+				m.Set("no_head_time", test.modTime)
+			}
+
+			f, err := NewFs(context.Background(), remoteName, "", m)
+			require.NoError(t, err)
+			o, err := f.NewObject(context.Background(), "four/under four.txt")
+			require.NoError(t, err)
+
+			assert.Equal(t, test.wantSize, o.Size())
+			assert.Equal(t, test.wantModTime, o.ModTime(context.Background()))
+		})
+	}
+}
+
 func TestMimeType(t *testing.T) {
 	f := prepare(t)
 
@@ -640,7 +688,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 
 	// open makes a remote pointing at ts with or without headers
 	// and opens file.txt on it
-	open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
+	open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {
 		configfile.Install()
 		m := configmap.Simple{
 			"type":    "http",
@@ -652,6 +700,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 		}
 		ctx, ci := fs.AddConfig(context.Background())
 		ci.InsecureSkipVerify = true
+		ci.AllowHTTPSDowngrade = allowDowngrade
 		f, err := NewFs(ctx, remoteName, "", m)
 		require.NoError(t, err)
 
@@ -663,13 +712,25 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
 
 	t.Run("WithHeaders", func(t *testing.T) {
 		// Headers could leak so the downgrade must be refused
-		_, err := open(t, true)
+		_, err := open(t, true, false)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {
+		// Headers could leak so the downgrade is refused even when allowed
+		_, err := open(t, true, true)
 		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
 	})
 
 	t.Run("WithoutHeaders", func(t *testing.T) {
-		// Nothing to leak so the downgrade is followed
-		fd, err := open(t, false)
+		// The transport refuses downgrades by default
+		_, err := open(t, false, false)
+		require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {
+		// Nothing to leak so the downgrade is followed when allowed
+		fd, err := open(t, false, true)
 		require.NoError(t, err)
 		data, err := io.ReadAll(fd)
 		require.NoError(t, err)

backend/huaweidrive/huaweidrive.go

@@ -50,6 +50,10 @@ const (
 	defaultChunkSize            = 8 * fs.Mebi
 )
 
+// jsonContentType is sent with JSON request bodies as the API doesn't
+// decode non-ASCII file names in them as UTF-8 unless told the charset.
+const jsonContentType = "application/json; charset=UTF-8"
+
 // OAuth2 configuration
 var oauthConfig = &oauthutil.Config{
 	Scopes: []string{
@@ -547,6 +551,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File
@@ -1015,6 +1020,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	copyReq := api.CopyFileRequest{
@@ -1199,9 +1205,10 @@ func (f *Fs) updateFile(ctx context.Context, fileID string, parameters url.Value
 	}
 	parameters.Set("fields", "*")
 	opts := rest.Opts{
-		Method:     "PATCH",
-		Path:       "/files/" + fileID,
-		Parameters: parameters,
+		Method:       "PATCH",
+		Path:         "/files/" + fileID,
+		Parameters:   parameters,
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, req, &info)
@@ -1671,6 +1678,7 @@ func (o *Object) uploadEmpty(ctx context.Context, leaf, directoryID string, src
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File
@@ -1846,6 +1854,7 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory
 			"fields":     []string{"*"},
 		},
 		ExtraHeaders: map[string]string{
+			"Content-Type":            jsonContentType,
 			"X-Upload-Content-Length": strconv.FormatInt(size, 10),
 		},
 		RootURL: o.fs.uploadURL,
@@ -2145,6 +2154,7 @@ func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
 		Parameters: url.Values{
 			"fields": []string{"*"},
 		},
+		ExtraHeaders: map[string]string{"Content-Type": jsonContentType},
 	}
 
 	var info *api.File

backend/iclouddrive/iclouddrive.go

@@ -694,6 +694,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
+		CaseInsensitive:         true, // names are matched with EqualFold, see findLeafItem
 		CanHaveEmptyDirectories: true,
 		PartialUploads:          false,
 	}).Fill(ctx, f)

backend/imagekit/client/upload.go

@@ -77,6 +77,8 @@ func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadPara
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading file if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	opts := rest.Opts{
 		Method:      "POST",

backend/jottacloud/jottacloud.go

@@ -86,8 +86,8 @@ func getServices() []service {
 		{"jottacloud", "Jottacloud", "id.jottacloud.com", "jottacloud", "desktop", []string{"openid", "jotta-default", "offline_access"}}, // Chose client id "desktop" here, will be identified as "Jottacloud for Desktop" in "My logged in devices", but could have used "jottacli" here as well.
 		{"elgiganten_dk", "Elgiganten Cloud (Denmark)", "cloud.elgiganten.dk", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
 		{"elgiganten_se", "Elgiganten Cloud (Sweden)", "cloud.elgiganten.se", "elgiganten", "desktop", []string{"openid", "jotta-default", "offline_access"}},
-		{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "jotta-default", "offline_access"}},
-		{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "jotta-default", "offline_access"}},
+		{"elkjop", "Elkjøp Cloud (Norway)", "cloud.elkjop.no", "elkjop", "desktop", []string{"openid", "offline_access"}},
+		{"elko", "ELKO Cloud (Iceland)", "cloud.elko.is", "elko", "desktop", []string{"openid", "offline_access"}},
 		{"gigantti", "Gigantti Cloud (Finland)", "cloud.gigantti.fi", "gigantti", "desktop", []string{"openid", "jotta-default", "offline_access"}},
 		{"letsgo", "Let's Go Cloud (Germany)", "letsgo.jotta.cloud", "letsgo", "desktop-win", []string{"openid", "offline_access"}},
 		{"mediamarkt", "MediaMarkt Cloud (Multiregional)", "mediamarkt.jottacloud.com", "mediamarkt", "desktop", []string{"openid", "jotta-default", "offline_access"}},
@@ -258,6 +258,7 @@ service, for the official service on https://www.jottacloud.com/web/secure.`)
 		}
 		m.Set("configVersion", fmt.Sprint(configVersion))
 		m.Set(configClientID, service.clientID)
+		m.Set(configClientSecret, "")
 		m.Set(configTokenURL, wellKnown.TokenEndpoint)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
 			OAuth2Config: &oauthutil.Config{
@@ -1500,9 +1501,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
 	if err == nil {
-		var createTime time.Time
+		createTime := srcObj.createTime
 		var createTimeMeta bool
-		var modTime time.Time
+		modTime := srcObj.modTime
 		var modTimeMeta bool
 		if meta != nil {
 			createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
@@ -1518,6 +1519,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
 			fs.Debugf(src, "Server-side copied to trashed destination, restoring")
 			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
+			// The restored file keeps the name it was trashed with, which may differ in case
+			if err == nil && info != nil && info.Name != f.opt.Enc.FromStandardName(path.Base(remote)) {
+				fs.Debugf(src, "Renaming restored %q to %q", info.Name, path.Base(remote))
+				info, err = f.copyOrMove(ctx, "mv", f.filePath(remote), remote)
+			}
 		} else if createTimeMeta || modTimeMeta {
 			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
 		}

backend/kdrive/api/types.go

@@ -0,0 +1,235 @@
+// Package api has type definitions for kDrive
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+)
+
+// Time represents unix timestamp date and time information for the
+// kDrive API
+type Time time.Time
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	if string(data) == "null" {
+		return nil
+	}
+
+	timestamp, err := strconv.ParseInt(string(data), 10, 64)
+	if err != nil {
+		return err
+	}
+	newT := time.Unix(timestamp, 0)
+	*t = Time(newT)
+	return nil
+}
+
+// ResultStatus return error details from kdrive when things go wrong
+//
+// If Status is "success" then everything is OK
+type ResultStatus struct {
+	Status      string `json:"result"`
+	ErrorDetail struct {
+		Result      string `json:"code"`
+		ErrorString string `json:"description"`
+		Context     struct {
+			Values []string `json:"values"`
+		} `json:"context"`
+		Errors []struct {
+			Result      string `json:"code"`
+			ErrorString string `json:"description"`
+		} `json:"errors"`
+	} `json:"error"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e *ResultStatus) Error() string {
+	var details string
+	for i := range e.ErrorDetail.Errors {
+		details += "|" + e.ErrorDetail.Errors[i].Result
+	}
+
+	return fmt.Sprintf("kDrive error: %s (%s: %s %s)", e.Status, e.ErrorDetail.Result, e.ErrorDetail.ErrorString, details)
+}
+
+// IsError returns true if there is an error
+func (e *ResultStatus) IsError() bool {
+	return e.Status != "success"
+}
+
+// Update returns err directly if it was != nil, otherwise it returns
+// an Error or nil if no error was detected
+func (e *ResultStatus) Update(err error) error {
+	if err != nil {
+		return err
+	}
+	if e.IsError() {
+		return e
+	}
+	return nil
+}
+
+// Check ResultStatus satisfies the error interface
+var _ error = (*ResultStatus)(nil)
+
+// Item describes a folder or a file as returned by Get Folder Items and others
+type Item struct {
+	ID             int    `json:"id"`
+	Name           string `json:"name"`
+	Type           string `json:"type"`
+	FullPath       string `json:"path"`
+	Status         string `json:"status"`
+	Hash           string `json:"hash"`
+	Size           int64  `json:"size"`
+	Visibility     string `json:"visibility"`
+	DriveID        int    `json:"drive_id"`
+	Depth          int    `json:"depth"`
+	CreatedBy      int    `json:"created_by"`
+	CreatedAt      Time   `json:"created_at"`
+	AddedAt        int    `json:"added_at"`
+	LastModifiedAt Time   `json:"last_modified_at"`
+	LastModifiedBy int    `json:"last_modified_by"`
+	RevisedAt      int    `json:"revised_at"`
+	UpdatedAt      int    `json:"updated_at"`
+	MimeType       string `json:"mime_type"`
+	ParentID       int    `json:"parent_id"`
+	Color          string `json:"color"`
+}
+
+// ItemResult is returned when an item is queried
+type ItemResult struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// SearchResult is returned when a list of items is requested
+type SearchResult struct {
+	ResultStatus
+	Data       []Item `json:"data"`
+	Cursor     string `json:"cursor"`
+	HasMore    bool   `json:"has_more"`
+	ResponseAt int    `json:"response_at"`
+}
+
+// ModTime returns the modification time of the item
+func (i *Item) ModTime() (t time.Time) {
+	t = time.Time(i.LastModifiedAt)
+	if t.IsZero() {
+		t = time.Time(i.CreatedAt)
+	}
+	return t
+}
+
+// CancelResource is a kdrive resource that can be cancelled after some action
+type CancelResource struct {
+	CancelID   string `json:"cancel_id"`
+	ValidUntil int    `json:"valid_until"`
+}
+
+// CancellableResponse is returned from kdrive when an action can be cancelled afterwards
+type CancellableResponse struct {
+	ResultStatus
+	Data CancelResource `json:"data"`
+}
+
+// CreateDirResult is returned from kdrive after a call to MkDir
+type CreateDirResult struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// FileCopyResponse is returned from kdrive after a call to Copy
+type FileCopyResponse struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// UploadFileResponse is returned from kdrive after a call to Upload
+type UploadFileResponse struct {
+	ResultStatus
+	Data Item `json:"data"`
+}
+
+// ChecksumFileResult is returned from kdrive after a call to Hash
+type ChecksumFileResult struct {
+	ResultStatus
+	Data struct {
+		Hash string `json:"hash"`
+	} `json:"data"`
+}
+
+// PubLinkResult is returned from kdrive after a call to public link API
+type PubLinkResult struct {
+	ResultStatus
+	Data struct {
+		URL          string `json:"url"`
+		FileID       int    `json:"file_id"`
+		Right        string `json:"right"`
+		ValidUntil   int    `json:"valid_until"`
+		CreatedBy    int    `json:"created_by"`
+		CreatedAt    int    `json:"created_at"`
+		UpdatedAt    int    `json:"updated_at"`
+		Capabilities struct {
+			CanEdit          bool `json:"can_edit"`
+			CanSeeStats      bool `json:"can_see_stats"`
+			CanSeeInfo       bool `json:"can_see_info"`
+			CanDownload      bool `json:"can_download"`
+			CanComment       bool `json:"can_comment"`
+			CanRequestAccess bool `json:"can_request_access"`
+		} `json:"capabilities"`
+		AccessBlocked bool `json:"access_blocked"`
+	} `json:"data"`
+}
+
+// PubLinkUpdateResult is returned from kdrive after a updating a public link
+type PubLinkUpdateResult struct {
+	ResultStatus
+	Data bool `json:"data"`
+}
+
+// QuotaInfo is return from kdrive after a call get drive info
+type QuotaInfo struct {
+	ResultStatus
+	Data struct {
+		Size     int64 `json:"size"`
+		UsedSize int64 `json:"used_size"`
+	} `json:"data"`
+}
+
+// SessionStartResponse is used to start a multichunk upload session
+type SessionStartResponse struct {
+	ResultStatus
+	Data struct {
+		Token     string `json:"token"`
+		UploadURL string `json:"upload_url"` // URL pour uploader les chunks
+	} `json:"data"`
+}
+
+// ChunkUploadResponse is returned by kdrive after a chunk upload
+type ChunkUploadResponse struct {
+	ResultStatus
+	Data struct {
+		Hash string `json:"hash,omitempty"` // Only present when requested with with:'hash'
+	} `json:"data"`
+}
+
+// SessionFinishResponse is used to end a multichunk upload session
+type SessionFinishResponse struct {
+	ResultStatus
+	Data struct {
+		Token   string `json:"token"`
+		File    Item   `json:"file"`
+		Result  bool   `json:"result"`
+		Message string `json:"message"`
+	} `json:"data"`
+}
+
+// SessionCancelResponse is used to cancel a multichunk upload session
+type SessionCancelResponse struct {
+	ResultStatus
+	Data bool `json:"data"`
+}

backend/kdrive/kdrive_internal_test.go

@@ -0,0 +1,551 @@
+// Unit tests for the kdrive backend
+package kdrive
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/kdrive/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestCheckChunkSize checks chunk_size validation against the API limits
+func TestCheckChunkSize(t *testing.T) {
+	for _, test := range []struct {
+		in      fs.SizeSuffix
+		wantErr string
+	}{
+		{in: 20 * fs.Mebi, wantErr: ""},                    // default
+		{in: 1000 * 1000, wantErr: ""},                     // API minimum
+		{in: 1000 * 1000 * 1000, wantErr: ""},              // API maximum
+		{in: 0, wantErr: "too small"},                      // panics without validation
+		{in: 999 * 1000, wantErr: "too small"},             // below API minimum
+		{in: 1000*1000*1000 + 1, wantErr: "too large"},     // above API maximum
+		{in: 2 * 1000 * 1000 * 1000, wantErr: "too large"}, // way above API maximum
+	} {
+		t.Run(test.in.String(), func(t *testing.T) {
+			err := checkChunkSize(test.in)
+			if test.wantErr == "" {
+				assert.NoError(t, err)
+			} else {
+				require.Error(t, err)
+				assert.Contains(t, err.Error(), test.wantErr)
+			}
+		})
+	}
+}
+
+// TestOpenChunkWriterClampsCallerChunkSize checks that a caller
+// supplied chunk size hint below the API minimum is raised to it
+func TestOpenChunkWriterClampsCallerChunkSize(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = fmt.Fprintln(w, `{"result":"success","data":{"token":"tok","upload_url":"http://unused"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0)}
+
+	info, _, err := f.OpenChunkWriter(context.Background(), "file.txt", src, &fs.ChunkOption{ChunkSize: 512 * 1024})
+	require.NoError(t, err)
+	assert.Equal(t, int64(minChunkSize), info.ChunkSize, "a chunk size hint below the API minimum must be raised to it")
+}
+
+// TestObjectSetMetaDataResetsStaleHash checks that setMetaData always
+// replaces the hash (a chunked upload finish response carries none)
+func TestObjectSetMetaDataResetsStaleHash(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		staleHash     string
+		staleNested   bool
+		newHash       string
+		wantHash      string
+		wantIsNested  bool
+		wantHashKnown bool
+	}{
+		{
+			name:          "empty hash clears stale plain hash",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "",
+			wantHash:      "",
+			wantIsNested:  false,
+			wantHashKnown: false,
+		},
+		{
+			name:          "empty hash clears stale nested hash",
+			staleHash:     "deadbeef",
+			staleNested:   true,
+			newHash:       "",
+			wantHash:      "",
+			wantIsNested:  false,
+			wantHashKnown: false,
+		},
+		{
+			name:          "plain hash is set",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "xxh3:12345678",
+			wantHash:      "12345678",
+			wantIsNested:  false,
+			wantHashKnown: true,
+		},
+		{
+			name:          "nested hash is set",
+			staleHash:     "deadbeef",
+			staleNested:   false,
+			newHash:       "N:xxh3:87654321",
+			wantHash:      "87654321",
+			wantIsNested:  true,
+			wantHashKnown: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o := &Object{
+				remote:      "file.txt",
+				hasMetaData: true,
+				xxh3:        test.staleHash,
+				nestedHash:  test.staleNested,
+			}
+			info := &api.Item{
+				ID:   42,
+				Name: "file.txt",
+				Type: "file",
+				Hash: test.newHash,
+			}
+
+			err := o.setMetaData(info)
+			require.NoError(t, err)
+			assert.Equal(t, test.wantHash, o.xxh3, "hash after setMetaData")
+			assert.Equal(t, test.wantIsNested, o.nestedHash, "nested flag after setMetaData")
+			assert.True(t, o.hasMetaData)
+
+			// Hash() must reflect the new state. Only checked when a
+			// hash is known: with no hash Hash() would query the API.
+			// Nested hashes are not comparable with plain XXH3, so
+			// Hash() returns "" for them.
+			if test.wantHashKnown {
+				got, gotErr := o.Hash(context.Background(), hash.XXH3)
+				require.NoError(t, gotErr)
+				if test.wantIsNested {
+					assert.Equal(t, "", got, "nested hash must not be returned as XXH3")
+				} else {
+					assert.Equal(t, test.wantHash, got)
+				}
+			}
+		})
+	}
+}
+
+// newTestFs creates an Fs wired to a test server with a plain client
+// (no OAuth) and a fast pacer
+func newTestFs(t *testing.T, endpoint string) *Fs {
+	t.Helper()
+	return &Fs{
+		name: "TestKdrive",
+		root: "",
+		opt:  Options{DriveID: "1234", Endpoint: endpoint},
+		srv:  rest.NewClient(&http.Client{}).SetRoot(endpoint),
+		pacer: fs.NewPacer(context.Background(),
+			pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(2*time.Millisecond), pacer.DecayConstant(2))),
+	}
+}
+
+// TestWriteChunkDetectsResultError checks that a 200 with result:error
+// is reported as an error
+func TestWriteChunkDetectsResultError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"chunk_error","description":"chunk upload failed"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	u := &uploadSession{
+		f:           f,
+		token:       "tok",
+		uploadURL:   ts.URL,
+		fileSize:    4,
+		chunkSize:   4,
+		chunkHashes: make(map[int64]string),
+	}
+
+	_, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
+	require.Error(t, err, "result:error with HTTP 200 must not be treated as success")
+	assert.Contains(t, err.Error(), "chunk upload failed")
+}
+
+// TestWriteChunkRetriesAndRewinds checks that transient failures are
+// retried with the chunk re-sent from the start of the reader
+func TestWriteChunkRetriesAndRewinds(t *testing.T) {
+	var mu sync.Mutex
+	var attempts int
+	var bodies []string
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		body, _ := io.ReadAll(r.Body)
+		mu.Lock()
+		attempts++
+		bodies = append(bodies, string(body))
+		current := attempts
+		mu.Unlock()
+
+		w.Header().Set("Content-Type", "application/json")
+		if current == 1 {
+			w.WriteHeader(http.StatusInternalServerError)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
+			return
+		}
+		_, _ = fmt.Fprintln(w, `{"result":"success","data":{"chunk_number":1,"status":"OK"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	u := &uploadSession{
+		f:           f,
+		token:       "tok",
+		uploadURL:   ts.URL,
+		fileSize:    4,
+		chunkSize:   4,
+		chunkHashes: make(map[int64]string),
+	}
+
+	written, err := u.WriteChunk(context.Background(), 0, bytes.NewReader([]byte("test")))
+	require.NoError(t, err)
+	assert.Equal(t, int64(4), written)
+
+	mu.Lock()
+	defer mu.Unlock()
+	require.Len(t, bodies, 2, "chunk upload must be retried once after a 5xx")
+	assert.Equal(t, "test", bodies[0], "first attempt sends the chunk")
+	assert.Equal(t, "test", bodies[1], "retry must re-send the chunk from the start of the reader")
+}
+
+// kdriveMockItem returns a JSON body for a single item result
+func kdriveMockItem(id int, name string) string {
+	return fmt.Sprintf(`{"result":"success","data":{"id":%d,"name":%q,"type":"file","path":%q,"last_modified_at":1700000000}}`, id, name, "/"+name)
+}
+
+// mockObjectInfo is a minimal fs.ObjectInfo for testing
+type mockObjectInfo struct {
+	remote  string
+	size    int64
+	modTime time.Time
+	hash    string
+}
+
+func (m *mockObjectInfo) Fs() fs.Info                         { return nil }
+func (m *mockObjectInfo) String() string                      { return m.remote }
+func (m *mockObjectInfo) Remote() string                      { return m.remote }
+func (m *mockObjectInfo) Size() int64                         { return m.size }
+func (m *mockObjectInfo) ModTime(_ context.Context) time.Time { return m.modTime }
+func (m *mockObjectInfo) Storable() bool                      { return true }
+func (m *mockObjectInfo) MimeType(_ context.Context) string   { return "application/octet-stream" }
+
+func (m *mockObjectInfo) Hash(_ context.Context, t hash.Type) (string, error) {
+	if m.hash == "" || t != hash.XXH3 {
+		return "", hash.ErrUnsupported
+	}
+	return m.hash, nil
+}
+
+// TestUpdateDirectNoRetryOnServerError checks that a failed single
+// part upload is not retried here: the input must not be buffered, the
+// higher layers retry the whole transfer
+func TestUpdateDirectNoRetryOnServerError(t *testing.T) {
+	var attempts int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&attempts, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		w.WriteHeader(http.StatusInternalServerError)
+		_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"internal_error","description":"transient"}}`)
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	o := &Object{fs: f, remote: "file.txt"}
+	src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: "12345678"}
+
+	err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
+	require.Error(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&attempts), "single part uploads must not be retried here")
+}
+
+// TestUpdateDirect checks the upload parameters and metadata of a
+// successful single part upload, and that total_chunk_hash is only
+// sent when the source hash is known
+func TestUpdateDirect(t *testing.T) {
+	for _, test := range []struct {
+		name          string
+		srcHash       string
+		wantTotalHash string
+	}{
+		{name: "hash known", srcHash: "12345678", wantTotalHash: "xxh3:12345678"},
+		{name: "hash unknown", srcHash: "", wantTotalHash: ""},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var gotTotalHash, gotBody string
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				w.Header().Set("Content-Type", "application/json")
+				gotTotalHash = r.URL.Query().Get("total_chunk_hash")
+				body, _ := io.ReadAll(r.Body)
+				gotBody = string(body)
+				_, _ = fmt.Fprintln(w, `{"result":"success","data":{"id":55,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:99998888","last_modified_at":1700000000}}`)
+			}))
+			defer ts.Close()
+
+			f := newTestFs(t, ts.URL)
+			o := &Object{fs: f, remote: "file.txt"}
+			src := &mockObjectInfo{remote: "file.txt", size: 4, modTime: time.Unix(1700000000, 0), hash: test.srcHash}
+
+			err := o.updateDirect(context.Background(), bytes.NewReader([]byte("test")), "1", "file.txt", src)
+			require.NoError(t, err)
+			assert.Equal(t, "test", gotBody, "the content must be streamed as-is")
+			assert.Equal(t, test.wantTotalHash, gotTotalHash, "total_chunk_hash")
+			assert.Equal(t, "55", o.id)
+			gotHash, hashErr := o.Hash(context.Background(), hash.XXH3)
+			require.NoError(t, hashErr)
+			assert.Equal(t, "99998888", gotHash)
+		})
+	}
+}
+
+// TestNewFsRequiresDriveID checks that NewFs fails fast when drive_id
+// is missing
+func TestNewFsRequiresDriveID(t *testing.T) {
+	m := configmap.Simple{"chunk_size": "20Mi"}
+	_, err := NewFs(context.Background(), "kdrive", "", m)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "drive_id")
+}
+
+// TestUploadSessionStringMasksToken checks that the upload session
+// String does not leak the session token in debug logs
+func TestUploadSessionStringMasksToken(t *testing.T) {
+	u := &uploadSession{token: "very-secret-uuid"}
+	assert.NotContains(t, u.String(), "very-secret-uuid")
+}
+
+// TestListNonRecursiveSingleRoundTrip checks that a non recursive
+// listing needs a single API request: without depth=unlimited the API
+// returns direct children only, so neither the parent item lookup nor
+// a child filter is needed
+func TestListNonRecursiveSingleRoundTrip(t *testing.T) {
+	var requests int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		atomic.AddInt32(&requests, 1)
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/files"):
+			// Direct children only, as returned without depth=unlimited
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":[`+
+				`{"id":10,"name":"sub","type":"dir","path":"/sub"},`+
+				`{"id":11,"name":"file.txt","type":"file","path":"/file.txt","hash":"xxh3:12345678","last_modified_at":1700000000}],"has_more":false}`)
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+
+	entries, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), atomic.LoadInt32(&requests), "non recursive listing must make a single API request")
+
+	names := make([]string, 0, len(entries))
+	for _, entry := range entries {
+		names = append(names, entry.Remote())
+	}
+	assert.ElementsMatch(t, []string{"sub", "file.txt"}, names)
+}
+
+// TestMoveConflictReturnsError checks that moving onto an occupied
+// destination reports the API conflict instead of resolving it: rclone
+// removes the destination itself before calling Move
+func TestMoveConflictReturnsError(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"conflict_error","description":"destination already exists"}}`)
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	_, err := f.Move(context.Background(), srcObj, "file2")
+	require.Error(t, err, "a move onto an occupied destination must fail")
+	assert.Contains(t, err.Error(), "conflict", "the API conflict must be reported")
+}
+
+// TestMoveDetectsForeignObjectAtDestination checks that Move does not
+// return a file re-created by another writer at the destination
+func TestMoveDetectsForeignObjectAtDestination(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			// The move API reports success...
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
+			// ...but the file at the destination is not the moved
+			// file: another writer re-created it
+			_, _ = fmt.Fprintln(w, kdriveMockItem(999, "file2"))
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	_, err := f.Move(context.Background(), srcObj, "file2")
+	require.Error(t, err, "Move must not report success when the destination is not the moved file")
+}
+
+// TestMoveSuccess checks the nominal move: the object returned is the
+// moved file (same ID as the source).
+func TestMoveSuccess(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = io.Copy(io.Discard, r.Body)
+		switch {
+		case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/files/111/move/1"):
+			_, _ = fmt.Fprintln(w, `{"result":"success","data":{"cancel_id":"abc","valid_until":1700000000}}`)
+		case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/files/1/name"):
+			// After the move, the moved file is at the destination
+			_, _ = fmt.Fprintln(w, kdriveMockItem(111, "file2"))
+		default:
+			t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path)
+			_, _ = fmt.Fprintln(w, `{"result":"error","error":{"code":"unexpected","description":"unexpected request"}}`)
+		}
+	}))
+	defer ts.Close()
+
+	f := newTestFs(t, ts.URL)
+	f.dirCache = dircache.New("", "1", f)
+	srcObj := &Object{fs: f, remote: "src.txt", id: "111", hasMetaData: true}
+
+	dstObj, err := f.Move(context.Background(), srcObj, "file2")
+	require.NoError(t, err)
+	moved, ok := dstObj.(*Object)
+	require.True(t, ok)
+	assert.Equal(t, "111", moved.id, "the returned object must be the moved file")
+}
+
+// TestFormatModTime checks that pre-1970 times format as negative
+// unix timestamps instead of wrapping through a uint64 cast
+func TestFormatModTime(t *testing.T) {
+	for _, test := range []struct {
+		in   time.Time
+		want string
+	}{
+		{in: time.Unix(1700000000, 0), want: "1700000000"},
+		{in: time.Unix(0, 0), want: "0"},
+		{in: time.Unix(-1577923200, 0), want: "-1577923200"}, // Sat Jan 2 1965
+	} {
+		t.Run(test.want, func(t *testing.T) {
+			assert.Equal(t, test.want, formatModTime(test.in))
+		})
+	}
+}
+
+// TestObjectSetMetaDataIsDir checks that directories are rejected
+func TestObjectSetMetaDataIsDir(t *testing.T) {
+	o := &Object{remote: "dir"}
+	info := &api.Item{ID: 42, Name: "dir", Type: "dir"}
+	err := o.setMetaData(info)
+	assert.Equal(t, fs.ErrorIsDir, err)
+}
+
+// TestShouldRetryLockError checks that transient locks held by server
+// side copies are retried while locks held by applications and other
+// errors are not
+func TestShouldRetryLockError(t *testing.T) {
+	for _, test := range []struct {
+		name      string
+		body      string
+		status    int
+		wantRetry bool
+	}{
+		{
+			name:      "lock held by internal copy",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy","context":{"values":["internal_copy"]}}}`,
+			status:    http.StatusForbidden,
+			wantRetry: true,
+		},
+		{
+			name:      "lock held by an application",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by onlyoffice","context":{"values":["onlyoffice"]}}}`,
+			status:    http.StatusForbidden,
+			wantRetry: false,
+		},
+		{
+			name:      "lock without context",
+			body:      `{"result":"error","error":{"code":"lock_error","description":"There is a lock on this file, locked by internal_copy"}}`,
+			status:    http.StatusForbidden,
+			wantRetry: false,
+		},
+		{
+			name:      "other api error",
+			body:      `{"result":"error","error":{"code":"object_not_found","description":"File not found"}}`,
+			status:    http.StatusNotFound,
+			wantRetry: false,
+		},
+		{
+			name:      "server error",
+			status:    http.StatusServiceUnavailable,
+			wantRetry: true,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			var apiErr error
+			if test.body != "" {
+				e := new(api.ResultStatus)
+				require.NoError(t, json.Unmarshal([]byte(test.body), e))
+				apiErr = e
+			}
+			resp := &http.Response{StatusCode: test.status, Header: http.Header{}}
+			got, gotErr := shouldRetry(context.Background(), resp, apiErr)
+			assert.Equal(t, test.wantRetry, got)
+			assert.Equal(t, apiErr, gotErr)
+		})
+	}
+}

backend/kdrive/kdrive_test.go

@@ -0,0 +1,25 @@
+// Test kdrive filesystem interface
+package kdrive
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestKdrive:",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+			MaxChunkSize: maxChunkSize,
+		},
+	})
+}
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setChunkSize(cs)
+}

backend/kdrive/khash/khash.go

@@ -0,0 +1,67 @@
+// Package khash provides the kDrive specific hash handling.
+//
+// kDrive uses XXH3-64 for file hashing. Files uploaded in chunks (by
+// rclone, or by the kDrive apps which each use their own chunk size)
+// get a "nested" hash: the XXH3 of the concatenated lowercase hex
+// strings of the individual chunk hashes. Such a nested hash can't be
+// compared with a plain XXH3 of the file contents.
+//
+// Hash formats returned by the kDrive API:
+//   - "xxh3:HASH" for a plain XXH3 of the file contents
+//   - "N:xxh3:HASH" for the nested hash of a chunked upload
+package khash
+
+import (
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/zeebo/xxh3"
+)
+
+// ParseHash parses a kDrive hash string from the API and returns the hex hash value
+// along with whether it is a nested (chunked) hash.
+func ParseHash(hashStr string) (hexHash string, isNested bool, err error) {
+	if hashStr == "" {
+		return "", false, nil
+	}
+
+	// Check for nested hash format: "N:xxh3:HASH"
+	if strings.HasPrefix(hashStr, "N:xxh3:") {
+		hashStr = strings.TrimPrefix(hashStr, "N:xxh3:")
+		isNested = true
+	} else {
+		hashStr = strings.TrimPrefix(hashStr, "xxh3:")
+	}
+
+	// Validate hex string
+	if _, decodeErr := hex.DecodeString(hashStr); decodeErr != nil {
+		return "", false, fmt.Errorf("invalid hash format: %w", decodeErr)
+	}
+
+	return hashStr, isNested, nil
+}
+
+// NestedChunkHash computes the kDrive nested hash from the hashes of the
+// file chunks, in order.
+//
+// kDrive stores the nested hash of a chunked file as the XXH3 of the
+// concatenated lowercase hex strings of the individual chunk hashes.
+// Each chunk hash may be passed with or without the "xxh3:" prefix.
+func NestedChunkHash(chunkHashes []string) (string, error) {
+	if len(chunkHashes) == 0 {
+		return "", errors.New("no chunk hashes to nest")
+	}
+	hexHashes := make([]string, len(chunkHashes))
+	for i, chunkHash := range chunkHashes {
+		hexHash, _, err := ParseHash(chunkHash)
+		if err != nil {
+			return "", fmt.Errorf("invalid chunk hash %q: %w", chunkHash, err)
+		}
+		hexHashes[i] = hexHash
+	}
+	hasher := xxh3.New()
+	_, _ = hasher.Write([]byte(strings.Join(hexHashes, "")))
+	return hex.EncodeToString(hasher.Sum(nil)), nil
+}

backend/kdrive/khash/khash_test.go

@@ -0,0 +1,150 @@
+package khash
+
+import "testing"
+
+func TestParseHash(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        string
+		expectedHash string
+		isNested     bool
+		expectError  bool
+	}{
+		{
+			name:         "simple hash with prefix",
+			input:        "xxh3:1db797b96febd334",
+			expectedHash: "1db797b96febd334",
+			isNested:     false,
+		},
+		{
+			name:         "nested hash",
+			input:        "N:xxh3:877abf3579f0a5c0",
+			expectedHash: "877abf3579f0a5c0",
+			isNested:     true,
+		},
+		{
+			name:         "plain hex",
+			input:        "1db797b96febd334",
+			expectedHash: "1db797b96febd334",
+			isNested:     false,
+		},
+		{
+			name:         "empty string",
+			input:        "",
+			expectedHash: "",
+			isNested:     false,
+		},
+		{
+			name:        "invalid hex",
+			input:       "xxh3:notahexvalue",
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			hash, isNested, err := ParseHash(tt.input)
+
+			if tt.expectError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if hash != tt.expectedHash {
+				t.Errorf("Hash: got %q, expected %q", hash, tt.expectedHash)
+			}
+			if isNested != tt.isNested {
+				t.Errorf("IsNested: got %t, expected %t", isNested, tt.isNested)
+			}
+		})
+	}
+}
+
+func TestNestedChunkHash(t *testing.T) {
+	// Golden values: the nested hash is the XXH3 of the concatenated
+	// lowercase hex strings of the chunk hashes, in order
+	chunk1 := "xxh3:d447b1ea40e6988b" // XXH3 of "hello world"
+	chunk2 := "2d06800538d394c2"      // XXH3 of ""
+
+	tests := []struct {
+		name        string
+		chunkHashes []string
+		expected    string
+		expectError bool
+	}{
+		{
+			name:        "two chunks in order",
+			chunkHashes: []string{chunk1, chunk2},
+			expected:    "6d51edccc9c05a8b",
+		},
+		{
+			name:        "order matters",
+			chunkHashes: []string{chunk2, chunk1},
+			expected:    "596c127c7bcc8820",
+		},
+		{
+			name:        "single chunk",
+			chunkHashes: []string{chunk1},
+			expected:    "b73081dff8fe443c",
+		},
+		{
+			name:        "mixed prefixes",
+			chunkHashes: []string{chunk1, "xxh3:" + chunk2, chunk1},
+			expected:    "ca0b13abcdcd1661",
+		},
+		{
+			name:        "no chunks",
+			chunkHashes: []string{},
+			expectError: true,
+		},
+		{
+			name:        "invalid chunk hash",
+			chunkHashes: []string{chunk1, "notahexvalue"},
+			expectError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			nested, err := NestedChunkHash(tt.chunkHashes)
+
+			if tt.expectError {
+				if err == nil {
+					t.Error("Expected error but got none")
+				}
+				return
+			}
+
+			if err != nil {
+				t.Fatalf("Unexpected error: %v", err)
+			}
+
+			if nested != tt.expected {
+				t.Errorf("NestedChunkHash: got %q, expected %q", nested, tt.expected)
+			}
+		})
+	}
+}
+
+func TestNestedChunkHashAcceptsPrefixedHashes(t *testing.T) {
+	plain := []string{"d447b1ea40e6988b", "2d06800538d394c2"}
+	prefixed := []string{"xxh3:d447b1ea40e6988b", "N:xxh3:2d06800538d394c2"}
+
+	gotPlain, err := NestedChunkHash(plain)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	gotPrefixed, err := NestedChunkHash(prefixed)
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if gotPlain != gotPrefixed {
+		t.Errorf("Prefixed hashes should give the same nested hash: %q != %q", gotPlain, gotPrefixed)
+	}
+}

backend/kdrive/multipart.go

@@ -0,0 +1,383 @@
+package kdrive
+
+import (
+	"context"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/backend/kdrive/api"
+	"github.com/rclone/rclone/backend/kdrive/khash"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/chunksize"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/zeebo/xxh3"
+	"golang.org/x/text/unicode/norm"
+)
+
+const (
+	// maxUploadParts is the maximum number of chunks the kDrive API accepts per file
+	maxUploadParts = 10000
+
+	// maxChunkSize is the largest chunk size the kDrive API accepts
+	maxChunkSize = 1 * 1000 * 1000 * 1000
+
+	// minChunkSize is the smallest chunk size the kDrive API accepts
+	minChunkSize = 1 * 1000 * 1000
+)
+
+// checkChunkSize checks the user supplied chunk_size against the
+// limits the kDrive API accepts
+func checkChunkSize(chunkSize fs.SizeSuffix) error {
+	if chunkSize < minChunkSize {
+		return fmt.Errorf("chunk_size %s is too small: the kDrive API requires chunks of at least 1 MB", chunkSize)
+	}
+	if chunkSize > maxChunkSize {
+		return fmt.Errorf("chunk_size %s is too large: the kDrive API accepts chunks of at most 1 GB (1000000000 bytes)", chunkSize)
+	}
+	return nil
+}
+
+// setChunkSize sets the chunk size, returning the old value
+func (f *Fs) setChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkChunkSize(cs)
+	if err == nil {
+		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	}
+	return
+}
+
+// formatModTime formats the modification time of an upload as the
+// unix timestamp string the kDrive API expects
+func formatModTime(modTime time.Time) string {
+	return fmt.Sprintf("%d", modTime.Unix())
+}
+
+// uploadSession implements fs.ChunkWriter for kdrive multipart uploads
+type uploadSession struct {
+	f         *Fs
+	token     string
+	uploadURL string
+	fileInfo  *api.Item
+	fileSize  int64 // total size of the file being uploaded
+	chunkSize int64 // size of the chunks of this session
+
+	mu          sync.Mutex
+	chunkHashes map[int64]string // chunk number -> hash of the chunk contents
+}
+
+// OpenChunkWriter returns the chunk size and a ChunkWriter
+//
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+	fileSize := src.Size()
+	if fileSize < 0 {
+		return info, nil, errors.New("kdrive can't upload files with unknown size")
+	}
+
+	dir, leaf := path.Split(remote)
+	dir = strings.TrimSuffix(dir, "/")
+
+	// Create parent directories if they don't exist
+	parentID, err := f.dirCache.FindDir(ctx, dir, true)
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to find parent directory: %w", err)
+	}
+
+	// Calculate the chunk size, honouring any fs.ChunkOption hint
+	chunkSize := f.opt.ChunkSize
+	for _, opt := range options {
+		if chunkOpt, ok := opt.(*fs.ChunkOption); ok && chunkOpt.ChunkSize > 0 {
+			chunkSize = fs.SizeSuffix(chunkOpt.ChunkSize)
+		}
+	}
+	chunkSize = chunksize.Calculator(src, fileSize, maxUploadParts, chunkSize)
+	if chunkSize < minChunkSize {
+		chunkSize = minChunkSize
+	}
+	if chunkSize > maxChunkSize {
+		chunkSize = maxChunkSize
+	}
+
+	// Calculate the total number of chunks
+	totalChunks := max((fileSize+int64(chunkSize)-1)/int64(chunkSize), 1)
+
+	sessionReq := struct {
+		Conflict       string `json:"conflict"`
+		DirectoryID    string `json:"directory_id"`
+		FileName       string `json:"file_name"`
+		LastModifiedAt string `json:"last_modified_at"`
+		TotalChunks    int64  `json:"total_chunks"`
+		TotalSize      int64  `json:"total_size"`
+	}{
+		Conflict:       "version",
+		DirectoryID:    parentID,
+		FileName:       f.opt.Enc.FromStandardName(norm.NFC.String(leaf)),
+		LastModifiedAt: formatModTime(src.ModTime(ctx)),
+		TotalChunks:    totalChunks,
+		TotalSize:      fileSize,
+	}
+
+	// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/start
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   fmt.Sprintf("/3/drive/%s/upload/session/start", f.opt.DriveID),
+	}
+	var sessionResp api.SessionStartResponse
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = f.srv.CallJSON(ctx, &opts, &sessionReq, &sessionResp)
+		callErr = sessionResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to start upload session: %w", err)
+	}
+
+	info = fs.ChunkWriterInfo{
+		ChunkSize:   int64(chunkSize),
+		Concurrency: f.opt.UploadConcurrency,
+	}
+
+	writer = &uploadSession{
+		f:           f,
+		token:       sessionResp.Data.Token,
+		uploadURL:   sessionResp.Data.UploadURL,
+		fileSize:    fileSize,
+		chunkSize:   int64(chunkSize),
+		chunkHashes: make(map[int64]string),
+	}
+
+	fs.Debugf(f, "open chunk writer: started upload session for %d chunks of %v", totalChunks, chunkSize)
+	return info, writer, nil
+}
+
+// WriteChunk uploads a single chunk
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
+func (u *uploadSession) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (bytesWritten int64, err error) {
+	if chunkNumber < 0 {
+		return -1, fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
+	}
+
+	// Calculate the size of this chunk from the session parameters so
+	// the chunk can be streamed to the API in a single pass
+	chunkLen := u.chunkSize
+	if remaining := u.fileSize - int64(chunkNumber)*u.chunkSize; remaining < chunkLen {
+		chunkLen = remaining
+	}
+	if chunkLen <= 0 {
+		return 0, nil
+	}
+
+	sourceChunkNumber := chunkNumber + 1 // kDrive API uses 1-based numbering
+
+	// The chunk_hash parameter is optional, so the chunk is streamed
+	// in a single pass, hashing the contents on the way with a
+	// TeeReader. The server returns the hash of the chunk it received,
+	// which is checked below. Retries seek back to the chunk start
+	// and hash with a fresh hasher.
+	var chunkHash string
+	err = u.f.pacer.Call(func() (bool, error) {
+		if _, seekErr := reader.Seek(0, io.SeekStart); seekErr != nil {
+			return false, fmt.Errorf("failed to rewind chunk %d: %w", sourceChunkNumber, seekErr)
+		}
+		chunkHasher := xxh3.New()
+
+		// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/chunk
+		chunkOpts := rest.Opts{
+			Method:  "POST",
+			RootURL: u.uploadURL,
+			Path:    fmt.Sprintf("/3/drive/%s/upload/session/%s/chunk", u.f.opt.DriveID, u.token),
+			Parameters: url.Values{
+				"chunk_number": {fmt.Sprintf("%d", sourceChunkNumber)},
+				"chunk_size":   {fmt.Sprintf("%d", chunkLen)},
+				"with":         {"hash"},
+			},
+			Body:          io.TeeReader(reader, chunkHasher),
+			ContentLength: &chunkLen,
+		}
+
+		var chunkResp api.ChunkUploadResponse
+		resp, callErr := u.f.srv.CallJSON(ctx, &chunkOpts, nil, &chunkResp)
+		callErr = chunkResp.ResultStatus.Update(callErr)
+		if callErr != nil {
+			return shouldRetry(ctx, resp, callErr)
+		}
+
+		chunkHash = hex.EncodeToString(chunkHasher.Sum(nil))
+
+		// The server hashed the chunk it received, so compare with what we
+		// sent to detect any corruption immediately
+		if chunkResp.Data.Hash != "" {
+			if serverHash, _, parseErr := khash.ParseHash(chunkResp.Data.Hash); parseErr == nil {
+				if !strings.EqualFold(chunkHash, serverHash) {
+					return false, fmt.Errorf("chunk %d hash mismatch: client=%s, server=%s", sourceChunkNumber, chunkHash, serverHash)
+				}
+			} else {
+				fs.Debugf(u, "Failed to parse server chunk hash %q: %v", chunkResp.Data.Hash, parseErr)
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		return -1, err
+	}
+
+	u.mu.Lock()
+	u.chunkHashes[int64(chunkNumber)] = chunkHash
+	u.mu.Unlock()
+
+	fs.Debugf(u, "uploaded chunk %d (size: %d, hash: %s)", sourceChunkNumber, chunkLen, chunkHash)
+	return chunkLen, nil
+}
+
+// Close finalizes the upload session, verifies the hash of the uploaded
+// file and returns the created file info
+// @see https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
+func (u *uploadSession) Close(ctx context.Context) error {
+	// https://developer.infomaniak.com/docs/api/post/3/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D/finish
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   fmt.Sprintf("/3/drive/%s/upload/session/%s/finish", u.f.opt.DriveID, u.token),
+	}
+	var resp *http.Response
+	var finishResp api.SessionFinishResponse
+	err := u.f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &finishResp)
+		callErr = finishResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to finish upload session: %w", err)
+	}
+
+	u.fileInfo = &finishResp.Data.File
+	fs.Debugf(u, "multipart upload completed: file id %d", finishResp.Data.File.ID)
+
+	return u.verifyHash(ctx)
+}
+
+// verifyHash checks the hash of the uploaded file against the hashes of
+// the chunks we uploaded, deleting the file if it is corrupted
+func (u *uploadSession) verifyHash(ctx context.Context) (err error) {
+	remoteHash := u.fileInfo.Hash
+	if remoteHash == "" {
+		// Hash might be missing in the finish response, try to fetch it
+		obj := &Object{
+			fs: u.f,
+			id: strconv.Itoa(u.fileInfo.ID),
+		}
+		var hashErr error
+		if remoteHash, hashErr = obj.retrieveHash(ctx); hashErr != nil {
+			// skip verification
+			fs.Debugf(u, "Failed to retrieve hash for verification: %v", hashErr)
+			return nil
+		}
+	}
+	parsedRemoteHash, isNested, err := khash.ParseHash(remoteHash)
+	if err != nil {
+		fs.Debugf(u, "Failed to parse server hash %q: %v", remoteHash, err)
+		return nil
+	}
+
+	u.mu.Lock()
+	defer u.mu.Unlock()
+
+	// Rebuild the hash the server should have stored from the chunk
+	// hashes we uploaded, in order
+	chunkCount := int64(len(u.chunkHashes))
+	if chunkCount == 0 {
+		return nil
+	}
+	chunkHashes := make([]string, 0, chunkCount)
+	for i := int64(0); i < chunkCount; i++ {
+		chunkHash, ok := u.chunkHashes[i]
+		if !ok {
+			fs.Debugf(u, "Chunk %d hash missing, skipping hash verification", i)
+			return nil
+		}
+		chunkHashes = append(chunkHashes, chunkHash)
+	}
+
+	var localHash string
+	if isNested {
+		if localHash, err = khash.NestedChunkHash(chunkHashes); err != nil {
+			fs.Debugf(u, "Failed to compute nested hash: %v", err)
+			return nil
+		}
+	} else if chunkCount == 1 {
+		localHash, _, _ = khash.ParseHash(chunkHashes[0])
+	} else {
+		// The server returned a simple hash for a multi chunk upload,
+		// nothing to compare it against
+		fs.Debugf(u, "Server returned a simple hash for a %d chunk upload, skipping hash verification", chunkCount)
+		return nil
+	}
+
+	if !strings.EqualFold(localHash, parsedRemoteHash) {
+		err = fmt.Errorf(
+			"multipart upload hash mismatch: local=%s, remote=%s",
+			localHash, parsedRemoteHash,
+		)
+
+		// Remove the corrupted file
+		obj := &Object{
+			fs: u.f,
+			id: strconv.Itoa(u.fileInfo.ID),
+		}
+		if delErr := obj.Remove(ctx); delErr != nil {
+			fs.Errorf(nil, "Failed to remove corrupted file after hash mismatch: %v", delErr)
+		} else {
+			fs.Debugf(nil, "Removed corrupted file after hash mismatch")
+		}
+
+		return err
+	}
+	fs.Debugf(u, "Multipart upload hash verified: %s", localHash)
+
+	return nil
+}
+
+// Abort the upload session
+// @see  https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
+func (u *uploadSession) Abort(ctx context.Context) error {
+	// https://developer.infomaniak.com/docs/api/delete/2/drive/%7Bdrive_id%7D/upload/session/%7Bsession_token%7D
+	opts := rest.Opts{
+		Method: "DELETE",
+		Path:   fmt.Sprintf("/2/drive/%s/upload/session/%s", u.f.opt.DriveID, u.token),
+	}
+	var resp *http.Response
+	var cancelResp api.SessionCancelResponse
+	err := u.f.pacer.Call(func() (bool, error) {
+		var callErr error
+		resp, callErr = u.f.srv.CallJSON(ctx, &opts, nil, &cancelResp)
+		callErr = cancelResp.ResultStatus.Update(callErr)
+		return shouldRetry(ctx, resp, callErr)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to cancel upload session: %w", err)
+	}
+
+	fs.Debugf(u, "upload session cancelled")
+	return nil
+}
+
+// String implements fmt.Stringer
+func (u *uploadSession) String() string {
+	return "kdrive upload session"
+}
+
+// Verify interface compliance
+var _ fs.ChunkWriter = (*uploadSession)(nil)

backend/local/clone_darwin.go

@@ -6,11 +6,13 @@ package local
 import (
 	"context"
 	"fmt"
+	"os"
 	"path/filepath"
 	"runtime"
 
 	"github.com/go-darwin/apfs"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/random"
 )
 
 // Copy src to this remote using server-side copy operations.
@@ -63,7 +65,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = Clone(srcPath, dstPath)
+	err = cloneOver(srcPath, dstPath)
 	if err != nil {
 		return nil, err
 	}
@@ -79,6 +81,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
+// cloneOver clones src to dst, replacing dst if it already exists.
+//
+// COPYFILE_CLONE implies COPYFILE_EXCL so Clone fails if dst exists.
+// Cloning to a temporary name in the same directory and renaming it
+// over dst avoids this and leaves dst untouched if the clone fails.
+func cloneOver(src, dst string) error {
+	if _, err := os.Lstat(dst); os.IsNotExist(err) {
+		return Clone(src, dst)
+	}
+	tmp := filepath.Join(filepath.Dir(dst), "."+random.String(16)+".rclone-clone")
+	err := Clone(src, tmp)
+	if err == nil {
+		err = os.Rename(tmp, dst)
+	}
+	if err != nil {
+		_ = os.Remove(tmp)
+		return fmt.Errorf("failed to clone %q to %q: %w", src, dst, err)
+	}
+	return nil
+}
+
 // Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)
 // note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical
 // https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html

backend/local/local.go

@@ -87,8 +87,10 @@ Metadata is supported on files and directories.
 When restoring metadata with ` + "`--metadata`" + ` rclone applies the
 "mode", "uid" and "gid" from the source. These come from the source
 remote which may not be trusted, so restoring metadata as root from an
-untrusted source can change file ownership and is not recommended. The
-setuid, setgid and sticky bits are not restored by default - see the
+untrusted source can change file ownership and is not recommended. A
+user other than root keeps its own uid and sets the gid only to a group
+it is a member of, as rsync does. The setuid, setgid and sticky bits are
+not restored by default - see the
 ` + "`--local-metadata-restore-special-bits`" + ` flag.
 `,
 		},
@@ -252,6 +254,13 @@ deduplicated by default (i.e. having two identical files does not consume more
 storage than having just one.)  However, for use cases where data redundancy is
 preferable, --local-no-clone can be used to disable cloning and force "deep" copies.
 
+Cloning also preserves macOS-specific metadata such as extended attributes,
+which a normal copy does not, so this metadata is lost when cloning is disabled.
+
+If the destination file already exists, cloning replaces it with a new file,
+even with --inplace, so hard links to it are broken. Use --local-no-clone to
+update existing files in place.
+
 Currently, cloning is only supported when using APFS on macOS (support for other
 platforms may be added in the future.)`,
 				Default:  false,
@@ -951,13 +960,13 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
 	if err != nil {
 		return nil, err
 	}
-	fi, err := f.lstat(localPath)
+	fi, err := f.lstatRoot(localPath)
 	if errors.Is(err, os.ErrNotExist) {
 		err := f.Mkdir(ctx, dir)
 		if err != nil {
 			return nil, fmt.Errorf("mkdir metadata: failed make directory: %w", err)
 		}
-		fi, err = f.lstat(localPath)
+		fi, err = f.lstatRoot(localPath)
 		if err != nil {
 			return nil, fmt.Errorf("mkdir metadata: failed to read info: %w", err)
 		}
@@ -989,20 +998,34 @@ func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata
 // Rmdir removes the directory
 //
 // If it isn't empty it will return an error
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	localPath, err := f.localPath(dir)
 	if err != nil {
 		return err
 	}
-	if fi, err := os.Stat(localPath); err != nil {
+	// When translating symlinks go through os.Root so a symlink planted at
+	// any parent can't redirect the removal out of the destination, and
+	// don't follow the leaf as a symlink is a file in this mode. The root
+	// itself is the trusted destination and is removed directly.
+	stat, removeFn, chmod := os.Stat, os.Remove, os.Chmod
+	if f.opt.TranslateSymlinks && localPath != f.root {
+		var root *os.Root
+		root, localPath, err = f.osRoot(localPath)
+		if err != nil {
+			return err
+		}
+		defer fs.CheckClose(root, &err)
+		stat, removeFn, chmod = root.Lstat, root.Remove, root.Chmod
+	}
+	if fi, err := stat(localPath); err != nil {
 		return err
 	} else if !fi.IsDir() {
 		return fs.ErrorIsFile
 	}
-	err = os.Remove(localPath)
+	err = removeFn(localPath)
 	if runtime.GOOS == "windows" && errors.Is(err, iofs.ErrPermission) { // https://github.com/golang/go/issues/26295
-		if os.Chmod(localPath, 0o600) == nil {
-			err = os.Remove(localPath)
+		if chmod(localPath, 0o600) == nil {
+			err = removeFn(localPath)
 		}
 	}
 	return err
@@ -1179,9 +1202,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	// Create parent of destination
+	// Create parent of destination. The root itself is the trusted
+	// destination so its parent, which lies outside it, is created directly.
 	dstParentPath := filepath.Dir(dstPath)
-	err = file.MkdirAll(dstParentPath, 0777)
+	if dstPath == f.root {
+		err = file.MkdirAll(dstParentPath, 0777)
+	} else {
+		err = f.mkdirAll(dstParentPath)
+	}
 	if err != nil {
 		return err
 	}
@@ -1598,10 +1626,26 @@ func (f *Fs) mkdirAll(localPath string) (err error) {
 	return root.MkdirAll(rel, 0777)
 }
 
-// openFile opens localPath for writing. When translating symlinks it goes
-// through os.Root so a symlink planted at the path, or at any parent, is never
-// followed out of the destination. The returned file is independent of the
-// root, which is closed before returning.
+// lstatRoot returns the info for localPath without following a symlink at
+// the leaf. When translating symlinks it goes through os.Root so a symlink
+// planted at any parent can't make a path outside the destination look like
+// it exists, which would let its metadata be written there.
+func (f *Fs) lstatRoot(localPath string) (fi os.FileInfo, err error) {
+	if !f.opt.TranslateSymlinks {
+		return f.lstat(localPath)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return nil, err
+	}
+	defer fs.CheckClose(root, &err)
+	return root.Lstat(rel)
+}
+
+// openFile opens localPath with the given flags and perm. When translating
+// symlinks it goes through os.Root so a symlink planted at the path, or at any
+// parent, is never followed out of the destination. The returned file is
+// independent of the root, which is closed before returning.
 func (f *Fs) openFile(localPath string, flags int, perm os.FileMode) (fi *os.File, err error) {
 	if !f.opt.TranslateSymlinks {
 		return file.OpenFile(localPath, flags, perm)
@@ -1670,6 +1714,21 @@ func (f *Fs) chtimes(localPath string, atime, mtime time.Time) (err error) {
 	return root.Chtimes(rel, atime, mtime)
 }
 
+// remove removes the file, symlink or empty directory at localPath. When
+// translating symlinks it goes through os.Root so a symlink planted at any
+// parent can't redirect the removal out of the destination.
+func (f *Fs) remove(localPath string) (err error) {
+	if !f.opt.TranslateSymlinks {
+		return remove(localPath, os.Remove)
+	}
+	root, rel, err := f.osRoot(localPath)
+	if err != nil {
+		return err
+	}
+	defer fs.CheckClose(root, &err)
+	return remove(rel, root.Remove)
+}
+
 // mkdirAll makes all the directories needed to store the object
 func (o *Object) mkdirAll() error {
 	return o.fs.mkdirAll(filepath.Dir(o.path))
@@ -1794,7 +1853,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
-		if removeErr := os.Remove(o.path); removeErr != nil {
+		if removeErr := o.fs.remove(o.path); removeErr != nil {
 			fs.Errorf(o, "Failed to remove partially written file: %v", removeErr)
 		}
 		return err
@@ -1942,7 +2001,7 @@ func (o *Object) lstat() error {
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	o.clearHashCache()
-	return remove(o.path)
+	return o.fs.remove(o.path)
 }
 
 // Metadata returns metadata for an object

backend/local/local_internal_test.go

@@ -99,6 +99,31 @@ func TestVerifyCopy(t *testing.T) {
 	assert.Error(t, err)
 }
 
+// Test server-side copy over an existing file
+func TestCopyOverExisting(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	doCopy := r.Flocal.Features().Copy
+	if doCopy == nil {
+		t.Skip("local backend has no server-side copy on this platform")
+	}
+	t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	t2 := fstest.Time("2002-02-03T04:05:06.499999999Z")
+	file1 := r.WriteFile("a.txt", "hello1", t1)
+	file2 := r.WriteFile("b.txt", "HELLO2 longer", t2)
+
+	src, err := r.Flocal.NewObject(ctx, file1.Path)
+	require.NoError(t, err)
+	dst, err := doCopy(ctx, src, file2.Path)
+	require.NoError(t, err)
+	assert.Equal(t, file2.Path, dst.Remote())
+
+	file2.Size = file1.Size
+	file2.Hashes = file1.Hashes
+	file2.ModTime = file1.ModTime
+	r.CheckLocalItems(t, file1, file2)
+}
+
 // skipIfNoSymlinks skips the test if this process can't create symlinks.
 //
 // Windows grants the privilege only to an elevated process or one running with
@@ -535,6 +560,109 @@ func TestDirBTimeThroughPlantedSymlinkBlocked(t *testing.T) {
 	require.Equal(t, before, readBTime(), "btime escaped through planted symlink to %q", evilDir)
 }
 
+// TestMkdirMetadataThroughPlantedSymlinkBlocked checks MkdirMetadata can't
+// be fooled into treating a directory outside the root, reached through a
+// planted symlink, as the directory to apply metadata to.
+func TestMkdirMetadataThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	require.NoError(t, os.Mkdir(filepath.Join(evilDir, "sub"), 0700))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: an existing in-tree directory is found as normal.
+	require.NoError(t, f.Mkdir(ctx, "sub"))
+	_, err := f.MkdirMetadata(ctx, "sub", nil)
+	require.NoError(t, err)
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// "pwn/sub" exists outside the root but must not be found through the
+	// planted symlink, and must not be created there either.
+	_, err = f.MkdirMetadata(ctx, "pwn/sub", nil)
+	require.Error(t, err, "finding a directory through a planted symlink should be refused")
+	_, err = f.MkdirMetadata(ctx, "pwn/new", nil)
+	require.Error(t, err, "making a directory through a planted symlink should be refused")
+	_, err = os.Lstat(filepath.Join(evilDir, "new"))
+	require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evilDir)
+}
+
+// TestRemoveThroughPlantedSymlinkBlocked checks Remove and Rmdir can't
+// delete anything outside the root through a planted symlink.
+func TestRemoveThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+	evilFile := filepath.Join(evil, "secret.txt")
+	require.NoError(t, os.WriteFile(evilFile, []byte("keep me"), 0600))
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: in-tree removals work as normal.
+	require.NoError(t, putFile(ctx, f, "file.txt", "data"))
+	o, err := f.NewObject(ctx, "file.txt")
+	require.NoError(t, err)
+	require.NoError(t, o.Remove(ctx))
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	require.NoError(t, f.Rmdir(ctx, "dir"))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evil))
+
+	// Removing the outside file and directory named through the planted
+	// symlink must be refused.
+	o, err = f.NewObject(ctx, "pwn/secret.txt")
+	require.NoError(t, err)
+	err = o.Remove(ctx)
+	require.Error(t, err, "removing through a planted symlink should be refused")
+	_, err = os.Lstat(evilFile)
+	require.NoError(t, err, "a file was removed outside the destination at %q", evilFile)
+
+	err = f.Rmdir(ctx, "pwn/secret.d")
+	require.Error(t, err, "removing a directory through a planted symlink should be refused")
+	_, err = os.Lstat(evilDir)
+	require.NoError(t, err, "a directory was removed outside the destination at %q", evilDir)
+}
+
+// TestDirMoveThroughPlantedSymlinkBlocked checks DirMove can't move a
+// directory out of the root through a planted symlink.
+func TestDirMoveThroughPlantedSymlinkBlocked(t *testing.T) {
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	evil := t.TempDir()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	// Control: an in-tree directory move works as normal.
+	require.NoError(t, f.Mkdir(ctx, "dir"))
+	require.NoError(t, f.DirMove(ctx, f, "dir", "sub/moved"))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evil))
+
+	err := f.DirMove(ctx, f, "sub/moved", "pwn/moved")
+	require.Error(t, err, "moving a directory through a planted symlink should be refused")
+	_, err = os.Lstat(filepath.Join(evil, "moved"))
+	require.True(t, os.IsNotExist(err), "a directory escaped the destination into %q", evil)
+	_, err = os.Lstat(filepath.Join(f.root, "sub", "moved"))
+	require.NoError(t, err, "the directory should still be in the destination")
+}
+
 // TestEncodingEscapeBlocked checks that a name from a malicious source can't
 // be decoded into path syntax which writes outside the destination.
 func TestEncodingEscapeBlocked(t *testing.T) {

backend/local/metadata.go

@@ -5,7 +5,9 @@ import (
 	"math"
 	"os"
 	"runtime"
+	"slices"
 	"strconv"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -121,13 +123,15 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	uid, hasUID := o.parseMetadataInt(m, "uid", 10)
 	gid, hasGID := o.parseMetadataInt(m, "gid", 10)
 	if hasUID {
-		// FIXME should read UID and GID of current user and only attempt to set it if different
 		if !hasGID {
 			gid = uid
 		}
 		if runtime.GOOS == "windows" || runtime.GOOS == "plan9" {
 			fs.Debugf(o, "Ignoring request to set ownership %o.%o on this OS", gid, uid)
+		} else if setUID, setGID := ownerToSet(uid, gid); setUID < 0 && setGID < 0 {
+			fs.Debugf(o, "Not setting ownership %d.%d: only root can", uid, gid)
 		} else {
+			uid, gid = setUID, setGID
 			if o.translatedLink {
 				err = os.Lchown(o.path, uid, gid)
 			} else {
@@ -169,3 +173,27 @@ func (o *Object) writeMetadataToFile(m fs.Metadata) (outErr error) {
 	// FIXME not parsing rdev yet
 	return outErr
 }
+
+// processGroups are the supplementary groups of this process
+var processGroups = sync.OnceValue(func() []int {
+	groups, _ := os.Getgroups()
+	return groups
+})
+
+// ownerToSet returns the uid and gid to pass to chown for the wanted
+// ownership: unchanged for root, else -1 (leave it as it is) for a uid
+// other than our own and for a group we are not a member of, which
+// only root may set.
+func ownerToSet(uid, gid int) (setUID, setGID int) {
+	euid := os.Geteuid()
+	if euid <= 0 {
+		return uid, gid
+	}
+	if uid != euid {
+		uid = -1
+	}
+	if gid != os.Getegid() && !slices.Contains(processGroups(), gid) {
+		gid = -1
+	}
+	return uid, gid
+}

backend/local/metadata_owner_unix_test.go

@@ -0,0 +1,50 @@
+//go:build !windows && !plan9 && !js
+
+package local
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"slices"
+	"syscall"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetadataOwnershipAsNonRoot(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("needs a user other than root")
+	}
+	uid, gid := os.Geteuid(), os.Getegid()
+	other := uid + 1
+	notMember := gid + 1
+	for slices.Contains(processGroups(), notMember) {
+		notMember++
+	}
+
+	// only what a non-root user may set: its own uid, its groups
+	setUID, setGID := ownerToSet(other, notMember)
+	assert.Equal(t, [2]int{-1, -1}, [2]int{setUID, setGID})
+	setUID, setGID = ownerToSet(other, gid)
+	assert.Equal(t, [2]int{-1, gid}, [2]int{setUID, setGID})
+	setUID, setGID = ownerToSet(uid, notMember)
+	assert.Equal(t, [2]int{uid, -1}, [2]int{setUID, setGID})
+
+	// a file of another user (e.g. root:root) is copied with our ownership
+	// instead of failing with "operation not permitted"
+	r := fstest.NewRun(t)
+	r.WriteFile("owned.txt", "payload", time.Now())
+	f := r.Flocal.(*Fs)
+	obj, err := f.NewObject(context.Background(), "owned.txt")
+	require.NoError(t, err)
+	require.NoError(t, obj.(*Object).writeMetadataToFile(fs.Metadata{"uid": "0", "gid": "0"}))
+	fi, err := os.Stat(filepath.Join(f.root, "owned.txt"))
+	require.NoError(t, err)
+	assert.Equal(t, uint32(uid), fi.Sys().(*syscall.Stat_t).Uid)
+}

backend/local/remove_other.go

@@ -2,9 +2,7 @@
 
 package local
 
-import "os"
-
-// Removes name, retrying on a sharing violation
-func remove(name string) error {
-	return os.Remove(name)
+// remove removes name with removeFn, retrying on a sharing violation
+func remove(name string, removeFn func(string) error) error {
+	return removeFn(name)
 }

backend/local/remove_test.go

@@ -38,7 +38,7 @@ func TestRemove(t *testing.T) {
 		require.NoError(t, fd.Close())
 	})
 	// delete the open file
-	err = remove(name)
+	err = remove(name, os.Remove)
 	require.NoError(t, err)
 	// check it no longer exists
 	assert.False(t, exists())

backend/local/remove_windows.go

@@ -10,12 +10,12 @@ import (
 	"golang.org/x/sys/windows"
 )
 
-// Removes name, retrying on a sharing violation
-func remove(name string) (err error) {
+// remove removes name with removeFn, retrying on a sharing violation
+func remove(name string, removeFn func(string) error) (err error) {
 	const maxTries = 10
 	var sleepTime = 1 * time.Millisecond
 	for i := range maxTries {
-		err = os.Remove(name)
+		err = removeFn(name)
 		if err == nil {
 			break
 		}

backend/local/xattr.go

@@ -4,6 +4,7 @@ package local
 
 import (
 	"fmt"
+	"os"
 	"strings"
 	"syscall"
 
@@ -95,6 +96,11 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 	if !xattrSupported || o.fs.xattrSupported.Load() == 0 {
 		return nil
 	}
+	// When translating symlinks (-l/--links) write the xattrs through a
+	// descriptor opened under os.Root so a symlink planted in the destination
+	// can't redirect the write outside it (CWE-59). A translated link is
+	// itself the leaf, so it keeps the non-following path based write.
+	var fd *os.File
 	for k, value := range metadata {
 		k = strings.ToLower(k)
 		if _, found := systemMetadataInfo[k]; found {
@@ -102,9 +108,27 @@ func (o *Object) setXattr(metadata fs.Metadata) (err error) {
 		}
 		k = xattrPrefix + k
 		v := []byte(value)
-		if o.fs.opt.FollowSymlinks {
+		switch {
+		case o.fs.opt.TranslateSymlinks && !o.translatedLink:
+			if fd == nil {
+				fd, err = o.fs.openFile(o.path, os.O_RDONLY, 0)
+				if os.IsPermission(err) {
+					// A write only file can't be opened for reading but its
+					// owner may still set xattrs on it, so try opening it for
+					// writing instead (which can't succeed for a directory).
+					if wfd, werr := o.fs.openFile(o.path, os.O_WRONLY, 0); werr == nil {
+						fd, err = wfd, nil
+					}
+				}
+				if err != nil {
+					return fmt.Errorf("failed to open for xattr: %w", err)
+				}
+				defer fs.CheckClose(fd, &err)
+			}
+			err = xattr.FSet(fd, k, v)
+		case o.fs.opt.FollowSymlinks:
 			err = xattr.Set(o.path, k, v)
-		} else {
+		default:
 			err = xattr.LSet(o.path, k, v)
 		}
 		if err != nil {

backend/local/xattr_internal_test.go

@@ -0,0 +1,96 @@
+//go:build !openbsd && !plan9
+
+package local
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/pkg/xattr"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestXattrThroughPlantedSymlinkBlocked checks user metadata (xattrs)
+// can't be written outside the root through a planted symlink with
+// --links, while in-tree writes still work.
+func TestXattrThroughPlantedSymlinkBlocked(t *testing.T) {
+	if !xattrSupported {
+		t.Skip("xattrs not supported on this OS")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	metadata := fs.Metadata{
+		"pwned": "attacker-controlled-value",
+	}
+
+	// Control: an in-tree directory gets the xattr as normal.
+	_, err := f.MkdirMetadata(ctx, "sub", metadata)
+	require.NoError(t, err)
+	if f.xattrSupported.Load() == 0 {
+		t.Skip("xattrs not supported on this filesystem")
+	}
+	v, err := xattr.LGet(filepath.Join(f.root, "sub"), "user.pwned")
+	require.NoError(t, err)
+	assert.Equal(t, "attacker-controlled-value", string(v))
+
+	// A directory outside the destination, with a pre-existing child,
+	// whose xattrs the attacker targets.
+	evil := t.TempDir()
+	evilDir := filepath.Join(evil, "secret.d")
+	require.NoError(t, os.Mkdir(evilDir, 0700))
+	evilSub := filepath.Join(evilDir, "sub")
+	require.NoError(t, os.Mkdir(evilSub, 0700))
+
+	// The planted symlink dst/pwn -> outside, faithfully reproduced by --links.
+	require.NoError(t, putLink(ctx, f, "pwn", evilDir))
+
+	// The source now presents "pwn/sub" as a directory with attacker-chosen
+	// user metadata. Applying it must not reach through the planted symlink.
+	_, err = f.MkdirMetadata(ctx, "pwn/sub", metadata)
+	require.Error(t, err, "applying metadata through a planted symlink should be refused")
+
+	_, err = xattr.LGet(evilSub, "user.pwned")
+	require.Error(t, err, "xattr escaped through planted symlink to %q", evilSub)
+}
+
+// TestXattrWriteOnlyFile checks user metadata can still be set on a
+// write-only file with --links, where the confined open can't read it.
+func TestXattrWriteOnlyFile(t *testing.T) {
+	if !xattrSupported {
+		t.Skip("xattrs not supported on this OS")
+	}
+	skipIfNoSymlinks(t)
+	ctx := context.Background()
+
+	r := fstest.NewRun(t)
+	f := r.Flocal.(*Fs)
+	linksMode(f)
+
+	require.NoError(t, putFile(ctx, f, "wo.txt", "write only"))
+	path := filepath.Join(f.root, "wo.txt")
+	require.NoError(t, os.Chmod(path, 0200))
+
+	obj, err := f.NewObject(ctx, "wo.txt")
+	require.NoError(t, err)
+	o := obj.(*Object)
+	err = o.setXattr(fs.Metadata{"potato": "chips"})
+	require.NoError(t, err)
+	if f.xattrSupported.Load() == 0 {
+		t.Skip("xattrs not supported on this filesystem")
+	}
+	// Reading xattrs needs read permission
+	require.NoError(t, os.Chmod(path, 0600))
+	v, err := xattr.LGet(path, "user.potato")
+	require.NoError(t, err)
+	assert.Equal(t, "chips", string(v))
+}

backend/netstorage/netstorage.go

@@ -208,6 +208,24 @@ func (d Du) GetName() xml.Name {
 
 //------------------------------------------------------------------------------
 
+// apiHeaders are the NetStorage API request headers which carry the
+// account name, the request signature and the signed action. They must
+// not be forwarded when a redirect leaves the configured host.
+//
+// The action header is sent uncanonicalised, hence the "*" prefix.
+var apiHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "*X-Akamai-ACS-Action"}
+
+// newClient makes the http client used for all NetStorage requests.
+//
+// The signer adds the API headers to every request, so the client
+// strips them on cross host redirects and refuses an HTTPS to HTTP
+// downgrade.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(apiHeaders...)
+	return client
+}
+
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 //
@@ -240,7 +258,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
-	client := fshttp.NewClient(ctx)
+	client := newClient(ctx)
 
 	f := &Fs{
 		name:        name,

backend/netstorage/netstorage_internal_test.go

@@ -4,12 +4,17 @@ import (
 	"context"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -59,3 +64,142 @@ func TestListResumeCannotChangeHost(t *testing.T) {
 	assert.Equal(t, 1, listRequests, "list request sent to the configured host")
 	assert.Equal(t, 0, requests, "request sent to the other host")
 }
+
+// apiTestHeaders is a deliberately literal copy of apiHeaders, without
+// the "*" prefix, so that a header dropped from the production list
+// shows up as a leak in the cross host tests rather than silently
+// losing coverage.
+var apiTestHeaders = []string{"X-Akamai-ACS-Auth-Data", "X-Akamai-ACS-Auth-Sign", "X-Akamai-ACS-Action"}
+
+// statXML is a minimal stat response saying the root is a directory
+const statXML = `<?xml version="1.0" encoding="UTF-8"?>
+<stat directory="/dir"><file type="dir" name="dir" mtime="1"/></stat>`
+
+// newTestFs makes an Fs via NewFs pointing at serverURL.
+//
+// NewFs makes a stat request for the root and treats any failure as
+// the root not existing, so callers check what the servers received
+// or make a further request rather than relying on its error.
+func newTestFs(ctx context.Context, t *testing.T, serverURL string) *Fs {
+	u, err := url.Parse(serverURL)
+	require.NoError(t, err)
+	m := configmap.Simple{
+		"type":     "netstorage",
+		"protocol": u.Scheme,
+		"host":     u.Host + "/dir",
+		"account":  "test-account",
+		"secret":   obscure.MustObscure("test-secret"),
+	}
+	f, err := NewFs(ctx, "netstorage-test", "", m)
+	require.NoError(t, err)
+	return f.(*Fs)
+}
+
+// assertAPIHeaders checks all the API headers are present (want) or
+// absent (!want) in r.
+func assertAPIHeaders(t *testing.T, r *http.Request, want bool) {
+	for _, header := range apiTestHeaders {
+		if want {
+			assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
+		} else {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+	}
+}
+
+// TestRedirectStripsAPIHeadersOnCrossHost checks the API headers are
+// sent to the configured host but not to another host it redirects to.
+func TestRedirectStripsAPIHeadersOnCrossHost(t *testing.T) {
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAPIHeaders(t, r, false)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer other.Close()
+
+	var originHits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAPIHeaders(t, r, true)
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	newTestFs(context.Background(), t, origin.URL)
+	assert.Equal(t, int32(1), originHits.Load())
+	assert.Equal(t, int32(1), otherHits.Load())
+}
+
+// TestUploadStripsAPIHeadersOnCrossHost checks an upload, which
+// carries the API headers on the PUT as well as in the trailers, does
+// not send them to another host the configured host redirects to.
+func TestUploadStripsAPIHeadersOnCrossHost(t *testing.T) {
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAPIHeaders(t, r, false)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer other.Close()
+
+	var originHits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAPIHeaders(t, r, true)
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusMovedPermanently)
+	}))
+	defer origin.Close()
+
+	ctx := context.Background()
+	f := newTestFs(ctx, t, origin.URL)
+	o := &Object{fs: f, remote: "file"}
+	src := object.NewStaticObjectInfo("file", time.Now(), 4, true, nil, f)
+	err := o.netStorageUploadRequest(ctx, strings.NewReader("data"), src)
+	require.NoError(t, err)
+	// NewFs stat, PUT and the stat after the upload
+	assert.Equal(t, int32(3), originHits.Load())
+	assert.Equal(t, int32(3), otherHits.Load())
+}
+
+// TestRedirectKeepsAPIHeadersOnSameHost checks the API headers are
+// still sent after a redirect which stays on the configured host.
+func TestRedirectKeepsAPIHeadersOnSameHost(t *testing.T) {
+	var hits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		assertAPIHeaders(t, r, true)
+		if !strings.HasSuffix(r.URL.Path, "/moved") {
+			http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
+			return
+		}
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer origin.Close()
+
+	newTestFs(context.Background(), t, origin.URL)
+	assert.Equal(t, int32(2), hits.Load())
+}
+
+// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
+// plaintext HTTP is not followed.
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+	var targetHits atomic.Int32
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		targetHits.Add(1)
+		_, _ = w.Write([]byte(statXML))
+	}))
+	defer target.Close()
+
+	origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.InsecureSkipVerify = true
+	f := newTestFs(ctx, t, origin.URL)
+	_, err := f.netStorageStatRequest(ctx, f.url(""), true)
+	require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	assert.Equal(t, int32(0), targetHits.Load())
+}

backend/onedrive/onedrive.go

@@ -41,6 +41,7 @@ import (
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/oauth2"
 )
 
 const (
@@ -844,19 +845,33 @@ type Options struct {
 
 // Fs represents a remote OneDrive
 type Fs struct {
-	name         string             // name of this remote
-	root         string             // the path we are working on
-	opt          Options            // parsed options
-	ci           *fs.ConfigInfo     // global config
-	features     *fs.Features       // optional features
-	srv          *rest.Client       // the connection to the OneDrive server
-	unAuth       *rest.Client       // no authentication connection to the OneDrive server
-	dirCache     *dircache.DirCache // Map of directory path to directory id
-	pacer        *fs.Pacer          // pacer for API calls
-	tokenRenewer *oauthutil.Renew   // renew the token on expiry
-	driveID      string             // ID to use for querying Microsoft Graph
-	driveType    string             // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
-	hashType     hash.Type          // type of the hash we are using
+	name         string                 // name of this remote
+	root         string                 // the path we are working on
+	opt          Options                // parsed options
+	ci           *fs.ConfigInfo         // global config
+	features     *fs.Features           // optional features
+	srv          *rest.Client           // the connection to the OneDrive server
+	unAuth       *rest.Client           // no authentication connection to the OneDrive server
+	dirCache     *dircache.DirCache     // Map of directory path to directory id
+	pacer        *fs.Pacer              // pacer for API calls
+	tokenRenewer *oauthutil.Renew       // renew the token on expiry
+	tokenSource  *oauthutil.TokenSource // source of the API token
+	sharePoint   *sharePointClients     // authenticated clients for SharePoint hosts
+	driveID      string                 // ID to use for querying Microsoft Graph
+	driveType    string                 // https://developer.microsoft.com/en-us/graph/docs/api-reference/v1.0/resources/drive
+	hashType     hash.Type              // type of the hash we are using
+}
+
+// sharePointClients holds the clients authenticated for SharePoint hosts
+type sharePointClients struct {
+	mu    sync.Mutex
+	hosts map[string]sharePointAuth
+}
+
+// sharePointAuth is a client authenticated for a SharePoint host
+type sharePointAuth struct {
+	client      *rest.Client
+	tokenSource oauth2.TokenSource
 }
 
 // Object describes a OneDrive object
@@ -1076,6 +1091,154 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
+// errPreauthRetired explains an authentication failure on a URL which
+// SharePoint used to return pre-authenticated.
+var errPreauthRetired = errors.New("SharePoint refused to authenticate a request - Microsoft is retiring pre-authenticated (tempauth) URLs, see https://rclone.org/onedrive/#pre-authenticated-urls")
+
+// sharePointDomains are the domains of SharePoint Online in the
+// different Microsoft clouds
+var sharePointDomains = []string{".sharepoint.com", ".sharepoint.us", ".sharepoint-mil.us", ".sharepoint.cn", ".sharepoint.de"}
+
+// stripTempauth removes the tempauth parameter from the URLs the API
+// returns, which simulates the retirement of pre-authenticated URLs
+// in tests.
+var stripTempauth = false
+
+// returnedURL processes a URL returned by the API before use
+func returnedURL(rawURL string) string {
+	if !stripTempauth {
+		return rawURL
+	}
+	u, err := url.Parse(rawURL)
+	if err != nil {
+		return rawURL
+	}
+	q := u.Query()
+	q.Del("tempauth")
+	u.RawQuery = q.Encode()
+	return u.String()
+}
+
+// urlAuth says how a URL returned by the API is authenticated
+type urlAuth int
+
+const (
+	urlAuthNone       urlAuth = iota // call without authentication
+	urlAuthAPI                       // call with the API token
+	urlAuthSharePoint                // call with a token for the SharePoint host
+)
+
+// apiHost returns the host the API calls made with f.srv go to.
+func (f *Fs) apiHost() string {
+	endpoint := graphAPIEndpoint[f.opt.Region]
+	if f.opt.TenantURL != "" {
+		endpoint = f.opt.TenantURL
+	}
+	u, err := url.Parse(endpoint)
+	if err != nil || u.Scheme != "https" {
+		return ""
+	}
+	return u.Host
+}
+
+// urlAuthFor returns how to authenticate a URL returned by the API,
+// such as an upload session or an async job monitor.
+//
+// Pre-authenticated URLs fail if they are sent an Authorization
+// header and tokens must not be sent to other hosts, so only HTTPS
+// URLs without tempauth on the API host or a SharePoint host are
+// authenticated.
+func (f *Fs) urlAuthFor(rawURL string) urlAuth {
+	u, err := url.Parse(rawURL)
+	if err != nil || u.Scheme != "https" || u.Query().Has("tempauth") {
+		return urlAuthNone
+	}
+	if apiHost := f.apiHost(); apiHost != "" && strings.EqualFold(u.Host, apiHost) {
+		return urlAuthAPI
+	}
+	if isSharePointHost(u.Hostname()) {
+		return urlAuthSharePoint
+	}
+	return urlAuthNone
+}
+
+// isSharePointHost returns true if host is in a SharePoint Online domain.
+func isSharePointHost(host string) bool {
+	host = strings.ToLower(host)
+	for _, domain := range sharePointDomains {
+		if strings.HasSuffix(host, domain) {
+			return true
+		}
+	}
+	return false
+}
+
+// checkSharePointRedirect stops the SharePoint token being sent to a
+// host which isn't SharePoint, as the token is added to every request
+// the client makes.
+func checkSharePointRedirect(req *http.Request, via []*http.Request) error {
+	if len(via) >= 10 {
+		return errors.New("stopped after 10 redirects")
+	}
+	if req.URL.Scheme != "https" || !isSharePointHost(req.URL.Hostname()) {
+		return fmt.Errorf("refusing to send a SharePoint token on a redirect to %s://%s", req.URL.Scheme, req.URL.Host)
+	}
+	return nil
+}
+
+// clientForURL returns the client to call a URL returned by the API with.
+func (f *Fs) clientForURL(ctx context.Context, rawURL string) (*rest.Client, error) {
+	switch f.urlAuthFor(rawURL) {
+	case urlAuthAPI:
+		return f.srv, nil
+	case urlAuthSharePoint:
+		u, _ := url.Parse(rawURL)
+		return f.sharePointClient(ctx, u.Host)
+	}
+	return f.unAuth, nil
+}
+
+// sharePointClient returns a client authenticated for the SharePoint
+// host, which upload session and async job monitor URLs need once
+// they are no longer pre-authenticated.
+//
+// The token is got from the existing credentials, which Entra ID
+// allows for SharePoint as the user has consented to the Graph file
+// scopes.
+func (f *Fs) sharePointClient(ctx context.Context, host string) (*rest.Client, error) {
+	if f.tokenSource == nil || f.sharePoint == nil {
+		return nil, fmt.Errorf("%w: no credentials to get a SharePoint token with", errPreauthRetired)
+	}
+	host = strings.ToLower(host)
+	f.sharePoint.mu.Lock()
+	defer f.sharePoint.mu.Unlock()
+	sp, ok := f.sharePoint.hosts[host]
+	if !ok {
+		tokenSource := f.tokenSource.ScopedTokenSource([]string{"https://" + host + "/.default"})
+		client := oauth2.NewClient(oauthutil.Context(ctx, fshttp.NewClient(ctx)), tokenSource)
+		client.CheckRedirect = checkSharePointRedirect
+		sp = sharePointAuth{client: rest.NewClient(client), tokenSource: tokenSource}
+		f.sharePoint.hosts[host] = sp
+		fs.Debugf(f, "Using a SharePoint token for %s", host)
+	}
+	// Get the token now so failures are reported before the client is used
+	if _, err := sp.tokenSource.Token(); err != nil {
+		return nil, fmt.Errorf("%w: failed to get a SharePoint token: %w", errPreauthRetired, err)
+	}
+	return sp.client, nil
+}
+
+// checkPreauthRetired adds an explanation to a 401 error from a URL
+// returned by the API which needs a SharePoint token, as these were
+// pre-authenticated until Microsoft retired them.
+func (f *Fs) checkPreauthRetired(rawURL string, resp *http.Response, err error) error {
+	if err == nil || resp == nil || resp.StatusCode != http.StatusUnauthorized ||
+		f.driveType == driveTypePersonal || errors.Is(err, errPreauthRetired) || f.urlAuthFor(rawURL) != urlAuthSharePoint {
+		return err
+	}
+	return fmt.Errorf("%w: %w", errPreauthRetired, err)
+}
+
 func checkUploadChunkSize(cs fs.SizeSuffix) error {
 	const minChunkSize = fs.SizeSuffixBase
 	if cs%chunkSizeMultiple != 0 {
@@ -1144,16 +1307,18 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:      name,
-		root:      root,
-		opt:       *opt,
-		ci:        ci,
-		driveID:   opt.DriveID,
-		driveType: opt.DriveType,
-		srv:       rest.NewClient(oAuthClient).SetRoot(rootURL),
-		unAuth:    rest.NewClient(client).SetRoot(rootURL),
-		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		hashType:  QuickXorHashType,
+		name:        name,
+		root:        root,
+		opt:         *opt,
+		ci:          ci,
+		driveID:     opt.DriveID,
+		driveType:   opt.DriveType,
+		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
+		tokenSource: ts,
+		sharePoint:  &sharePointClients{hosts: make(map[string]sharePointAuth)},
+		unAuth:      rest.NewClient(client).SetRoot(rootURL),
+		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		hashType:    QuickXorHashType,
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:          true,
@@ -1725,11 +1890,25 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 		var resp *http.Response
 		var err error
 		var body []byte
+		var redirected bool
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = http.Get(location)
+			req, err := http.NewRequestWithContext(ctx, http.MethodGet, location, nil)
+			if err != nil {
+				return false, err
+			}
+			client, err := f.clientForURL(ctx, location)
+			if err != nil {
+				return false, err
+			}
+			resp, err = client.Do(req)
 			if err != nil {
 				return fserrors.ShouldRetry(err), err
 			}
+			redirected = resp.Request.URL.String() != req.URL.String()
+			if resp.StatusCode == http.StatusUnauthorized {
+				_ = resp.Body.Close()
+				return false, f.checkPreauthRetired(location, resp, fmt.Errorf("async job monitor: HTTP error %s", resp.Status))
+			}
 			body, err = rest.ReadBody(resp)
 			return fserrors.ShouldRetry(err), err
 		})
@@ -1742,6 +1921,11 @@ func (f *Fs) waitForJob(ctx context.Context, location string, o *Object) error {
 		if err != nil {
 			return fmt.Errorf("async status result not JSON: %q: %w", body, err)
 		}
+		// An authenticated monitor redirects to the new item when the
+		// job has finished instead of returning its status
+		if status.Status == "" && redirected {
+			status.Status = "completed"
+		}
 
 		switch status.Status {
 		case "failed":
@@ -1841,7 +2025,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	}
 
 	// read location header
-	location := resp.Header.Get("Location")
+	location := returnedURL(resp.Header.Get("Location"))
 	if location == "" {
 		return nil, errors.New("didn't receive location header in copy response")
 	}
@@ -1852,6 +2036,10 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		fs.Debugf(src, "Server-side copy failed - file not shared between drives")
 		return nil, fs.ErrorCantCopy
 	}
+	if errors.Is(err, errPreauthRetired) {
+		fs.Debugf(src, "Server-side copy can't be monitored - copying another way: %v", err)
+		return nil, fs.ErrorCantCopy
+	}
 	if err != nil {
 		return nil, err
 	}
@@ -2526,16 +2714,23 @@ func (o *Object) openContentPrefer(ctx context.Context, options ...fs.OpenOption
 	return o.openWithRedirect(ctx, &opts)
 }
 
-// openWithRedirect downloads via /content style endpoints that 302 to a preauthenticated URL.
+// openWithRedirect downloads via /content style endpoints that 302 to
+// a download URL, which is pre-authenticated or needs a SharePoint
+// token.
 func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.ReadCloser, err error) {
 	var resp *http.Response
-	// Make a note of the redirect target as we need to call it without Auth
+	// Make a note of the redirect target as it needs different auth
 	var redirectReq *http.Request
 	opts.CheckRedirect = func(req *http.Request, via []*http.Request) error {
 		if len(via) >= 10 {
 			return errors.New("stopped after 10 redirects")
 		}
-		// Preauthenticated download URLs must not carry the Graph Authorization header.
+		if stripTempauth {
+			if u, err := url.Parse(returnedURL(req.URL.String())); err == nil {
+				req.URL = u
+			}
+		}
+		// Download URLs must not carry the Graph Authorization header.
 		req.Header.Del("Authorization")
 		// Keep Prefer on the redirect when forcing infected download; some SharePoint
 		// endpoints honor it only on the final download request.
@@ -2574,17 +2769,24 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
 		return nil, err
 	}
 	if redirectReq != nil {
+		redirectURL := redirectReq.URL.String()
+		var client *rest.Client
+		client, err = o.fs.clientForURL(ctx, redirectURL)
+		if err != nil {
+			return nil, err
+		}
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.unAuth.Do(redirectReq)
+			resp, err = client.Do(redirectReq)
 			if err != nil {
 				return shouldRetry(ctx, resp, err)
 			}
-			// unAuth.Do does not check status; treat non-2xx as failure so a malware
+			// Do does not check status; treat non-2xx as failure so a malware
 			// JSON body is not written out as file content.
 			if resp.StatusCode < 200 || resp.StatusCode > 299 {
 				body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
 				_ = resp.Body.Close()
 				err = fmt.Errorf("HTTP error %d (%s) %s", resp.StatusCode, resp.Status, strings.TrimSpace(string(body)))
+				err = o.fs.checkPreauthRetired(redirectURL, resp, err)
 				if strings.Contains(string(body), "malwareDetected") || resp.Header.Get("X-Virus-Infected") != "" {
 					err = malwareDownloadError(o.fs.opt.AVOverride, err)
 				}
@@ -2606,7 +2808,19 @@ func (o *Object) openWithRedirect(ctx context.Context, opts *rest.Opts) (in io.R
 		// Overwrite size with actual size since size readings from Onedrive is unreliable.
 		o.size = resp.ContentLength
 	}
-	return resp.Body, err
+	return o.checkRange(resp, opts.Options)
+}
+
+// checkRange returns the body of a download, checking it is the range
+// requested in options.
+func (o *Object) checkRange(resp *http.Response, options []fs.OpenOption) (in io.ReadCloser, err error) {
+	// The size isn't passed as it is unreliable on OneDrive
+	err = rest.CheckContentRange(resp, options, -1)
+	if err != nil {
+		_ = resp.Body.Close()
+		return nil, err
+	}
+	return resp.Body, nil
 }
 
 // malwareDownloadError formats an error when the server blocks a malware-flagged file.
@@ -2646,12 +2860,16 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
 	}
 	var info api.UploadFragmentResponse
 	var resp *http.Response
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return 0, err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
+		resp, err = client.CallJSON(ctx, &opts, nil, &info)
 		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
-		return 0, err
+		return 0, o.fs.checkPreauthRetired(url, resp, err)
 	}
 	if len(info.NextExpectedRanges) != 1 {
 		return 0, fmt.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
@@ -2675,6 +2893,10 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 	var resp *http.Response
 	var body []byte
 	skip := int64(0)
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return nil, err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		toSend := chunkSize - skip
 		opts := rest.Opts{
@@ -2686,7 +2908,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			Options:       options,
 		}
 		_, _ = chunk.Seek(skip, io.SeekStart)
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = client.Call(ctx, &opts)
 		if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
 			fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
 			pos, posErr := o.getPosition(ctx, url)
@@ -2725,7 +2947,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 		}
 		return false, nil
 	})
-	return info, err
+	return info, o.fs.checkPreauthRetired(url, resp, err)
 }
 
 // cancelUploadSession cancels an upload session
@@ -2736,11 +2958,15 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
 		NoResponse: true,
 	}
 	var resp *http.Response
+	client, err := o.fs.clientForURL(ctx, url)
+	if err != nil {
+		return err
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.unAuth.Call(ctx, &opts)
+		resp, err = client.Call(ctx, &opts)
 		return shouldRetry(ctx, resp, err)
 	})
-	return
+	return o.fs.checkPreauthRetired(url, resp, err)
 }
 
 // uploadMultipart uploads a file using multipart upload
@@ -2758,7 +2984,20 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 	if err != nil {
 		return nil, err
 	}
-	uploadURL := session.UploadURL
+	uploadURL := returnedURL(session.UploadURL)
+
+	// Check a session which needs SharePoint authentication can get it
+	// before reading any data, so it can be uploaded in a single part
+	// instead if it can't.
+	if o.fs.urlAuthFor(uploadURL) == urlAuthSharePoint {
+		if _, err = o.getPosition(ctx, uploadURL); err != nil {
+			if !errors.Is(err, errPreauthRetired) || size >= int64(maxSinglePartSize) {
+				return nil, err
+			}
+			fs.Debugf(o, "Uploading in a single part as the upload session can't be used: %v", err)
+			return o.uploadSinglepart(ctx, in, src, options...)
+		}
+	}
 
 	// Cancel the session if something went wrong
 	defer atexit.OnError(&err, func() {

backend/onedrive/onedrive_internal_test.go

@@ -3,6 +3,7 @@ package onedrive
 import (
 	"context"
 	"encoding/json"
+	"flag"
 	"fmt"
 	"net/http"
 	"net/http/httptest"
@@ -15,6 +16,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/backend/onedrive/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -43,6 +45,10 @@ const (
 	// https://learn.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_invite?view=odsp-graph-online#http-request-1
 )
 
+func init() {
+	flag.BoolVar(&stripTempauth, "strip-tempauth", false, "Remove tempauth from URLs the API returns to simulate the retirement of pre-authenticated URLs")
+}
+
 // TestMain drives the tests
 func TestMain(m *testing.M) {
 	fstest.TestMain(m)
@@ -770,3 +776,96 @@ func TestListDirectorySize(t *testing.T) {
 	assert.Equal(t, int64(2646204760), dir.Size())
 	assert.Equal(t, int64(3), dir.Items())
 }
+
+// TestWaitForJobUsesConfiguredClient checks that the async job monitor is
+// polled with the configured HTTP client and request context rather than
+// http.DefaultClient, so proxy, TLS and timeout options and cancellation apply.
+func TestWaitForJobUsesConfiguredClient(t *testing.T) {
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.UserAgent = "rclone-waitforjob-test"
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "rclone-waitforjob-test", r.UserAgent())
+		w.Header().Set("Content-Type", "application/json")
+		_ = json.NewEncoder(w).Encode(api.AsyncOperationStatus{Status: "deleteFailed"})
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(ctx))
+
+	err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "deleteFailed")
+}
+
+// TestWaitForJobCancel checks that cancelling the context stops a poll of
+// the async job monitor which is not answering.
+func TestWaitForJobCancel(t *testing.T) {
+	release := make(chan struct{})
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-r.Context().Done():
+		case <-release:
+		}
+	}))
+	t.Cleanup(server.Close)
+	t.Cleanup(func() { close(release) })
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	done := make(chan error, 1)
+	go func() { done <- f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt"}) }()
+	select {
+	case err := <-done:
+		assert.ErrorIs(t, err, context.DeadlineExceeded)
+	case <-time.After(5 * time.Second):
+		t.Fatal("waitForJob did not return after the context was cancelled")
+	}
+}
+
+// TestWaitForJobUnauthorized checks that a monitor URL which refuses
+// the request fails straight away instead of polling until the timeout.
+func TestWaitForJobUnauthorized(t *testing.T) {
+	var polls atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		polls.Add(1)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(http.StatusUnauthorized)
+		_ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "unauthenticated", "message": "no token"}})
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+	f.driveType = driveTypeBusiness
+
+	err := f.waitForJob(context.Background(), server.URL+"/monitor", &Object{remote: "file.txt"})
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "401")
+	assert.Equal(t, int32(1), polls.Load())
+}
+
+// TestWaitForJobRedirectCompleted checks that a monitor which
+// redirects to the new item, as it does when authenticated, is taken
+// to mean the job has finished.
+func TestWaitForJobRedirectCompleted(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/monitor":
+			http.Redirect(w, r, "/item", http.StatusSeeOther)
+		case "/item":
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"id":"newid","name":"file.txt","size":5,"file":{}}`)
+		default:
+			t.Errorf("unexpected request %s", r.URL)
+		}
+	}))
+	t.Cleanup(server.Close)
+	f, _ := newChangeNotifyTestFs(t, http.NotFoundHandler())
+	f.unAuth = rest.NewClient(fshttp.NewClient(context.Background()))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	err := f.waitForJob(ctx, server.URL+"/monitor", &Object{remote: "file.txt", hasMetaData: true})
+	require.NoError(t, err)
+}

backend/onedrive/upload_internal_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,11 +14,14 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/oauth2"
 )
 
 // fragmentServer is a fake OneDrive upload session which records the
@@ -66,6 +70,8 @@ func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		w.WriteHeader(http.StatusAccepted)
 		_, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)
+	case r.Method == "DELETE" && r.URL.Path == "/upload":
+		w.WriteHeader(http.StatusNoContent)
 	default:
 		s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
 		w.WriteHeader(http.StatusNotFound)
@@ -118,3 +124,327 @@ func TestUploadMultipartSkip(t *testing.T) {
 	assert.Equal(t, 2, s.fragments)
 	assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")
 }
+
+// TestURLAuthFor checks that tokens are only sent to HTTPS URLs on
+// the API host or a SharePoint host which aren't pre-authenticated.
+func TestURLAuthFor(t *testing.T) {
+	const tenant = "https://contoso-my.sharepoint.com/_api"
+	for _, tc := range []struct {
+		name      string
+		region    string
+		tenantURL string
+		url       string
+		want      urlAuth
+	}{
+		{"graph", "global", "", "https://graph.microsoft.com/v1.0/drives/x/items/y/uploadSession", urlAuthAPI},
+		{"graph case", "global", "", "https://Graph.Microsoft.com/v1.0/monitor/abc", urlAuthAPI},
+		{"graph tempauth", "global", "", "https://graph.microsoft.com/v1.0/upload?tempauth=abc", urlAuthNone},
+		{"graph http", "global", "", "http://graph.microsoft.com/v1.0/upload", urlAuthNone},
+		{"graph other region", "global", "", "https://graph.microsoft.us/v1.0/upload", urlAuthNone},
+		{"graph us", "us", "", "https://graph.microsoft.us/v1.0/upload", urlAuthAPI},
+		{"graph lookalike", "global", "", "https://graph.microsoft.com.example.com/upload", urlAuthNone},
+		{"sharepoint tempauth", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x&tempauth=abc", urlAuthNone},
+		{"sharepoint", "global", "", "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x", urlAuthSharePoint},
+		{"sharepoint case", "global", "", "https://Contoso.SharePoint.com/_api/v2.0/monitor/abc", urlAuthSharePoint},
+		{"sharepoint us", "us", "", "https://contoso.sharepoint.us/_api/v2.0/upload", urlAuthSharePoint},
+		{"sharepoint cn", "cn", "", "https://contoso.sharepoint.cn/_api/v2.0/upload", urlAuthSharePoint},
+		{"sharepoint http", "global", "", "http://contoso-my.sharepoint.com/_api/v2.0/upload", urlAuthNone},
+		{"sharepoint lookalike", "global", "", "https://contoso.sharepoint.com.example.com/upload", urlAuthNone},
+		{"sharepoint suffix only", "global", "", "https://evilsharepoint.com/upload", urlAuthNone},
+		{"personal", "global", "", "https://api.onedrive.com/rup/abc", urlAuthNone},
+		{"personal upload", "global", "", "https://my.microsoftpersonalcontent.com/personal/abc/_api/v2.0/drive/items/x/uploadSession", urlAuthNone},
+		{"unparseable", "global", "", "https://graph.microsoft.com/%zz", urlAuthNone},
+		{"tenant", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/monitor/abc", urlAuthAPI},
+		{"tenant tempauth", "global", tenant, "https://contoso-my.sharepoint.com/_api/v2.0/upload?tempauth=abc", urlAuthNone},
+		{"tenant graph", "global", tenant, "https://graph.microsoft.com/v1.0/upload", urlAuthNone},
+		{"tenant other sharepoint", "global", tenant, "https://contoso.sharepoint.com/_api/v2.0/upload", urlAuthSharePoint},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &Fs{}
+			f.opt.Region = tc.region
+			f.opt.TenantURL = tc.tenantURL
+			assert.Equal(t, tc.want, f.urlAuthFor(tc.url))
+		})
+	}
+}
+
+// TestCheckSharePointRedirect checks the SharePoint token is only
+// sent on redirects to SharePoint over HTTPS.
+func TestCheckSharePointRedirect(t *testing.T) {
+	for _, tc := range []struct {
+		url string
+		ok  bool
+	}{
+		{"https://contoso-my.sharepoint.com/download.aspx", true},
+		{"https://205807-ipv4v6.gr.global.aa-rt.sharepoint.com/download.aspx?tempauth=x", true},
+		{"https://contoso.sharepoint.us/download.aspx", true},
+		{"http://contoso-my.sharepoint.com/download.aspx", false},
+		{"https://contoso.sharepoint.com.example.com/download.aspx", false},
+		{"https://graph.microsoft.com/v1.0/me", false},
+		{"https://example.com/", false},
+	} {
+		req, err := http.NewRequest("GET", tc.url, nil)
+		require.NoError(t, err)
+		err = checkSharePointRedirect(req, nil)
+		if tc.ok {
+			assert.NoError(t, err, tc.url)
+		} else {
+			assert.Error(t, err, tc.url)
+		}
+	}
+	req, err := http.NewRequest("GET", "https://contoso-my.sharepoint.com/", nil)
+	require.NoError(t, err)
+	assert.Error(t, checkSharePointRedirect(req, make([]*http.Request, 10)), "too many redirects")
+}
+
+// TestCheckPreauthRetired checks which 401 errors get the explanation.
+func TestCheckPreauthRetired(t *testing.T) {
+	const sharePointURL = "https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x"
+	errBase := errors.New("HTTP error 401")
+	for _, tc := range []struct {
+		name      string
+		driveType string
+		url       string
+		status    int
+		want      bool
+	}{
+		{"sharepoint 401", driveTypeBusiness, sharePointURL, http.StatusUnauthorized, true},
+		{"library 401", driveTypeSharepoint, sharePointURL, http.StatusUnauthorized, true},
+		{"personal 401", driveTypePersonal, sharePointURL, http.StatusUnauthorized, false},
+		{"sharepoint 403", driveTypeBusiness, sharePointURL, http.StatusForbidden, false},
+		{"tempauth 401", driveTypeBusiness, sharePointURL + "&tempauth=abc", http.StatusUnauthorized, false},
+		{"graph 401", driveTypeBusiness, "https://graph.microsoft.com/v1.0/upload", http.StatusUnauthorized, false},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			f := &Fs{driveType: tc.driveType}
+			f.opt.Region = "global"
+			err := f.checkPreauthRetired(tc.url, &http.Response{StatusCode: tc.status}, errBase)
+			assert.ErrorIs(t, err, errBase)
+			assert.Equal(t, tc.want, errors.Is(err, errPreauthRetired), "err = %v", err)
+		})
+	}
+	f := &Fs{driveType: driveTypeBusiness}
+	assert.NoError(t, f.checkPreauthRetired(sharePointURL, &http.Response{StatusCode: http.StatusUnauthorized}, nil))
+	assert.ErrorIs(t, f.checkPreauthRetired(sharePointURL, nil, errBase), errBase)
+}
+
+// TestSharePointClient checks that the client for a SharePoint host
+// is authenticated with a token for that host got from the existing
+// refresh token, which is fetched once and not stored.
+func TestSharePointClient(t *testing.T) {
+	var scopes []string
+	var auths []string
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/token":
+			_ = r.ParseForm()
+			assert.Equal(t, "refresh_token", r.PostForm.Get("grant_type"))
+			assert.Equal(t, "refresh", r.PostForm.Get("refresh_token"))
+			scopes = append(scopes, r.PostForm.Get("scope"))
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"access_token":"sharepoint_token","token_type":"Bearer","expires_in":3600}`)
+		case "/upload":
+			auths = append(auths, r.Header.Get("Authorization"))
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = fmt.Fprint(w, `{"nextExpectedRanges":["0-"]}`)
+		case "/redirect":
+			http.Redirect(w, r, "/upload", http.StatusFound)
+		default:
+			t.Errorf("unexpected request %s %s", r.Method, r.URL)
+		}
+	}))
+	t.Cleanup(server.Close)
+
+	storedToken := `{"access_token":"graph_token","token_type":"Bearer","refresh_token":"refresh","expiry":"2099-01-01T00:00:00Z"}`
+	m := configmap.Simple{"token": storedToken}
+	ctx := context.Background()
+	_, ts, err := oauthutil.NewClientWithBaseClient(ctx, "remote", m, &oauthutil.Config{
+		ClientID:     "client_id",
+		ClientSecret: "client_secret",
+		TokenURL:     server.URL + "/token",
+	}, server.Client())
+	require.NoError(t, err)
+	f := &Fs{tokenSource: ts, sharePoint: &sharePointClients{hosts: make(map[string]sharePointAuth)}}
+
+	for range 2 {
+		client, err := f.sharePointClient(ctx, "Contoso-my.sharepoint.com")
+		require.NoError(t, err)
+		_, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/upload"})
+		require.NoError(t, err)
+	}
+	assert.Equal(t, []string{"https://contoso-my.sharepoint.com/.default"}, scopes, "token should be fetched once for the host")
+	assert.Equal(t, []string{"Bearer sharepoint_token", "Bearer sharepoint_token"}, auths)
+	assert.Equal(t, storedToken, m["token"], "the stored token must not change")
+
+	// The token isn't sent on a redirect to a host which isn't SharePoint
+	client, err := f.sharePointClient(ctx, "contoso-my.sharepoint.com")
+	require.NoError(t, err)
+	_, err = client.Call(ctx, &rest.Opts{Method: "GET", RootURL: server.URL + "/redirect"})
+	assert.ErrorContains(t, err, "refusing to send a SharePoint token")
+	assert.Len(t, auths, 2)
+
+	// Without credentials there is no client and the error explains why
+	_, err = (&Fs{}).sharePointClient(ctx, "contoso-my.sharepoint.com")
+	assert.ErrorIs(t, err, errPreauthRetired)
+}
+
+// TestUploadSessionFallback checks that an upload session which needs
+// SharePoint authentication which can't be got is replaced by a single
+// part upload before any data is read.
+func TestUploadSessionFallback(t *testing.T) {
+	content := bytes.Repeat([]byte("onedrive"), 1024)
+	var requests []string
+	var uploaded []byte
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requests = append(requests, r.Method+" "+r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		switch {
+		case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":
+			_, _ = fmt.Fprint(w, `{"uploadUrl":"https://contoso-my.sharepoint.com/_api/v2.0/upload?guid=x","nextExpectedRanges":["0-"]}`)
+		case r.Method == "PUT" && r.URL.Path == "/root/content":
+			var err error
+			uploaded, err = io.ReadAll(r.Body)
+			require.NoError(t, err)
+			_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
+		case r.Method == "PATCH":
+			_, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, len(uploaded))
+		default:
+			t.Errorf("unexpected request %s %s", r.Method, r.URL)
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(server.Close)
+	client := rest.NewClient(server.Client()).SetRoot(server.URL)
+	f := &Fs{
+		ci:        fs.GetConfig(context.Background()),
+		srv:       client,
+		unAuth:    client,
+		driveType: driveTypeBusiness,
+		pacer:     fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+	}
+	f.opt.Region = "global"
+	f.opt.ChunkSize = fs.SizeSuffix(len(content) / 2)
+	o := &Object{fs: f}
+
+	src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)
+	_, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)
+	require.NoError(t, err)
+	assert.Equal(t, content, uploaded)
+	assert.Equal(t, []string{"POST /root/createUploadSession", "PUT /root/content"}, requests[:2])
+}
+
+// redirectTransport sends every request to the test server, so URLs
+// on other hosts can be tested.
+type redirectTransport struct {
+	server *httptest.Server
+}
+
+func (t redirectTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	req = req.Clone(req.Context())
+	req.URL.Scheme = "http"
+	req.URL.Host = t.server.Listener.Addr().String()
+	return t.server.Client().Transport.RoundTrip(req)
+}
+
+// newDownloadTest returns an Object which downloads content from a
+// server which handles the /content call with contentHandler and
+// serves download.aspx with Range support, recording the
+// Authorization header of each download.
+func newDownloadTest(t *testing.T, content []byte, contentHandler http.HandlerFunc) (o *Object, auths *[]string) {
+	auths = new([]string)
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/items/fake/content":
+			contentHandler(w, r)
+		case "/download.aspx":
+			*auths = append(*auths, r.Header.Get("Authorization"))
+			http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(content))
+		default:
+			t.Errorf("unexpected request %s %s", r.Method, r.URL)
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	t.Cleanup(server.Close)
+	transport := redirectTransport{server: server}
+	sharePointToken := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "sharepoint_token"})
+	sharePointHTTP := &http.Client{Transport: &oauth2.Transport{Source: sharePointToken, Base: transport}}
+	f := &Fs{
+		ci:          fs.GetConfig(context.Background()),
+		srv:         rest.NewClient(server.Client()).SetRoot(server.URL),
+		unAuth:      rest.NewClient(&http.Client{Transport: transport}),
+		driveType:   driveTypeBusiness,
+		tokenSource: &oauthutil.TokenSource{},
+		sharePoint: &sharePointClients{hosts: map[string]sharePointAuth{
+			"contoso-my.sharepoint.com": {client: rest.NewClient(sharePointHTTP), tokenSource: sharePointToken},
+		}},
+		pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),
+	}
+	f.opt.Region = "global"
+	return &Object{fs: f, id: "fake", size: int64(len(content))}, auths
+}
+
+// TestOpenRedirectAuth checks that downloads are redirected to a
+// pre-authenticated URL without authentication and to a SharePoint
+// URL with a SharePoint token.
+func TestOpenRedirectAuth(t *testing.T) {
+	content := []byte("0123456789abcdefghij")
+	for _, test := range []struct {
+		name     string
+		location string
+		wantAuth string
+	}{
+		{"tempauth", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x&tempauth=v1.secret", ""},
+		{"sharepoint", "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", "Bearer sharepoint_token"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
+				http.Redirect(w, r, test.location, http.StatusFound)
+			})
+			in, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
+			require.NoError(t, err)
+			got, err := io.ReadAll(in)
+			require.NoError(t, err)
+			require.NoError(t, in.Close())
+			assert.Equal(t, "56789", string(got))
+			assert.Equal(t, []string{test.wantAuth}, *auths)
+		})
+	}
+
+	// Without credentials the error explains why
+	o, auths := newDownloadTest(t, content, func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "https://contoso-my.sharepoint.com/download.aspx?UniqueId=x", http.StatusFound)
+	})
+	o.fs.tokenSource = nil
+	_, err := o.Open(context.Background())
+	assert.ErrorIs(t, err, errPreauthRetired)
+	assert.Empty(t, *auths)
+}
+
+// TestOpenRangeCheck checks that a download which doesn't return the
+// requested range is an error rather than wrong data.
+func TestOpenRangeCheck(t *testing.T) {
+	content := []byte("0123456789abcdefghij")
+	for _, test := range []struct {
+		name    string
+		handler http.HandlerFunc
+		wantErr error
+	}{
+		{"ignored", func(w http.ResponseWriter, r *http.Request) {
+			_, _ = w.Write(content)
+		}, fs.ErrorRangeIgnored},
+		{"wrong range", func(w http.ResponseWriter, r *http.Request) {
+			w.Header().Set("Content-Range", "bytes 0-4/20")
+			w.WriteHeader(http.StatusPartialContent)
+			_, _ = w.Write(content[:5])
+		}, nil},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			o, _ := newDownloadTest(t, content, test.handler)
+			_, err := o.Open(context.Background(), &fs.RangeOption{Start: 5, End: 9})
+			require.Error(t, err)
+			if test.wantErr != nil {
+				assert.ErrorIs(t, err, test.wantErr)
+			}
+		})
+	}
+}

backend/oracleobjectstorage/client.go

@@ -126,6 +126,10 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 		if ociError.GetCode() == "RequestTimeout" {
 			return true, err
 		}
+		// Parallel part uploads can collide on the same object
+		if ociError.GetHTTPStatusCode() == http.StatusConflict && ociError.GetCode() == "ConcurrentObjectUpdate" {
+			return true, err
+		}
 	}
 	// Ok, not an oci error, check for generic failure conditions
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err

backend/oracleobjectstorage/client_test.go

@@ -102,3 +102,34 @@ func TestGzipEncoding(t *testing.T) {
 		assert.Equal(t, "", md5sum)
 	})
 }
+
+// serviceError is a minimal common.ServiceError for shouldRetry
+type serviceError struct {
+	status int
+	code   string
+}
+
+func (e serviceError) Error() string           { return e.code }
+func (e serviceError) GetHTTPStatusCode() int  { return e.status }
+func (e serviceError) GetMessage() string      { return e.code }
+func (e serviceError) GetCode() string         { return e.code }
+func (e serviceError) GetOpcRequestID() string { return "" }
+
+func TestShouldRetryServiceError(t *testing.T) {
+	ctx := context.Background()
+	for _, test := range []struct {
+		status int
+		code   string
+		want   bool
+	}{
+		{http.StatusConflict, "ConcurrentObjectUpdate", true},
+		{http.StatusConflict, "BucketAlreadyExists", false},
+		{http.StatusRequestTimeout, "RequestTimeout", true},
+		{http.StatusNotFound, "ObjectNotFound", false},
+	} {
+		err := serviceError{status: test.status, code: test.code}
+		retry, gotErr := shouldRetry(ctx, nil, err)
+		assert.Equal(t, test.want, retry, test.code)
+		assert.Equal(t, error(err), gotErr)
+	}
+}

backend/pcloud/pcloud.go

@@ -1537,6 +1537,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		if err != nil {
 			return fmt.Errorf("failed to make multipart upload for 0 length file: %w", err)
 		}
+		// Stop the form writer reading in if the request fails before sending it all
+		defer func() { _ = formReader.Close() }()
 
 		contentLength := overhead + size
 

backend/pikpak/helper.go

@@ -184,7 +184,7 @@ func (f *Fs) patchFile(ctx context.Context, ID string, req *api.File) (info *api
 }
 
 // getTask gets api.Task from API for the ID passed
-func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api.Task, err error) {
+func (f *Fs) getTask(ctx context.Context, ID string) (info *api.Task, err error) {
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   "/drive/v1/tasks/" + ID,
@@ -192,27 +192,48 @@ func (f *Fs) getTask(ctx context.Context, ID string, checkPhase bool) (info *api
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
-		if checkPhase {
-			if err == nil && info.Phase != api.PhaseTypeComplete {
-				// could be pending right after the task is created
-				return true, fmt.Errorf("%s (%s) is still in %s", info.Name, info.Type, info.Phase)
-			}
-		}
 		return f.shouldRetry(ctx, resp, err)
 	})
 	return
 }
 
-// waitTask waits for async tasks to be completed
+// waitTask waits up to taskTimeout for the async task ID to complete
 func (f *Fs) waitTask(ctx context.Context, ID string) (err error) {
-	time.Sleep(taskWaitTime)
-	if info, err := f.getTask(ctx, ID, true); err != nil {
-		if info == nil {
-			return fmt.Errorf("can't verify the task is completed: %q", ID)
+	start := time.Now()
+	sleep := taskWaitTime
+	for {
+		if err := sleepCtx(ctx, sleep); err != nil {
+			return err
 		}
-		return fmt.Errorf("can't verify the task is completed: %#v", info)
+		info, err := f.getTask(ctx, ID)
+		if err != nil {
+			return fmt.Errorf("can't verify the task %q is completed: %w", ID, err)
+		}
+		switch info.Phase {
+		case api.PhaseTypeComplete:
+			fs.Debugf(f, "task %q (%s) completed after %v", ID, info.Type, time.Since(start))
+			return nil
+		case api.PhaseTypeError:
+			return fmt.Errorf("task failed: %#v", info)
+		}
+		// Tasks can stay pending for tens of seconds when the server is busy
+		if time.Since(start) > taskTimeout {
+			return fmt.Errorf("can't verify the task is completed after %v: %#v", time.Since(start), info)
+		}
+		sleep = min(2*sleep, taskMaxSleep)
+	}
+}
+
+// sleepCtx sleeps for d or until ctx is cancelled
+func sleepCtx(ctx context.Context, d time.Duration) error {
+	timer := time.NewTimer(d)
+	defer timer.Stop()
+	select {
+	case <-timer.C:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
 	}
-	return
 }
 
 // deleteTask remove a task having the specified ID

backend/pikpak/pikpak.go

@@ -72,6 +72,8 @@ const (
 	minSleep         = 100 * time.Millisecond
 	maxSleep         = 2 * time.Second
 	taskWaitTime     = 500 * time.Millisecond
+	taskMaxSleep     = 5 * time.Second
+	taskTimeout      = 5 * time.Minute
 	decayConstant    = 2 // bigger for slower decay, exponential
 	rootURL          = "https://api-drive.mypikpak.com"
 
@@ -1388,6 +1390,8 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
 	if err != nil {
 		return fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading in if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	contentLength := overhead + size
 	opts := rest.Opts{
@@ -1946,6 +1950,12 @@ func (o *Object) httpResponse(ctx context.Context, url, method string, options [
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
+		// client.Do does not fail a completed non-2xx response, so the
+		// HTML error page would be returned as the file. errorHandler
+		// closes the body; shouldRetry then retries retryErrorCodes.
+		if err == nil && (res.StatusCode < 200 || res.StatusCode > 299) {
+			err = errorHandler(res)
+		}
 		return o.fs.shouldRetry(ctx, res, err)
 	})
 	if err != nil {

backend/pikpak/pikpak_internal_test.go

@@ -0,0 +1,185 @@
+package pikpak
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"sync/atomic"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// closeTracker counts Close calls on a response body.
+type closeTracker struct {
+	io.ReadCloser
+	closed *atomic.Int32
+}
+
+func (b closeTracker) Close() error {
+	b.closed.Add(1)
+	return b.ReadCloser.Close()
+}
+
+type roundTripperFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) {
+	return f(r)
+}
+
+// newDownloadTestObject returns an object whose downloads are served by srv.
+// closed counts response bodies closed through the client.
+func newDownloadTestObject(t *testing.T, srv *httptest.Server, closed *atomic.Int32) *Object {
+	t.Helper()
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.LowLevelRetries = 3
+	client := *srv.Client()
+	base := client.Transport
+	if base == nil {
+		base = http.DefaultTransport
+	}
+	client.Transport = roundTripperFunc(func(req *http.Request) (*http.Response, error) {
+		res, err := base.RoundTrip(req)
+		if err == nil && res != nil && res.Body != nil {
+			res.Body = closeTracker{ReadCloser: res.Body, closed: closed}
+		}
+		return res, err
+	})
+	return &Object{
+		fs: &Fs{
+			client: &client,
+			pacer:  fs.NewPacer(ctx, &pacer.ZeroDelayCalculator{}),
+		},
+		size: 1,
+	}
+}
+
+// TestOpenHTTPStatus checks that a non-2xx download response is an error
+// (body closed, retriable statuses retried by the pacer) and that 200 and
+// 206 responses are still returned as file content.
+func TestOpenHTTPStatus(t *testing.T) {
+	const htmlBody = "<html><head><title>502 Bad Gateway</title></head><body>502</body></html>"
+	fileBody := []byte("pikpak-file-bytes-0123456789")
+
+	t.Run("502", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			w.Header().Set("Content-Type", "text/html")
+			w.WriteHeader(http.StatusBadGateway)
+			_, _ = io.WriteString(w, htmlBody)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		in, err := o.open(context.Background(), srv.URL)
+		if err == nil {
+			got, readErr := io.ReadAll(in)
+			require.NoError(t, readErr)
+			require.NoError(t, in.Close())
+			t.Fatalf("open returned the error page as file data (%d bytes): %q", len(got), got)
+		}
+		assert.Nil(t, in)
+		assert.ErrorContains(t, err, "502")
+		assert.NotContains(t, err.Error(), htmlBody)
+		assert.Equal(t, int32(3), hits.Load(), "502 should be retried by the pacer")
+		assert.Equal(t, hits.Load(), closed.Load(), "error response bodies should be closed")
+	})
+
+	t.Run("404", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			http.Error(w, "missing", http.StatusNotFound)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		in, err := o.open(context.Background(), srv.URL)
+		require.Error(t, err)
+		assert.Nil(t, in)
+		assert.Equal(t, int32(1), hits.Load(), "404 is not a retry status")
+		assert.Equal(t, hits.Load(), closed.Load())
+	})
+
+	t.Run("502 then 200", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if hits.Add(1) == 1 {
+				w.Header().Set("Content-Type", "text/html")
+				w.WriteHeader(http.StatusBadGateway)
+				_, _ = io.WriteString(w, htmlBody)
+				return
+			}
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write(fileBody)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		in, err := o.open(context.Background(), srv.URL)
+		require.NoError(t, err)
+		got, err := io.ReadAll(in)
+		require.NoError(t, err)
+		require.NoError(t, in.Close())
+		assert.Equal(t, fileBody, got)
+		assert.Equal(t, int32(2), hits.Load())
+		assert.Equal(t, int32(2), closed.Load())
+	})
+
+	t.Run("200", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			if r.Header.Get("Range") != "" {
+				t.Errorf("unexpected Range header %q", r.Header.Get("Range"))
+			}
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.WriteHeader(http.StatusOK)
+			_, _ = w.Write(fileBody)
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		o.size = int64(len(fileBody))
+		in, err := o.open(context.Background(), srv.URL)
+		require.NoError(t, err)
+		got, err := io.ReadAll(in)
+		require.NoError(t, err)
+		assert.Equal(t, int32(0), closed.Load(), "successful body stays open for the caller")
+		require.NoError(t, in.Close())
+		assert.Equal(t, fileBody, got)
+		assert.Equal(t, int32(1), hits.Load())
+		assert.Equal(t, int32(1), closed.Load())
+	})
+
+	t.Run("206", func(t *testing.T) {
+		var closed atomic.Int32
+		var hits atomic.Int32
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			hits.Add(1)
+			assert.Equal(t, "bytes=0-3", r.Header.Get("Range"))
+			w.Header().Set("Content-Type", "application/octet-stream")
+			w.Header().Set("Content-Range", "bytes 0-3/"+strconv.Itoa(len(fileBody)))
+			w.WriteHeader(http.StatusPartialContent)
+			_, _ = w.Write(fileBody[:4])
+		}))
+		t.Cleanup(srv.Close)
+		o := newDownloadTestObject(t, srv, &closed)
+		o.size = int64(len(fileBody))
+		in, err := o.open(context.Background(), srv.URL, &fs.RangeOption{Start: 0, End: 3})
+		require.NoError(t, err)
+		got, err := io.ReadAll(in)
+		require.NoError(t, err)
+		require.NoError(t, in.Close())
+		assert.Equal(t, fileBody[:4], got)
+		assert.Equal(t, int32(1), hits.Load())
+		assert.Equal(t, int32(1), closed.Load())
+	})
+}

backend/s3/provider/Autonomys.yaml

@@ -0,0 +1,10 @@
+name: Autonomys
+description: Autonomys Auto Drive
+endpoint:
+  s3.auto-drive.autonomys.xyz: Auto Drive mainnet S3 endpoint
+quirks:
+  force_path_style: true
+  # UploadPartCopy is not supported, so copy in a single request
+  copy_cutoff: 9223372036854775807
+  # Object Lock is always on in COMPLIANCE mode and can't be set by clients
+  object_lock_supported: false

backend/s3/provider/Cellar.yaml

@@ -0,0 +1,15 @@
+name: Cellar
+description: Clever Cloud Cellar Object Storage
+endpoint:
+  cellar-c2.services.clever-cloud.com: Paris, France
+acl: {}
+bucket_acl: true
+sse_customer_algorithm: true
+sse_customer_key: true
+sse_customer_key_base64: true
+sse_customer_key_md5: true
+quirks:
+  # Object Lock headers are not applied on CreateMultipartUpload, so the
+  # Object Lock tests are skipped. Users should set
+  # --s3-object-lock-set-after-upload when using Object Lock.
+  object_lock_supported: false

backend/s3/provider/HuaweiOBS.yaml

@@ -1,37 +1,73 @@
 name: HuaweiOBS
 description: Huawei Object Storage Service
 region:
+  af-north-1: AF-Cairo
   af-south-1: AF-Johannesburg
+  ap-southeast-1: CN-Hong Kong
   ap-southeast-2: AP-Bangkok
   ap-southeast-3: AP-Singapore
-  cn-east-3: CN East-Shanghai1
+  ap-southeast-4: AP-Jakarta
+  ap-southeast-5: AP-Manila
   cn-east-2: CN East-Shanghai2
+  cn-east-3: CN East-Shanghai1
+  cn-east-4: CN East-Qingdao
+  cn-east-5: CN East-Suzhou
   cn-north-1: CN North-Beijing1
+  cn-north-2: CN North-Beijing2
   cn-north-4: CN North-Beijing4
+  cn-north-9: CN North-Ulanqab
+  cn-north-11: CN North-Hebei
+  cn-north-12: CN North-Hohhot
   cn-south-1: CN South-Guangzhou
-  ap-southeast-1: CN-Hong Kong
-  sa-argentina-1: LA-Buenos Aires1
-  sa-peru-1: LA-Lima1
+  cn-south-2: CN South-Shenzhen
+  cn-south-4: CN South-Guangzhou-InvitationOnly
+  cn-southwest-2: CN Southwest-Guiyang1
+  cn-southwest-3: CN Southwest-Chongqing
+  eu-west-101: EU-Dublin
+  la-north-2: LA-Mexico City2
+  la-south-2: LA-Santiago
+  me-east-1: ME-Riyadh
   na-mexico-1: LA-Mexico City1
-  sa-chile-1: LA-Santiago2
-  sa-brazil-1: LA-Sao Paulo1
   ru-northwest-2: RU-Moscow2
+  sa-argentina-1: LA-Buenos Aires1
+  sa-brazil-1: LA-Sao Paulo1
+  sa-chile-1: LA-Santiago2
+  sa-peru-1: LA-Lima1
+  tr-west-1: TR-Istanbul
 endpoint:
+  obs.af-north-1.myhuaweicloud.com: AF-Cairo
   obs.af-south-1.myhuaweicloud.com: AF-Johannesburg
+  obs.ap-southeast-1.myhuaweicloud.com: CN-Hong Kong
   obs.ap-southeast-2.myhuaweicloud.com: AP-Bangkok
   obs.ap-southeast-3.myhuaweicloud.com: AP-Singapore
-  obs.cn-east-3.myhuaweicloud.com: CN East-Shanghai1
+  obs.ap-southeast-4.myhuaweicloud.com: AP-Jakarta
+  obs.ap-southeast-5.myhuaweicloud.com: AP-Manila
   obs.cn-east-2.myhuaweicloud.com: CN East-Shanghai2
+  obs.cn-east-3.myhuaweicloud.com: CN East-Shanghai1
+  obs.cn-east-4.myhuaweicloud.com: CN East-Qingdao
+  obs.cn-east-5.myhuaweicloud.com: CN East-Suzhou
   obs.cn-north-1.myhuaweicloud.com: CN North-Beijing1
+  obs.cn-north-2.myhuaweicloud.com: CN North-Beijing2
   obs.cn-north-4.myhuaweicloud.com: CN North-Beijing4
+  obs.cn-north-9.myhuaweicloud.com: CN North-Ulanqab
+  obs.cn-north-11.myhuaweicloud.com: CN North-Hebei
+  obs.cn-north-12.myhuaweicloud.com: CN North-Hohhot
   obs.cn-south-1.myhuaweicloud.com: CN South-Guangzhou
-  obs.ap-southeast-1.myhuaweicloud.com: CN-Hong Kong
-  obs.sa-argentina-1.myhuaweicloud.com: LA-Buenos Aires1
-  obs.sa-peru-1.myhuaweicloud.com: LA-Lima1
+  obs.cn-south-2.myhuaweicloud.com: CN South-Shenzhen
+  obs.cn-south-4.myhuaweicloud.com: CN South-Guangzhou-InvitationOnly
+  obs.cn-southwest-2.myhuaweicloud.com: CN Southwest-Guiyang1
+  obs.cn-southwest-3.myhuaweicloud.com: CN Southwest-Chongqing
+  obs.eu-west-101.myhuaweicloud.eu: EU-Dublin
+  obs.la-north-2.myhuaweicloud.com: LA-Mexico City2
+  obs.la-south-2.myhuaweicloud.com: LA-Santiago
+  obs.me-east-1.myhuaweicloud.com: ME-Riyadh
   obs.na-mexico-1.myhuaweicloud.com: LA-Mexico City1
-  obs.sa-chile-1.myhuaweicloud.com: LA-Santiago2
-  obs.sa-brazil-1.myhuaweicloud.com: LA-Sao Paulo1
   obs.ru-northwest-2.myhuaweicloud.com: RU-Moscow2
+  obs.sa-argentina-1.myhuaweicloud.com: LA-Buenos Aires1
+  obs.sa-brazil-1.myhuaweicloud.com: LA-Sao Paulo1
+  obs.sa-chile-1.myhuaweicloud.com: LA-Santiago2
+  obs.sa-peru-1.myhuaweicloud.com: LA-Lima1
+  obs.tr-west-1.myhuaweicloud.com: TR-Istanbul
 acl: {}
 bucket_acl: true
 quirks:

backend/s3/provider/Infomaniak.yaml

@@ -0,0 +1,18 @@
+name: Infomaniak
+description: Infomaniak Swiss Backup
+region:
+  us-east-1: |-
+    Default region for compatibility.
+    All data is stored in Switzerland.
+endpoint:
+  s3.swiss-backup01.infomaniak.com: Swiss Backup Location 01
+  s3.swiss-backup02.infomaniak.com: Swiss Backup Location 02
+  s3.swiss-backup03.infomaniak.com: Swiss Backup Location 03
+  s3.swiss-backup04.infomaniak.com: Swiss Backup Location 04
+location_constraint: {}
+acl: {}
+bucket_acl: true
+quirks:
+  force_path_style: true
+  use_multipart_etag: false
+

backend/s3/provider/Neon.yaml

@@ -0,0 +1,11 @@
+name: Neon
+description: Neon Object Storage
+region:
+  us-east-1: US East (N. Virginia)
+  us-east-2: US East (Ohio)
+  eu-central-1: Europe (Frankfurt)
+  ap-southeast-1: Asia Pacific (Singapore)
+endpoint: {}
+quirks:
+  force_path_style: true # virtual-hosted style requests are not supported
+  object_lock_supported: false # object lock config is stored but enforcement is limited

backend/s3/s3.go

@@ -190,7 +190,7 @@ header is added and the default (private) will be used.
 			Name: "sse_customer_key",
 			Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
 
-Alternatively you can provide --sse-customer-key-base64.`,
+Alternatively you can provide --s3-sse-customer-key-base64.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -201,7 +201,7 @@ Alternatively you can provide --sse-customer-key-base64.`,
 			Name: "sse_customer_key_base64",
 			Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
 
-Alternatively you can provide --sse-customer-key.`,
+Alternatively you can provide --s3-sse-customer-key.`,
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "",
@@ -1446,10 +1446,15 @@ func fixupRequest(o *s3.Options, opt *Options) {
 	)
 
 	o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error {
-		if err := stack.Finalize.Insert(fixup, "Signing", middleware.Before); err != nil {
+		// Presigning replaces the "Signing" middleware before APIOptions run
+		signing := "Signing"
+		if _, ok := stack.Finalize.Get(signing); !ok {
+			signing = "PresignHTTPRequest"
+		}
+		if err := stack.Finalize.Insert(fixup, signing, middleware.Before); err != nil {
 			return err
 		}
-		if err := stack.Finalize.Insert(restore, "Signing", middleware.After); err != nil {
+		if err := stack.Finalize.Insert(restore, signing, middleware.After); err != nil {
 			return err
 		}
 		return nil
@@ -1905,6 +1910,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.Provider == "Rabata" {
 		f.features.Copy = nil
 	}
+	if opt.Provider == "Autonomys" {
+		// Versions are permanent, so Purge's version deletes are always refused.
+		f.features.Purge = nil
+		// There are no bucket endpoints: CreateBucket would be stored as an object.
+		f.opt.NoCheckBucket = true
+	}
 	if opt.Provider == "TencentCOS" && strings.Contains(opt.Endpoint, "cos.accelerate.myqcloud.com") {
 		// Global Acceleration endpoint does not support bucket creation.
 		f.opt.NoCheckBucket = true
@@ -2972,8 +2983,9 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 // Returns an error if it isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	bucket, directory := f.split(dir)
-	// Remove directory marker file
-	if f.opt.DirectoryMarkers && bucket != "" && dir != "" {
+	// Remove directory marker file, including the one of the root
+	// when it is a directory in the bucket
+	if f.opt.DirectoryMarkers && bucket != "" && directory != "" {
 		o := &Object{
 			fs:     f,
 			remote: dir + "/",

backend/s3/s3_internal_test.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"path"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -891,6 +892,31 @@ func (f *Fs) InternalTestObjectLock(t *testing.T) {
 	})
 }
 
+// InternalTestRmdirRoot checks the directory marker of the root of a
+// remote is removed by Rmdir.
+func (f *Fs) InternalTestRmdirRoot(t *testing.T) {
+	if !f.opt.DirectoryMarkers {
+		t.Skip("needs --s3-directory-markers")
+	}
+	ctx := context.Background()
+	dir := "rmdir-root-" + random.String(8)
+	hasDir := func() bool {
+		entries, err := f.List(ctx, "")
+		require.NoError(t, err)
+		return slices.ContainsFunc(entries, func(entry fs.DirEntry) bool {
+			_, isDir := entry.(fs.Directory)
+			return isDir && entry.Remote() == dir
+		})
+	}
+
+	require.NoError(t, f.Mkdir(ctx, dir))
+	require.True(t, hasDir(), "directory marker not made")
+	fDir, err := cache.Get(ctx, bucket.Join(fs.ConfigStringFull(f), dir))
+	require.NoError(t, err)
+	require.NoError(t, fDir.Rmdir(ctx, ""))
+	assert.False(t, hasDir(), "directory marker not removed")
+}
+
 func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Metadata", f.InternalTestMetadata)
 	t.Run("NoHead", f.InternalTestNoHead)
@@ -898,6 +924,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("HasChildren", f.InternalTestHasChildren)
 	t.Run("Versions", f.InternalTestVersions)
 	t.Run("ObjectLock", f.InternalTestObjectLock)
+	t.Run("RmdirRoot", f.InternalTestRmdirRoot)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/s3/s3_link_test.go

@@ -225,3 +225,31 @@ func TestPublicLink(t *testing.T) {
 	_, err = f.PublicLink(context.Background(), "missing", fs.DurationOff, false)
 	assert.ErrorIs(t, err, fs.ErrorObjectNotFound)
 }
+
+func TestPublicLinkRequestFixups(t *testing.T) {
+	for _, tt := range []struct {
+		name               string
+		useXID             bool
+		signAcceptEncoding bool
+	}{
+		{"no x-id", false, true},
+		{"unsigned Accept-Encoding", true, false},
+		{"no x-id and unsigned Accept-Encoding", false, false},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			f := newLinkTestFs(t)
+			f.opt.UseXID = fs.Tristate{Value: tt.useXID, Valid: true}
+			f.opt.SignAcceptEncoding = fs.Tristate{Value: tt.signAcceptEncoding, Valid: true}
+			c, _, err := s3Connection(f.ctx, &f.opt, getClient(f.ctx, &f.opt))
+			require.NoError(t, err)
+			f.c = c
+			link, err := f.PublicLink(context.Background(), "file", fs.Duration(time.Hour), false)
+			require.NoError(t, err)
+			u, err := url.Parse(link)
+			require.NoError(t, err)
+			assert.Equal(t, tt.useXID, u.Query().Has("x-id"))
+			assert.Equal(t, "host", u.Query().Get("X-Amz-SignedHeaders"))
+			assert.Equal(t, u.Query().Get("X-Amz-Signature"), linkSignature(t, u))
+		})
+	}
+}

backend/seafile/seafile.go

@@ -30,6 +30,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/text/unicode/norm"
 )
 
 const (
@@ -205,7 +206,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:           *opt,
 		endpoint:      u,
 		endpointURL:   u.String(),
-		srv:           rest.NewClient(fshttp.NewClient(ctx)).SetRoot(u.String()),
+		srv:           rest.NewClient(newClient(ctx)).SetRoot(u.String()),
 		pacer:         getPacer(ctx, opt.URL),
 	}
 	f.features = (&fs.Features{
@@ -369,10 +370,10 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 
 		// Create rest client for getAuthorizationToken
 		url := u.String()
-		if !strings.HasPrefix(url, "/") {
+		if !strings.HasSuffix(url, "/") {
 			url += "/"
 		}
-		srv := rest.NewClient(fshttp.NewClient(ctx)).SetRoot(url)
+		srv := rest.NewClient(newClient(ctx)).SetRoot(url)
 
 		// We loop asking for a 2FA code
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
@@ -400,6 +401,17 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 	return nil, fmt.Errorf("unknown state %q", config.State)
 }
 
+// newClient returns the HTTP client used to talk to the Seafile API.
+//
+// It refuses redirects which downgrade HTTPS to plaintext HTTP as
+// following one would send the authorization token, or the password
+// during configuration, in cleartext.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.RefuseHTTPSDowngradeRedirectFn
+	return client
+}
+
 // Shutdown the Fs
 func (f *Fs) Shutdown(ctx context.Context) error {
 	if f.renew == nil {
@@ -1169,7 +1181,11 @@ func (f *Fs) mkLibrary(ctx context.Context, libraryName, password string) error
 	return nil
 }
 
-// splitPath returns the library name and the full path inside the library
+// splitPath returns the library name and the full path inside the library.
+//
+// The path is returned in NFC: Seafile stores names in NFC whatever form it
+// is given but looks them up byte for byte. Library names are stored verbatim
+// so they are left untouched.
 func (f *Fs) splitPath(dir string) (library, folder string) {
 	library = f.libraryName
 	folder = dir
@@ -1180,6 +1196,7 @@ func (f *Fs) splitPath(dir string) (library, folder string) {
 		// Adds the root folder to the path to get a full path
 		folder = path.Join(f.rootDirectory, folder)
 	}
+	folder = norm.NFC.String(folder)
 	return
 }
 

backend/seafile/seafile_internal_test.go

@@ -2,14 +2,25 @@ package seafile
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
 	"path"
+	"strings"
+	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/asyncreader"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/unicode/norm"
 )
 
 type pathData struct {
@@ -101,6 +112,28 @@ func TestSplitPath(t *testing.T) {
 			expectedLibrary: "Library",
 			expectedPath:    path.Join("root", "path", "subpath", "to", "file"),
 		},
+		// paths inside a library are returned in NFC, library names are kept as they are
+		{
+			configLibrary:   "Library",
+			configRoot:      "",
+			argumentPath:    norm.NFD.String("path/to/équivalent.txt"),
+			expectedLibrary: "Library",
+			expectedPath:    norm.NFC.String("path/to/équivalent.txt"),
+		},
+		{
+			configLibrary:   "Library",
+			configRoot:      norm.NFD.String("répertoire"),
+			argumentPath:    norm.NFD.String("équivalent.txt"),
+			expectedLibrary: "Library",
+			expectedPath:    norm.NFC.String("répertoire/équivalent.txt"),
+		},
+		{
+			configLibrary:   "",
+			configRoot:      "",
+			argumentPath:    norm.NFD.String("Bibliothèque/équivalent.txt"),
+			expectedLibrary: norm.NFD.String("Bibliothèque"),
+			expectedPath:    norm.NFC.String("équivalent.txt"),
+		},
 	}
 	for _, test := range testData {
 		fs := &Fs{
@@ -226,3 +259,138 @@ func Test2FAStateMachine(t *testing.T) {
 		})
 	}
 }
+
+// Test that a failed upload stops reading the source so it can be closed
+func TestUploadErrorReleasesSource(t *testing.T) {
+	ctx := context.Background()
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Reply without reading the body, like a server rejecting the upload early
+		http.Error(w, "rejected", http.StatusBadRequest)
+	}))
+	defer server.Close()
+
+	f := &Fs{
+		srv:   rest.NewClient(server.Client()),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault()),
+	}
+	// An AsyncReader holds its lock while writing to the upload
+	in, err := asyncreader.New(ctx, io.NopCloser(readers.NewPatternReader(64<<20)), 4)
+	require.NoError(t, err)
+
+	_, err = f.upload(ctx, in, server.URL+"/upload", "/file.bin")
+	require.Error(t, err)
+
+	closed := make(chan struct{})
+	go func() {
+		_ = in.Close()
+		close(closed)
+	}()
+	select {
+	case <-closed:
+	case <-time.After(10 * time.Second):
+		t.Fatal("source still in use after upload returned")
+	}
+}
+
+// testPassword is the password the downgrade tests look for on the
+// plaintext hop. It is distinctive so that it can't match the JSON
+// field names of the authentication request.
+const testPassword = "s3cret-pass-4242"
+
+// Test2FAGetToken checks the 2FA config flow posts the credentials to
+// the auth token endpoint without doubling the slash when the
+// configured URL ends in one, and saves the token it gets back.
+func Test2FAGetToken(t *testing.T) {
+	var gotPath string
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotPath = r.URL.Path
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"token":"token123"}`))
+	}))
+	defer srv.Close()
+
+	m := configmap.Simple{"url": srv.URL + "/", "2fa": "true", "user": "username", "pass": obscure.MustObscure(testPassword)}
+	out, err := Config(context.Background(), "test", m, fs.ConfigIn{State: "2fa_do", Result: "123456"})
+	require.NoError(t, err)
+	assert.Nil(t, out)
+	assert.Equal(t, "/api2/auth-token/", gotPath)
+	assert.Equal(t, "token123", m["auth_token"])
+	assert.Equal(t, "", m["pass"])
+}
+
+// newDowngradeServers returns an HTTPS server which answers the server
+// info request itself and redirects every other request to a plaintext
+// HTTP server on the same machine. The returned flags record whether
+// the plaintext server ever received an Authorization header or
+// testPassword in a request body.
+func newDowngradeServers(t *testing.T) (tlsSrv *httptest.Server, sawAuth, sawPassword *atomic.Bool) {
+	t.Helper()
+	sawAuth, sawPassword = new(atomic.Bool), new(atomic.Bool)
+
+	plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != "" {
+			sawAuth.Store(true)
+		}
+		body, _ := io.ReadAll(r.Body)
+		if strings.Contains(string(body), testPassword) {
+			sawPassword.Store(true)
+		}
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte("[]"))
+	}))
+	t.Cleanup(plain.Close)
+
+	tlsSrv = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/api2/server-info/" {
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"version":"9.0.0"}`))
+			return
+		}
+		http.Redirect(w, r, plain.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	t.Cleanup(tlsSrv.Close)
+
+	return tlsSrv, sawAuth, sawPassword
+}
+
+// TestRefuseHTTPSDowngradeRedirect checks that neither the
+// authorization token nor the password is sent over plaintext HTTP
+// when the server redirects an HTTPS request to an http:// URL.
+func TestRefuseHTTPSDowngradeRedirect(t *testing.T) {
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.InsecureSkipVerify = true // trust the httptest certificate
+
+	// Every API call made through the Fs carries the token.
+	t.Run("Token", func(t *testing.T) {
+		tlsSrv, sawAuth, _ := newDowngradeServers(t)
+		m := configmap.Simple{"url": tlsSrv.URL, "user": "username", "auth_token": "secret"}
+		fsys, err := NewFs(ctx, "test", "", m)
+		require.NoError(t, err)
+		f := fsys.(*Fs)
+		_, err = f.getLibraries(ctx)
+		assert.False(t, sawAuth.Load(), "authorization token was sent over plaintext HTTP")
+		assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	// Without a token NewFs posts the user's password to get one.
+	t.Run("Login", func(t *testing.T) {
+		tlsSrv, _, sawPassword := newDowngradeServers(t)
+		m := configmap.Simple{"url": tlsSrv.URL, "user": "username", "pass": obscure.MustObscure(testPassword)}
+		_, err := NewFs(ctx, "test", "", m)
+		assert.False(t, sawPassword.Load(), "password was sent over plaintext HTTP")
+		assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	})
+
+	// The 2FA config flow posts the user's password to get a token.
+	t.Run("Password", func(t *testing.T) {
+		tlsSrv, _, sawPassword := newDowngradeServers(t)
+		m := configmap.Simple{"url": tlsSrv.URL, "2fa": "true", "user": "username", "pass": obscure.MustObscure(testPassword)}
+		out, err := Config(ctx, "test", m, fs.ConfigIn{State: "2fa_do", Result: "123456"})
+		require.NoError(t, err)
+		require.NotNil(t, out)
+		assert.False(t, sawPassword.Load(), "password was sent over plaintext HTTP")
+		assert.Equal(t, "2fa_error", out.State)
+		require.NotNil(t, out.Option)
+		assert.Contains(t, out.Option.Help, rest.ErrHTTPSDowngrade.Error())
+	})
+}

backend/seafile/webapi.go

@@ -693,6 +693,8 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
 	if err != nil {
 		return nil, fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading in if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	opts := rest.Opts{
 		Method:      "POST",

backend/sftp/sftp.go

@@ -250,7 +250,7 @@ E.g. the second example above should be rewritten as:
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			Default:  encoder.Display,
+			Default:  encoder.Base,
 		}, {
 			Name:     "set_modtime",
 			Default:  true,
@@ -1883,8 +1883,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // mkParentDir makes the parent of remote if necessary and any
 // directories above that
 func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
-	parent := path.Dir(remote)
-	return f.mkdir(ctx, f.remotePath(parent))
+	return f.mkdir(ctx, path.Dir(f.remotePath(remote)))
 }
 
 // mkdir makes the directory and parents using native paths
@@ -2459,7 +2458,14 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		return "", hash.ErrUnsupported
 	}
 
-	shellPathArg, err := o.fs.quoteOrEscapeShellPath(o.shellPath())
+	shellPath := o.shellPath()
+	// Some restricted shells, such as rsync.net's, end the command at a
+	// newline even when it is quoted, which would hash a different file.
+	if o.fs.shellType == "unix" && strings.Contains(shellPath, "\n") {
+		fs.Debugf(o, "Can't calculate %v hash of a name containing a newline", r)
+		return "", nil
+	}
+	shellPathArg, err := o.fs.quoteOrEscapeShellPath(shellPath)
 	if err != nil {
 		return "", fmt.Errorf("failed to calculate %v hash: %w", r, err)
 	}
@@ -2547,6 +2553,10 @@ func (f *Fs) remoteShellPath(remote string) string {
 		if f.opt.PathOverride[0] == '@' {
 			shellPath = path.Join(strings.TrimPrefix(f.opt.PathOverride, "@"), f.absRoot, encodedRemote)
 		}
+		// A relative path starting with "-" would be parsed as an option by the hash command
+		if strings.HasPrefix(shellPath, "-") {
+			shellPath = "./" + shellPath
+		}
 		fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath)
 		return shellPath
 	}

backend/sftp/sftp_internal_test.go

@@ -18,6 +18,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fstest/fstests"
@@ -146,6 +147,31 @@ func TestRemoteShellPathEncodesPathOverrideNames(t *testing.T) {
 	assert.Equal(t, "/volume/srv/root/dir/file\uFF1Aname", f.remoteShellPath("dir/file:name"))
 }
 
+func TestRemoteShellPathRelativePathOverrideLeadingDash(t *testing.T) {
+	f := &Fs{
+		absRoot: "/",
+		opt: Options{
+			Enc:          encoder.Display,
+			PathOverride: "@.",
+		},
+	}
+
+	assert.Equal(t, "./-file", f.remoteShellPath("-file"))
+	assert.Equal(t, "dir/-file", f.remoteShellPath("dir/-file"))
+	assert.Equal(t, "file", f.remoteShellPath("file"))
+
+	f.opt.PathOverride = "."
+	assert.Equal(t, "./-file", f.remoteShellPath("-file"))
+}
+
+func TestDefaultEncodingStandardizesControlCharacters(t *testing.T) {
+	ri, err := fs.Find("sftp")
+	require.NoError(t, err)
+	enc := ri.Options.Get(config.ConfigEncoding).Default.(encoder.MultiEncoder)
+	assert.Equal(t, "a␊b", enc.ToStandardName("a\nb"))
+	assert.Equal(t, "a\nb", enc.FromStandardName("a␊b"))
+}
+
 func TestParseHash(t *testing.T) {
 	for i, test := range []struct {
 		sshOutput, checksum string

backend/sftp/ssh_external.go

@@ -11,6 +11,7 @@ import (
 	"slices"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/rclone/rclone/fs"
@@ -80,8 +81,9 @@ type sshSessionExternal struct {
 	cancel      func()
 	startCalled bool
 	runningSFTP bool
-	waitOnce    sync.Once // ensure Wait() is only called once
-	waitErr     error     // result of the Wait() call
+	waitOnce    sync.Once   // ensure Wait() is only called once
+	waitErr     error       // result of the Wait() call
+	hasExited   atomic.Bool // set once Wait() has returned
 }
 
 func (f *Fs) newSSHSessionExternal() *sshSessionExternal {
@@ -176,7 +178,8 @@ func (s *sshSessionExternal) StdoutPipe() (io.Reader, error) {
 
 // Return whether the command has finished or not
 func (s *sshSessionExternal) exited() bool {
-	return s.cmd.ProcessState != nil
+	// Don't read s.cmd.ProcessState as Wait() writes it from another goroutine
+	return s.hasExited.Load()
 }
 
 // Wait for the command to exit
@@ -185,6 +188,7 @@ func (s *sshSessionExternal) Wait() error {
 	// This is safe even if Wait() is called from multiple goroutines.
 	s.waitOnce.Do(func() {
 		s.waitErr = s.cmd.Wait()
+		s.hasExited.Store(true)
 		if s.waitErr == nil {
 			fs.Debugf(s.f, "ssh external: command exited OK")
 		} else {

backend/sftp/ssh_external_test.go

@@ -82,3 +82,21 @@ func TestSSHExternalCloseMultipleCalls(t *testing.T) {
 	// Verify the process has exited
 	assert.True(t, session.exited())
 }
+
+// TestSSHExternalExitedWhileWaiting verifies that exited() can be called
+// while another goroutine is waiting for the process. Run with -race.
+func TestSSHExternalExitedWhileWaiting(t *testing.T) {
+	f := &Fs{
+		opt: Options{
+			SSH: fs.SpaceSepList{"echo", "test"},
+		},
+	}
+	session := f.newSSHSessionExternal()
+	assert.NoError(t, session.Start("exit 0"))
+
+	go func() {
+		_ = session.Wait()
+	}()
+
+	assert.Eventually(t, session.exited, 10*time.Second, time.Millisecond)
+}

backend/swift/swift.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"io"
 	"maps"
+	"net/http"
 	"path"
 	"regexp"
 	"slices"
@@ -33,6 +34,7 @@ import (
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
 )
 
 // Constants
@@ -193,6 +195,18 @@ func init() {
 			}, {
 				Value: "https://authenticate.ain.net",
 				Help:  "Blomp Cloud Storage",
+			}, {
+				Value: "https://swiss-backup01.infomaniak.com/identity/v3",
+				Help:  "Infomaniak Swiss Backup Location 01",
+			}, {
+				Value: "https://swiss-backup02.infomaniak.com/identity/v3",
+				Help:  "Infomaniak Swiss Backup Location 02",
+			}, {
+				Value: "https://swiss-backup03.infomaniak.com/identity/v3",
+				Help:  "Infomaniak Swiss Backup Location 03",
+			}, {
+				Value: "https://swiss-backup04.infomaniak.com/identity/v3",
+				Help:  "Infomaniak Swiss Backup Location 04",
 			}},
 		}, {
 			Name:      "user_id",
@@ -470,6 +484,90 @@ func (o *Object) split() (container, containerPath string) {
 	return o.fs.split(o.remote)
 }
 
+// authHeaders are the request headers which carry the Swift
+// credentials: the token sent on every storage request and the user
+// name and API key sent on a v1 auth request. They must not be
+// forwarded when a redirect leaves the host of the original request.
+var authHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}
+
+// errRedirectResendsBody is returned when following a redirect which
+// leaves the host of the original request would resend that request's
+// body, such as the password in a v2 or v3 auth request, to the new
+// host.
+var errRedirectResendsBody = errors.New("refusing to follow redirect which leaves the original host: would resend the request body")
+
+// transport is the http.RoundTripper for the swift connection.
+//
+// The swift library makes its own http.Client with the default
+// redirect policy and does not let the caller replace it, so the
+// redirect policy is applied here at the transport level instead. A
+// redirect which downgrades HTTPS to plaintext HTTP is refused with
+// rest.ErrHTTPSDowngrade. Once the redirect chain has left the host
+// and port of the original request the auth headers are removed from
+// every further hop, and a hop which would resend the request body is
+// refused with errRedirectResendsBody.
+type transport struct {
+	*fshttp.Transport
+}
+
+// newTransport returns the transport for the swift connection
+func newTransport(ctx context.Context) *transport {
+	return &transport{Transport: fshttp.NewTransport(ctx)}
+}
+
+// RoundTrip implements http.RoundTripper
+func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	// net/http fills in req.Response on the requests it makes to
+	// follow a redirect, so a request without one is not a redirect.
+	if req.Response != nil {
+		origin, leftHost := redirectOrigin(req)
+		if origin.URL.Scheme == "https" && req.URL.Scheme == "http" {
+			if req.Body != nil {
+				_ = req.Body.Close()
+			}
+			return nil, fmt.Errorf("redirect to %s: %w", req.URL.Redacted(), rest.ErrHTTPSDowngrade)
+		}
+		if leftHost {
+			// net/http resends the body on a 307 or 308 if it
+			// can be rewound, which for an auth request means
+			// the password, so those redirects stop here.
+			if req.Body != nil && req.Body != http.NoBody {
+				_ = req.Body.Close()
+				return nil, fmt.Errorf("redirect to %s leaves %s: %w", req.URL.Redacted(), origin.URL.Redacted(), errRedirectResendsBody)
+			}
+			for _, header := range authHeaders {
+				if req.Header.Get(header) == "" {
+					continue
+				}
+				fs.Debugf(nil, "redirect to %s leaves %s: not sending %s", req.URL.Redacted(), origin.URL.Redacted(), header)
+				req.Header.Del(header)
+			}
+		}
+	}
+	return t.Transport.RoundTrip(req)
+}
+
+// redirectOrigin walks back along the chain of redirects which led to
+// req and returns the original request and whether any hop, including
+// req, was to a different host or port from it.
+//
+// net/http copies the headers afresh from the original request for
+// every hop, so once the chain has visited another host the headers
+// must be stripped from every subsequent hop, even one back to the
+// original host, as the other host chose the URL.
+func redirectOrigin(req *http.Request) (origin *http.Request, leftHost bool) {
+	origin = req
+	for origin.Response != nil && origin.Response.Request != nil {
+		origin = origin.Response.Request
+	}
+	for hop := req; hop != origin; hop = hop.Response.Request {
+		if !rest.SameHost(hop.URL, origin.URL) {
+			leftHost = true
+		}
+	}
+	return origin, leftHost
+}
+
 // swiftConnection makes a connection to swift
 func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Connection, error) {
 	ci := fs.GetConfig(ctx)
@@ -493,7 +591,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
 		EndpointType:                swift.EndpointType(opt.EndpointType),
 		ConnectTimeout:              time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
 		Timeout:                     time.Duration(10 * ci.Timeout),        // Use the timeouts in the transport
-		Transport:                   fshttp.NewTransport(ctx),
+		Transport:                   newTransport(ctx),
 		FetchUntilEmptyPage:         opt.FetchUntilEmptyPage,
 		PartialPageFetchThreshold:   opt.PartialPageFetchThreshold,
 	}

backend/swift/swift_internal_test.go

@@ -2,12 +2,22 @@ package swift
 
 import (
 	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/ncw/swift/v2"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/rest"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestInternalUrlEncode(t *testing.T) {
@@ -66,3 +76,201 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
 	assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
 
 }
+
+// testToken is the auth token the redirect tests look for. It is
+// distinctive so that it can't match anything else in a request.
+const testToken = "AUTH_tk_secret_4242"
+
+// authTestHeaders is a deliberately literal copy of authHeaders so
+// that a header dropped from the production list shows up as a leak
+// in the cross host tests rather than silently losing coverage.
+var authTestHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}
+
+// listHandler answers any request with an empty JSON listing.
+func listHandler(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = w.Write([]byte("[]"))
+}
+
+// newTestFs makes an Fs via NewFs for a container on storageURL,
+// using a pre-issued token so that no auth request is made.
+func newTestFs(ctx context.Context, t *testing.T, storageURL string) *Fs {
+	m := configmap.Simple{
+		"storage_url": storageURL + "/v1/AUTH_test",
+		"auth_token":  testToken,
+		"chunk_size":  "5Gi",
+	}
+	f, err := NewFs(ctx, "swift-test", "container", m)
+	require.NoError(t, err)
+	return f.(*Fs)
+}
+
+// assertAuthHeaders checks that every auth header is absent from r
+// (!want) or that the ones named in present are there (want).
+func assertAuthHeaders(t *testing.T, r *http.Request, want bool, present ...string) {
+	if want {
+		for _, header := range present {
+			assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
+		}
+		return
+	}
+	for _, header := range authTestHeaders {
+		assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+	}
+}
+
+// TestRedirectStripsAuthTokenOnCrossHost checks the token is sent to
+// the storage host but not to another host it redirects to, nor to
+// the storage host again once the chain has been elsewhere.
+func TestRedirectStripsAuthTokenOnCrossHost(t *testing.T) {
+	var originHits, otherHits, backHits atomic.Int32
+	var origin *httptest.Server
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAuthHeaders(t, r, false)
+		http.Redirect(w, r, origin.URL+"/back", http.StatusFound)
+	}))
+	defer other.Close()
+
+	origin = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path == "/back" {
+			backHits.Add(1)
+			assertAuthHeaders(t, r, false)
+			listHandler(w, r)
+			return
+		}
+		originHits.Add(1)
+		assertAuthHeaders(t, r, true, "X-Auth-Token")
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
+	}))
+	defer origin.Close()
+
+	ctx := context.Background()
+	f := newTestFs(ctx, t, origin.URL)
+	_, err := f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), originHits.Load())
+	assert.Equal(t, int32(1), otherHits.Load())
+	assert.Equal(t, int32(1), backHits.Load())
+}
+
+// TestRedirectKeepsAuthTokenOnSameHost checks the token is still sent
+// after a redirect which stays on the storage host.
+func TestRedirectKeepsAuthTokenOnSameHost(t *testing.T) {
+	var hits atomic.Int32
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		hits.Add(1)
+		assertAuthHeaders(t, r, true, "X-Auth-Token")
+		if !strings.HasSuffix(r.URL.Path, "/moved") {
+			http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
+			return
+		}
+		listHandler(w, r)
+	}))
+	defer origin.Close()
+
+	ctx := context.Background()
+	f := newTestFs(ctx, t, origin.URL)
+	_, err := f.List(ctx, "")
+	require.NoError(t, err)
+	assert.Equal(t, int32(2), hits.Load())
+}
+
+// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
+// plaintext HTTP is not followed, so the token never goes over the
+// network in the clear.
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+	var targetHits atomic.Int32
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		targetHits.Add(1)
+		listHandler(w, r)
+	}))
+	defer target.Close()
+
+	origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
+	}))
+	defer origin.Close()
+
+	// The transport is a process wide singleton so it has to be
+	// rebuilt to trust the httptest certificate, and rebuilt again
+	// afterwards so later tests don't inherit that.
+	fshttp.ResetTransport()
+	t.Cleanup(fshttp.ResetTransport)
+	ctx, ci := fs.AddConfig(context.Background())
+	ci.InsecureSkipVerify = true
+	f := newTestFs(ctx, t, origin.URL)
+	_, err := f.List(ctx, "")
+	require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+	assert.Equal(t, int32(0), targetHits.Load())
+}
+
+// TestAuthRedirectStripsCredentialsOnCrossHost checks the user name
+// and API key sent on a v1 auth request go to the auth host but not
+// to another host it redirects to.
+func TestAuthRedirectStripsCredentialsOnCrossHost(t *testing.T) {
+	var originHits, otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		assertAuthHeaders(t, r, false)
+		w.Header().Set("X-Storage-Url", "http://"+r.Host+"/v1/AUTH_test")
+		w.Header().Set("X-Auth-Token", testToken)
+	}))
+	defer other.Close()
+
+	origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		originHits.Add(1)
+		assertAuthHeaders(t, r, true, "X-Auth-Key", "X-Auth-User")
+		http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
+	}))
+	defer origin.Close()
+
+	m := configmap.Simple{
+		"auth":         origin.URL + "/v1.0",
+		"auth_version": "1",
+		"user":         "test-user",
+		"key":          "test-key",
+		"chunk_size":   "5Gi",
+	}
+	_, err := NewFs(context.Background(), "swift-test", "container", m)
+	require.NoError(t, err)
+	assert.Equal(t, int32(1), originHits.Load())
+	assert.Equal(t, int32(1), otherHits.Load())
+}
+
+// TestAuthRedirectRefusesToResendPassword checks a v2 or v3 auth
+// request, which carries the password in its body, is not resent to
+// another host the auth server redirects it to.
+func TestAuthRedirectRefusesToResendPassword(t *testing.T) {
+	for _, authVersion := range []string{"2", "3"} {
+		t.Run("v"+authVersion, func(t *testing.T) {
+			var originHits, otherHits atomic.Int32
+			other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				otherHits.Add(1)
+				body, _ := io.ReadAll(r.Body)
+				assert.NotContains(t, string(body), "test-key", "password was resent to another host")
+			}))
+			defer other.Close()
+
+			origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				originHits.Add(1)
+				body, _ := io.ReadAll(r.Body)
+				assert.Contains(t, string(body), "test-key")
+				http.Redirect(w, r, other.URL+r.URL.Path, http.StatusTemporaryRedirect)
+			}))
+			defer origin.Close()
+
+			m := configmap.Simple{
+				"auth":         origin.URL + "/v" + authVersion,
+				"auth_version": authVersion,
+				"user":         "test-user",
+				"key":          "test-key",
+				"chunk_size":   "5Gi",
+			}
+			_, err := NewFs(context.Background(), "swift-test", "container", m)
+			require.ErrorIs(t, err, errRedirectResendsBody)
+			assert.Equal(t, int32(1), originHits.Load())
+			assert.Equal(t, int32(0), otherHits.Load())
+		})
+	}
+}

backend/ulozto/ulozto.go

@@ -113,6 +113,22 @@ type Fs struct {
 	pacer    *fs.Pacer          // pacer for API calls
 }
 
+// authHeaders are the request headers carrying the app and session
+// tokens which must not be forwarded when a redirect leaves the
+// original host.
+var authHeaders = []string{"X-Auth-Token", "X-User-Token"}
+
+// newClient makes the http client used for all uloz.to requests.
+//
+// Downloads carry the auth headers to whatever URL the API hands out,
+// so the client strips them on cross host redirects and refuses an
+// HTTPS to HTTP downgrade.
+func newClient(ctx context.Context) *http.Client {
+	client := fshttp.NewClient(ctx)
+	client.CheckRedirect = rest.StripHeadersOnCrossHostRedirectFn(authHeaders...)
+	return client
+}
+
 // NewFs constructs a Fs from the path, container:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -125,7 +141,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	// Strip leading and trailing slashes, see https://github.com/rclone/rclone/issues/7796 for details.
 	root = strings.Trim(root, "/")
 
-	client := fshttp.NewClient(ctx)
+	client := newClient(ctx)
 
 	f := &Fs{
 		name:  name,

backend/ulozto/ulozto_internal_test.go

@@ -0,0 +1,95 @@
+package ulozto
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync/atomic"
+	"testing"
+
+	"github.com/rclone/rclone/backend/ulozto/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// authTestHeaders is a deliberately literal copy of authHeaders so
+// that a header dropped from the production list fails
+// TestAuthHeaders rather than silently losing test coverage.
+var authTestHeaders = map[string]string{
+	"X-Auth-Token": "secret-app-token",
+	"X-User-Token": "secret-user-token",
+}
+
+func TestAuthHeaders(t *testing.T) {
+	assert.ElementsMatch(t, []string{"X-Auth-Token", "X-User-Token"}, authHeaders)
+}
+
+// newTestFs makes an Fs talking to apiURL with the auth headers set,
+// without going through NewFs which would authenticate.
+func newTestFs(ctx context.Context, apiURL string) *Fs {
+	client := newClient(ctx)
+	f := &Fs{
+		name:  "ulozto-test",
+		cdn:   rest.NewClient(client),
+		rest:  rest.NewClient(client).SetRoot(apiURL),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+	}
+	f.rest.SetErrorHandler(errorHandler)
+	for header, value := range authTestHeaders {
+		f.rest.SetHeader(header, value)
+	}
+	return f
+}
+
+// TestOpenStripsAuthHeadersOnCrossHostRedirect checks that a download
+// sends the auth headers to the host the API named but not to a host
+// that download redirects to.
+func TestOpenStripsAuthHeadersOnCrossHostRedirect(t *testing.T) {
+	ctx := context.Background()
+
+	var otherHits atomic.Int32
+	other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		otherHits.Add(1)
+		for header := range authTestHeaders {
+			assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+		}
+		_, _ = w.Write([]byte("payload"))
+	}))
+	defer other.Close()
+
+	var cdnHits atomic.Int32
+	cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cdnHits.Add(1)
+		assert.Equal(t, "/file?hash=h", r.URL.RequestURI())
+		for header, value := range authTestHeaders {
+			assert.Equal(t, value, r.Header.Get(header), "%s should be sent to the download host", header)
+		}
+		http.Redirect(w, r, other.URL+"/payload", http.StatusTemporaryRedirect)
+	}))
+	defer cdn.Close()
+
+	apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "/v5/file/download-link/vipdata", r.URL.Path)
+		w.Header().Set("Content-Type", "application/json")
+		require.NoError(t, json.NewEncoder(w).Encode(api.GetDownloadLinkResponse{
+			Link: cdn.URL + "/file",
+			Hash: "h",
+		}))
+	}))
+	defer apiServer.Close()
+
+	o := &Object{fs: newTestFs(ctx, apiServer.URL), slug: "slug"}
+	rc, err := o.Open(ctx)
+	require.NoError(t, err)
+	body, err := io.ReadAll(rc)
+	require.NoError(t, err)
+	require.NoError(t, rc.Close())
+	assert.Equal(t, "payload", string(body))
+	assert.Equal(t, int32(1), cdnHits.Load())
+	assert.Equal(t, int32(1), otherHits.Load())
+}

backend/webdav/tus-uploader.go

@@ -20,16 +20,9 @@ type Uploader struct {
 	upload              *Upload
 	offset              int64
 	aborted             bool
-	uploadSubs          []chan Upload
-	notifyChan          chan bool
 	overridePatchMethod bool
 }
 
-// NotifyUploadProgress subscribes to progress updates.
-func (u *Uploader) NotifyUploadProgress(c chan Upload) {
-	u.uploadSubs = append(u.uploadSubs, c)
-}
-
 func (f *Fs) shouldRetryChunk(ctx context.Context, resp *http.Response, err error, newOff *int64) (bool, error) {
 	if resp == nil {
 		return true, err
@@ -156,36 +149,15 @@ func (u *Uploader) UploadChunk(ctx context.Context, cnt int, options ...fs.OpenO
 
 	u.upload.updateProgress(u.offset)
 
-	u.notifyChan <- true
-
 	return nil
 }
 
-// Waits for a signal to broadcast to all subscribers
-func (u *Uploader) broadcastProgress() {
-	for range u.notifyChan {
-		for _, c := range u.uploadSubs {
-			c <- *u.upload
-		}
-	}
-}
-
 // NewUploader creates a new Uploader.
 func NewUploader(f *Fs, url string, upload *Upload, offset int64) *Uploader {
-	notifyChan := make(chan bool)
-
-	uploader := &Uploader{
-		f,
-		url,
-		upload,
-		offset,
-		false,
-		nil,
-		notifyChan,
-		false,
+	return &Uploader{
+		fs:     f,
+		url:    url,
+		upload: upload,
+		offset: offset,
 	}
-
-	go uploader.broadcastProgress()
-
-	return uploader
 }

backend/webdav/tus-uploader_test.go

@@ -0,0 +1,113 @@
+package webdav
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+	"weak"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newTusTestUploader(t *testing.T, server *httptest.Server, contents string) *Uploader {
+	t.Helper()
+	f := &Fs{
+		opt:   Options{ChunkSize: 4},
+		srv:   rest.NewClient(server.Client()),
+		pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(0))),
+	}
+	// Wrap the reader so NewUpload buffers it, as it does for transfer readers.
+	upload := NewUpload(io.LimitReader(strings.NewReader(contents), int64(len(contents))), int64(len(contents)), nil, "")
+	require.NotNil(t, upload)
+	return NewUploader(f, server.URL, upload, 0)
+}
+
+func tusTestHandler(t *testing.T, contents string, fail *atomic.Bool) http.HandlerFunc {
+	t.Helper()
+	return func(w http.ResponseWriter, r *http.Request) {
+		assert.Equal(t, "PATCH", r.Method)
+		assert.Equal(t, "1.0.0", r.Header.Get("Tus-Resumable"))
+		if fail != nil && fail.Load() {
+			w.WriteHeader(http.StatusConflict)
+			return
+		}
+		offset, err := strconv.Atoi(r.Header.Get("Upload-Offset"))
+		if !assert.NoError(t, err) {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		body, err := io.ReadAll(r.Body)
+		assert.NoError(t, err)
+		end := min(offset+4, len(contents))
+		assert.Equal(t, contents[offset:end], string(body))
+		w.Header().Set("Upload-Offset", strconv.Itoa(end))
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+func TestTusUploadReleasesBuffer(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		contents string
+		fail     bool
+	}{
+		{name: "Success", contents: "abcdefghij"},
+		{name: "Error", contents: "abcdefghij", fail: true},
+		{name: "Empty"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			var fail atomic.Bool
+			fail.Store(tc.fail)
+			server := httptest.NewServer(tusTestHandler(t, tc.contents, &fail))
+			defer server.Close()
+
+			released := func() weak.Pointer[Upload] {
+				uploader := newTusTestUploader(t, server, tc.contents)
+				err := uploader.Upload(context.Background())
+				if tc.fail {
+					require.ErrorIs(t, err, ErrOffsetMismatch)
+				} else {
+					require.NoError(t, err)
+					assert.True(t, uploader.upload.Finished())
+					require.NoError(t, uploader.Upload(context.Background()))
+				}
+				ref := weak.Make(uploader.upload)
+				runtime.KeepAlive(uploader)
+				return ref
+			}()
+
+			require.Eventually(t, func() bool {
+				runtime.GC()
+				return released.Value() == nil
+			}, 5*time.Second, 10*time.Millisecond, "completed upload still retains its buffered contents")
+		})
+	}
+}
+
+func TestTusUploadRetry(t *testing.T) {
+	const contents = "abcdefghij"
+	var fail atomic.Bool
+	fail.Store(true)
+	server := httptest.NewServer(tusTestHandler(t, contents, &fail))
+	defer server.Close()
+
+	uploader := newTusTestUploader(t, server, contents)
+	require.ErrorIs(t, uploader.Upload(context.Background()), ErrOffsetMismatch)
+	assert.Zero(t, uploader.upload.Offset())
+	fail.Store(false)
+	require.NoError(t, uploader.Upload(context.Background()))
+	assert.Equal(t, int64(len(contents)), uploader.upload.Offset())
+	assert.True(t, uploader.upload.Finished())
+	require.NoError(t, uploader.Upload(context.Background()))
+}

backend/webdav/webdav.go

@@ -1783,7 +1783,7 @@ func (o *Object) updateSimple(ctx context.Context, body io.Reader, getBody func(
 		GetBody:       getBody,
 		Body:          body,
 		NoResponse:    true,
-		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
+		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud-snap/nextcloud-snap/issues/365
 		ContentType:   contentType,
 		Options:       options,
 		ExtraHeaders:  extraHeaders,

backend/zoho/zoho.go

@@ -1277,6 +1277,8 @@ func (f *Fs) upload(ctx context.Context, name string, parent string, size int64,
 	if err != nil {
 		return nil, fmt.Errorf("failed to make multipart upload: %w", err)
 	}
+	// Stop the form writer reading in if the request fails before sending it all
+	defer func() { _ = formReader.Close() }()
 
 	contentLength := overhead + size
 	opts := rest.Opts{

bin/make_manual.py

@@ -66,6 +66,7 @@ docs = [
     "internetarchive.md",
     "internxt.md",
     "jottacloud.md",
+    "kdrive.md",
     "koofr.md",
     "linkbox.md",
     "mailru.md",

cmd/bisync/bisync_test.go

@@ -19,6 +19,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	mutex "sync" // renamed as "sync" already in use
 	"testing"
 	"time"
 	"unicode/utf8"
@@ -61,6 +62,12 @@ const (
 
 var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
 
+// modtimeProbes holds the SetModTime error for each remote already probed by checkPreReqs
+var (
+	modtimeProbesMu mutex.Mutex
+	modtimeProbes   = map[string]error{}
+)
+
 /* Useful Command Shortcuts */
 // go test ./cmd/bisync -remote local -race
 // go test ./cmd/bisync -remote local -golden
@@ -88,7 +95,7 @@ var logReplacements = []string{
 	// ignore SFTP host key messages
 	`^NOTICE: .*?No host key validation is being performed.*$`, dropMe,
 	// ignore dropbox info messages
-	`^NOTICE: too_many_(requests|write_operations)/\.*: Too many requests or write operations.*$`, dropMe,
+	`^NOTICE: .*Too many requests or write operations.*$`, dropMe,
 	`^NOTICE: .*?: Forced to upload files to set modification times on this backend.$`, dropMe,
 	`^INFO  : .*? Committing uploads - please wait...$`, dropMe,
 	`^INFO  : .*? Committing uploads - done$`, dropMe,
@@ -102,6 +109,9 @@ var logReplacements = []string{
 	`^INFO  : .*cache expired.*$`, dropMe,
 	// ignore "Implicitly create directory" messages (TestnStorage:)
 	`^INFO  : .*Implicitly create directory.*$`, dropMe,
+	// ignore bucket creation messages from S3 providers which report success
+	// when creating a bucket which already exists (TestS3Wasabi:)
+	`^INFO  : .*: Bucket ".*" created with ACL ".*"$`, dropMe,
 	// ignore differences in backend features
 	`^.*?"HashType1":.*?$`, dropMe,
 	`^.*?"HashType2":.*?$`, dropMe,
@@ -197,6 +207,7 @@ type bisyncTest struct {
 	testTitle   string
 	testCase    string
 	sessionName string
+	hooks       map[string][]string // scenario steps to run once when the named hook fires
 	// test dirs
 	testDir    string
 	dataDir    string
@@ -230,6 +241,7 @@ type bisyncTest struct {
 	TestFn          bisync.TestFunc
 	ignoreModtime   bool // ignore modtimes when comparing final listings, for backends without support
 	ignoreBlankHash bool // ignore blank hashes for backends where we allow them to be blank
+	ignoreChecksum  bool // a step of this test case used --ignore-checksum
 }
 
 var color = bisync.Color
@@ -435,6 +447,7 @@ func (b *bisyncTest) cleanupAll() {
 func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase string) {
 	b.t = t
 	b.testCase = testCase
+	b.ignoreChecksum = false
 	var err error
 
 	b.fs1, b.parent1, b.path1, b.canonPath1 = b.makeTempRemote(ctx, b.argRemote1, "path1")
@@ -805,6 +818,19 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
 	case "test-func":
 		b.TestFn = testFunc
 		return
+	case "hook":
+		// hook <when> <step> queues a scenario step to run once, when the next bisync reaches <when>
+		b.checkArgs(args, 2, 0)
+		when := args[1]
+		if when != "before-check" && when != "during-sync" {
+			return fmt.Errorf("unknown hook %q (want before-check or during-sync)", when)
+		}
+		step := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(strings.TrimPrefix(line, args[0])), when))
+		if b.hooks == nil {
+			b.hooks = map[string][]string{}
+		}
+		b.hooks[when] = append(b.hooks[when], step)
+		return nil
 	case "concurrent-func":
 		b.TestFn = func() {
 			src := filepath.Join(b.dataDir, "file7.txt")
@@ -956,6 +982,10 @@ func splitLine(line string) (args []string) {
 		b := []byte(whitespaceReplacer.Replace(s))
 		b = regexChar.ReplaceAllFunc(b, func(b []byte) []byte {
 			c, _ := strconv.ParseUint(string(b[5:7]), 16, 8)
+			if c <= 0x1f || c == 0x7f {
+				// control characters are spelled with their glyph in rclone names
+				return []byte(encoder.Standard.Encode(string(rune(c))))
+			}
 			return []byte{byte(c)}
 		})
 		args = append(args, string(b))
@@ -965,8 +995,8 @@ func splitLine(line string) (args []string) {
 
 var whitespaceReplacer = strings.NewReplacer(
 	"{spc}", " ",
-	"{tab}", "\t",
-	"{eol}", eol,
+	"{tab}", "␉",
+	"{eol}", "␊",
 )
 var regexChar = regexp.MustCompile(`\{chr:([0-9a-f]{2})\}`)
 
@@ -1039,7 +1069,7 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
 		b.ignoreModtime = true
 	}
 	// test if modtimes are writeable
-	testSetModtime := func(f fs.Fs) {
+	probeSetModtime := func(f fs.Fs) error {
 		ctx := accounting.WithStatsGroup(ctx, random.String(8)) // keep stats separate
 		in := bytes.NewBufferString("modtime_write_test")
 		objinfo := object.NewStaticObjectInfo("modtime_write_test", initDate, int64(len("modtime_write_test")), true, nil, nil)
@@ -1050,7 +1080,7 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
 		}
 		err = obj.SetModTime(ctx, initDate)
 		if err == fs.ErrorCantSetModTime {
-			b.t.Skip("skipping test as at least one remote does not support setting modtime")
+			return err
 		}
 		if err == fs.ErrorCantSetModTimeWithoutDelete { // transfers stats expected to differ on this backend
 			logReplacements = append(logReplacements, `^.*There was nothing to transfer.*$`, dropMe)
@@ -1060,34 +1090,52 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
 		if !f.Features().IsLocal {
 			time.Sleep(time.Second) // avoid GoogleCloudStorage Error 429 rateLimitExceeded
 		}
-		err = obj.Remove(ctx)
-		require.NoError(b.t, err)
+		require.NoError(b.t, obj.Remove(ctx))
+		return err
+	}
+	// probing on every bisync step would sleep for minutes on non-local remotes
+	testSetModtime := func(f fs.Fs) {
+		modtimeProbesMu.Lock()
+		defer modtimeProbesMu.Unlock()
+		err, probed := modtimeProbes[f.Name()]
+		if !probed {
+			err = probeSetModtime(f)
+			modtimeProbes[f.Name()] = err
+		}
+		if err == fs.ErrorCantSetModTime {
+			b.t.Skip("skipping test as at least one remote does not support setting modtime")
+		}
 	}
 	if b.testCase != "nomodtime" {
 		testSetModtime(b.fs1)
 		testSetModtime(b.fs2)
 	}
 
-	if b.testCase == "normalization" || b.testCase == "extended_char_paths" || b.testCase == "extended_filenames" {
+	if b.testCase == "normalization" || b.testCase == "extended_filenames" {
 		// test whether remote is capable of running test
 		const chars = "ě{chr:81}{chr:fe}{spc}áñhࢺ_測試Рускийěáñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö"
 		testfilename1 := splitLine(norm.NFD.String(norm.NFC.String(chars)))[0]
 		testfilename2 := splitLine(norm.NFC.String(norm.NFD.String(chars)))[0]
 		tempDir, err := cache.Get(ctx, b.tempDir)
 		require.NoError(b.t, err)
+		ctx := accounting.WithStatsGroup(ctx, random.String(8)) // keep stats separate
 		preTest := func(f fs.Fs, testfilename string) string {
+			// the path roots already hold the scenario's files, so probe in a fresh dir
+			dir := "pretest-" + strings.ToLower(random.String(8))
 			in := bytes.NewBufferString(testfilename)
-			objinfo := object.NewStaticObjectInfo(testfilename, initDate, int64(len(testfilename)), true, nil, nil)
+			objinfo := object.NewStaticObjectInfo(path.Join(dir, testfilename), initDate, int64(len(testfilename)), true, nil, nil)
 			obj, err := f.Put(ctx, in, objinfo)
 			if err != nil {
 				b.t.Skipf("Fs is incapable of running test, skipping: %s (expected: \n%s (%s) actual: \n%s (%v))\n (fs: %s) \n", b.testCase, testfilename, detectEncoding(testfilename), "upload failed", err, f)
 			}
-			entries, err := f.List(ctx, "")
-			assert.NoError(b.t, err)
-			if entries.Len() == 1 && entries[0].Remote() != testfilename {
-				diffStr, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: []string{testfilename}, B: []string{entries[0].Remote()}})
+			entries, err := f.List(ctx, dir)
+			require.NoError(b.t, err)
+			require.Equal(b.t, 1, entries.Len())
+			got := path.Base(entries[0].Remote())
+			if got != testfilename {
+				diffStr, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{A: []string{testfilename}, B: []string{got}})
 				// we can still deal with this as long as both remotes auto-convert the same way.
-				b.t.Logf("Warning: this remote seems to auto-convert special characters (testcase: %s) (expected: \n%s (%s) actual: \n%s (%s))\n (fs: %s) \n%v", b.testCase, testfilename, detectEncoding(testfilename), entries[0].Remote(), detectEncoding(entries[0].Remote()), f, diffStr)
+				b.t.Logf("Warning: this remote seems to auto-convert special characters (testcase: %s) (expected: \n%s (%s) actual: \n%s (%s))\n (fs: %s) \n%v", b.testCase, testfilename, detectEncoding(testfilename), got, detectEncoding(got), f, diffStr)
 			}
 			// test whether we can fix-case
 			ctxFixCase, ci := fs.AddConfig(ctx)
@@ -1104,7 +1152,12 @@ func (b *bisyncTest) checkPreReqs(ctx context.Context, opt *bisync.Options) (con
 			if obj != nil {
 				require.NoError(b.t, upperObj.Remove(ctx))
 			}
-			return entries[0].Remote()
+			for _, d := range []fs.Fs{f, tempDir} {
+				if err := d.Rmdir(ctx, dir); err != nil && !errors.Is(err, fs.ErrorDirNotFound) {
+					require.NoError(b.t, err)
+				}
+			}
+			return got
 		}
 		got1 := preTest(b.fs1, testfilename1)
 		got1 += preTest(b.fs1, testfilename2)
@@ -1125,11 +1178,11 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
 		Workdir:       b.workDir,
 		NoCleanup:     true,
 		SaveQueues:    true,
-		MaxDelete:     bisync.DefaultMaxDelete,
 		CheckFilename: bisync.DefaultCheckFilename,
 		CheckSync:     bisync.CheckSyncTrue,
-		TestFn:        b.TestFn,
 	}
+	opt.TestFn = b.hookFn(ctx, "during-sync", b.TestFn)
+	opt.TestFnConflictCheck = b.hookFn(ctx, "before-check", nil)
 	ctx, opt = b.checkPreReqs(ctx, opt)
 	octx, ci := fs.AddConfig(ctx)
 	fs1, fs2 := b.fs1, b.fs2
@@ -1170,7 +1223,7 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
 		case "filters-file":
 			opt.FiltersFile = val
 		case "max-delete":
-			opt.MaxDelete, err = strconv.Atoi(val)
+			ci.MaxDelete, err = strconv.ParseInt(val, 10, 64)
 			require.NoError(b.t, err, "parsing max-delete=%q", val)
 		case "max-delete-renames-aware":
 			opt.MaxDeleteRenamesAware = true
@@ -1182,6 +1235,9 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
 			ci.SizeOnly = true
 		case "ignore-size":
 			ci.IgnoreSize = true
+		case "ignore-checksum":
+			ci.IgnoreChecksum = true
+			b.ignoreChecksum = true
 		case "checksum":
 			ci.CheckSum = true
 			opt.Compare.DownloadHash = true // allows us to test crypt and the like
@@ -1252,7 +1308,13 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
 	if err != nil {
 		b.logPrintf("Bisync error: %v", err)
 	}
-	if err == nil && !opt.DryRun && opt.TestFn == nil && opt.FiltersFile == "" {
+	for _, when := range []string{"before-check", "during-sync"} {
+		if len(b.hooks[when]) > 0 {
+			b.logPrintf("hook %s did not run, discarding %d steps", when, len(b.hooks[when]))
+		}
+	}
+	b.hooks = nil
+	if err == nil && !opt.DryRun && opt.TestFn == nil && opt.TestFnConflictCheck == nil && opt.FiltersFile == "" {
 		b.checkListingsMatchRemotes(ctx, fs1, fs2)
 	}
 	return nil
@@ -1297,6 +1359,30 @@ func (b *bisyncTest) checkListingsMatchRemotes(ctx context.Context, fs1, fs2 fs.
 	}
 }
 
+// hookFn returns a test hook that calls fn, if set, and then runs the
+// scenario steps queued for the hook named when, the first time it is called.
+func (b *bisyncTest) hookFn(ctx context.Context, when string, fn bisync.TestFunc) bisync.TestFunc {
+	if len(b.hooks[when]) == 0 {
+		return fn
+	}
+	return func() {
+		if fn != nil {
+			fn()
+		}
+		steps := b.hooks[when]
+		b.hooks[when] = nil
+		// log only through the captured bisync output, so hook steps appear once and in order
+		logFile, stepStr := b.logFile, b.stepStr
+		b.logFile, b.stepStr = nil, "hook "+when+":"
+		defer func() { b.logFile, b.stepStr = logFile, stepStr }()
+		for _, step := range steps {
+			if err := b.runTestStep(ctx, step); err != nil {
+				fs.Errorf(nil, "hook %s: %q failed: %v", when, step, err)
+			}
+		}
+	}
+}
+
 // saveTestListings creates a copy of test artifacts with given prefix
 // including listings (.lst*), queues (.que) and filters (.flt, .flt.md5)
 func (b *bisyncTest) saveTestListings(prefix string, keepSource bool) (err error) {
@@ -1722,6 +1808,16 @@ func (b *bisyncTest) mangleResult(dir, file string, golden bool) string {
 	if b.testCase == "dry_run" {
 		rep = append(rep, dryrunReplacements...)
 	}
+	if b.ignoreChecksum && b.fs1.Hashes().Overlap(b.fs2.Hashes()).Count() == 0 {
+		// with no hash in common, --ignore-checksum skips the check for identical files instead of running it
+		rep = append(rep,
+			`^.*Checking potential conflicts\.\.\.$`, dropMe,
+			`^.*: \d+ differences found$`, dropMe,
+			`^.*: \d+ matching files$`, dropMe,
+			`^.*Finished checking the potential conflicts.*$`, dropMe,
+			`^.*Not checking potential conflicts.*$`, dropMe,
+		)
+	}
 	repFrom := make([]*regexp.Regexp, len(rep)/2)
 	repTo := make([]string, len(rep)/2)
 	for i := 0; i < len(rep); i += 2 {

cmd/bisync/checkfn.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/rclone/rclone/backend/crypt"
 	"github.com/rclone/rclone/cmd/bisync/bilib"
@@ -79,7 +80,7 @@ func CheckFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool,
 		return true, false, err
 	}
 	if ht == hash.None {
-		return false, true, nil
+		return missingHashFn(ctx, dst, src)
 	}
 	if !same {
 		err = fmt.Errorf("%v differ", ht)
@@ -98,14 +99,14 @@ func (b *bisyncRun) CryptCheckFn(ctx context.Context, dst, src fs.Object) (diffe
 		return true, false, fmt.Errorf("error reading hash from underlying %v: %w", underlyingDst, err)
 	}
 	if underlyingHash == "" {
-		return false, true, nil
+		return missingHashFn(ctx, dst, src)
 	}
 	cryptHash, err := b.check.fcrypt.ComputeHash(ctx, cryptDst, src, b.check.hashType)
 	if err != nil {
 		return true, false, fmt.Errorf("error computing hash: %w", err)
 	}
 	if cryptHash == "" {
-		return false, true, nil
+		return missingHashFn(ctx, dst, src)
 	}
 	if cryptHash != underlyingHash {
 		err = fmt.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", b.check.fdst.Name(), b.check.fdst.Root(), cryptHash, b.check.fsrc.Name(), b.check.fsrc.Root(), underlyingHash)
@@ -124,6 +125,15 @@ func (b *bisyncRun) ReverseCryptCheckFn(ctx context.Context, dst, src fs.Object)
 	return b.CryptCheckFn(ctx, src, dst)
 }
 
+func missingHashFn(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+	ci := fs.GetConfig(ctx)
+	if ci.SizeOnly || ci.IgnoreChecksum {
+		return false, true, nil
+	}
+	fs.Debugf(src, "hash is missing, so using --download")
+	return DownloadCheckFn(ctx, dst, src)
+}
+
 // DownloadCheckFn is a slightly modified version of Check with --download
 func DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHash bool, err error) {
 	equal, err = operations.CheckIdenticalDownload(ctx, src, dst)
@@ -134,17 +144,37 @@ func DownloadCheckFn(ctx context.Context, dst, src fs.Object) (equal bool, noHas
 }
 
 // check potential conflicts (to avoid renaming if already identical)
-func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (bilib.Names, error) {
-	matches := bilib.Names{}
+func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter.Filter, fs1, fs2 fs.Fs) (matches bilib.Names, matches1, matches2 *fileList, err error) {
+	matches = bilib.Names{}
+	matches1, matches2 = newFileList(), newFileList()
 	if filterCheck.HaveFilesFrom() {
 		fs.Debugf(nil, "There are potential conflicts to check.")
+		if b.opt.TestFnConflictCheck != nil {
+			b.opt.TestFnConflictCheck()
+		}
+
+		// the sizes already match, and they are all a check could compare here
+		ci := fs.GetConfig(ctxCheck)
+		if ci.SizeOnly || (ci.IgnoreChecksum && fs1.Hashes().Overlap(fs2.Hashes()).Count() == 0) {
+			fs.Infof(nil, "Not checking potential conflicts, as only their sizes could be compared, and those already match.")
+			for file := range filterCheck.Files() {
+				matches.Add(file)
+				if b.march.ls1.has(file) {
+					b.march.ls1.getPut(file, matches1)
+				}
+				if b.march.ls2.has(file) {
+					b.march.ls2.getPut(file, matches2)
+				}
+			}
+			return matches, matches1, matches2, nil
+		}
 
 		opt, close, checkopterr := check.GetCheckOpt(fs1, fs2)
 		if checkopterr != nil {
 			b.critical = true
 			b.retryable = true
 			fs.Debugf(nil, "GetCheckOpt error: %v", checkopterr)
-			return matches, checkopterr
+			return matches, matches1, matches2, checkopterr
 		}
 		defer close()
 
@@ -152,6 +182,22 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter
 
 		opt = b.WhichCheck(ctxCheck, opt)
 
+		// record the objects found identical, so they can be compared with the listing snapshot
+		var mu sync.Mutex
+		checkFn := opt.Check
+		opt.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
+			differ, noHash, err = checkFn(ctx, dst, src)
+			if err == nil && !differ {
+				f1 := b.comparedFields(ctx, src, b.march.ls1.hash)
+				f2 := b.comparedFields(ctx, dst, b.march.ls2.hash)
+				mu.Lock()
+				matches1.put(src.Remote(), f1.size, f1.time, f1.hash, "", "-")
+				matches2.put(dst.Remote(), f2.size, f2.time, f2.hash, "", "-")
+				mu.Unlock()
+			}
+			return differ, noHash, err
+		}
+
 		fs.Infof(nil, "Checking potential conflicts...")
 		check := operations.CheckFn(ctxCheck, opt)
 		fs.Infof(nil, "Finished checking the potential conflicts. %s", check)
@@ -168,9 +214,22 @@ func (b *bisyncRun) checkconflicts(ctxCheck context.Context, filterCheck *filter
 		} else {
 			fs.Debugf(nil, "None of the conflicts were determined to be identical.")
 		}
-
 	}
-	return matches, nil
+	return matches, matches1, matches2, nil
+}
+
+// comparedFields returns the fields of o that bisync compares, the same way
+// the listing records them.
+func (b *bisyncRun) comparedFields(ctx context.Context, o fs.Object, hashType hash.Type) (fi fileInfo) {
+	fi.size = o.Size()
+	if b.opt.Compare.Modtime {
+		fi.time = o.ModTime(ctx).In(TZ)
+	}
+	if b.opt.Compare.Checksum && hashType != hash.None {
+		fi.hash, _ = o.Hash(ctx, hashType)
+		fi.hash, _ = b.tryDownloadHash(ctx, o, fi.hash)
+	}
+	return fi
 }
 
 // WhichEqual is similar to WhichCheck, but checks a single object.

cmd/bisync/checkfn_internal_test.go

@@ -0,0 +1,117 @@
+package bisync
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fstest/mockfs"
+	"github.com/rclone/rclone/fstest/mockobject"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// blankHashObject supports MD5 through its Fs but returns a blank hash
+type blankHashObject struct {
+	*mockobject.ContentMockObject
+}
+
+func (o blankHashObject) Hash(ctx context.Context, t hash.Type) (string, error) {
+	return "", nil
+}
+
+func newBlankHashObject(ctx context.Context, t *testing.T, fsName, content string) blankHashObject {
+	f, err := mockfs.NewFs(ctx, fsName, "", nil)
+	require.NoError(t, err)
+	f.(*mockfs.Fs).SetHashes(hash.NewHashSet(hash.MD5))
+	o := mockobject.New("file.txt").WithContent([]byte(content), mockobject.SeekModeNone)
+	o.SetFs(f)
+	return blankHashObject{o}
+}
+
+func TestCheckFnMissingHash(t *testing.T) {
+	for _, test := range []struct {
+		name       string
+		content    string
+		flag       func(ci *fs.ConfigInfo)
+		wantDiffer bool
+		wantNoHash bool
+	}{
+		{name: "identical", content: "hello"},
+		{name: "same size, different content", content: "HELLO", wantDiffer: true},
+		{name: "size-only", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantNoHash: true},
+		{name: "ignore-checksum", content: "HELLO", flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }, wantNoHash: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, ci := fs.AddConfig(context.Background())
+			if test.flag != nil {
+				test.flag(ci)
+			}
+			src := newBlankHashObject(ctx, t, "path1", "hello")
+			dst := newBlankHashObject(ctx, t, "path2", test.content)
+			differ, noHash, err := CheckFn(ctx, dst, src)
+			require.NoError(t, err)
+			assert.Equal(t, test.wantDiffer, differ, "differ")
+			assert.Equal(t, test.wantNoHash, noHash, "noHash")
+		})
+	}
+}
+
+func newEmptyFs(ctx context.Context, t *testing.T, fsName string, hashes hash.Set) fs.Fs {
+	f, err := mockfs.NewFs(ctx, fsName, "", nil)
+	require.NoError(t, err)
+	f.(*mockfs.Fs).SetHashes(hashes)
+	return f
+}
+
+func TestCheckConflictsSkipsCheck(t *testing.T) {
+	md5 := hash.NewHashSet(hash.MD5)
+	none := hash.NewHashSet()
+	for _, test := range []struct {
+		name     string
+		hashes2  hash.Set
+		flag     func(ci *fs.ConfigInfo)
+		wantSkip bool
+	}{
+		{name: "common hash", hashes2: md5},
+		{name: "no common hash", hashes2: none},
+		{name: "size-only with a common hash", hashes2: md5, flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantSkip: true},
+		{name: "size-only without a common hash", hashes2: none, flag: func(ci *fs.ConfigInfo) { ci.SizeOnly = true }, wantSkip: true},
+		{name: "ignore-checksum with a common hash", hashes2: md5, flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }},
+		{name: "ignore-checksum without a common hash", hashes2: none, flag: func(ci *fs.ConfigInfo) { ci.IgnoreChecksum = true }, wantSkip: true},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, ci := fs.AddConfig(context.Background())
+			if test.flag != nil {
+				test.flag(ci)
+			}
+			ctx, fi := filter.AddConfig(ctx)
+			require.NoError(t, fi.AddFile("file.txt"))
+
+			b := &bisyncRun{opt: &Options{}}
+			b.opt.Compare.Size = true
+			b.opt.Compare.Modtime = true
+			modtime := time.Date(2001, 2, 3, 4, 5, 6, 0, time.UTC)
+			b.march.ls1, b.march.ls2 = newFileList(), newFileList()
+			b.march.ls1.put("file.txt", 5, modtime, "", "-", "-")
+			b.march.ls2.put("file.txt", 5, modtime, "", "-", "-")
+
+			fs1 := newEmptyFs(ctx, t, "path1", md5)
+			fs2 := newEmptyFs(ctx, t, "path2", test.hashes2)
+			matches, matches1, matches2, err := b.checkconflicts(ctx, fi, fs1, fs2)
+			require.NoError(t, err)
+
+			assert.Equal(t, test.wantSkip, matches.Has("file.txt"), "matched a file that is on neither path")
+			if test.wantSkip {
+				assert.True(t, b.sameCompared(b.march.ls1, matches1, "file.txt", fs1), "Path1 differs from the listing")
+				assert.True(t, b.sameCompared(b.march.ls2, matches2, "file.txt", fs2), "Path2 differs from the listing")
+			} else {
+				assert.False(t, matches1.has("file.txt"), "Path1 checked")
+				assert.False(t, matches2.has("file.txt"), "Path2 checked")
+			}
+		})
+	}
+}

cmd/bisync/cmd.go

@@ -38,7 +38,7 @@ type Options struct {
 	CheckSync             CheckSyncMode
 	CreateEmptySrcDirs    bool
 	RemoveEmptyDirs       bool
-	MaxDelete             int // percentage from 0 to 100
+	MaxDelete             int // percentage from 0 to 100, set by Bisync from --max-delete
 	MaxDeleteRenamesAware bool
 	Force                 bool
 	FiltersFile           string
@@ -46,13 +46,14 @@ type Options struct {
 	OrigBackupDir         string
 	BackupDir1            string
 	BackupDir2            string
-	DryRun                bool
+	DryRun                bool // set by Bisync from --dry-run
 	NoCleanup             bool
 	SaveQueues            bool // save extra debugging files (test only flag)
 	IgnoreListingChecksum bool
 	Resilient             bool
 	Recover               bool
 	TestFn                TestFunc // test-only option, for mocking errors
+	TestFnConflictCheck   TestFunc // test-only option, called before checking potential conflicts
 	Compare               CompareOpt
 	CompareFlag           string
 	DebugName             string
@@ -178,7 +179,6 @@ var commandDefinition = &cobra.Command{
 
 		ctx := context.Background()
 		opt := Opt
-		opt.applyContext(ctx)
 		if tzLocal {
 			TZ = time.Local
 		}
@@ -204,22 +204,23 @@ var commandDefinition = &cobra.Command{
 	},
 }
 
-func (opt *Options) applyContext(ctx context.Context) {
+func (opt *Options) applyContext(ctx context.Context) error {
 	maxDelete := DefaultMaxDelete
 	ci := fs.GetConfig(ctx)
+	if ci.MaxDelete > 100 {
+		return fmt.Errorf("--max-delete is a percentage for bisync and must be from 0 to 100, not %d", ci.MaxDelete)
+	}
 	if ci.MaxDelete >= 0 {
 		maxDelete = int(ci.MaxDelete)
 	}
 	if maxDelete < 0 {
 		maxDelete = 0
 	}
-	if maxDelete > 100 {
-		maxDelete = 100
-	}
 	opt.MaxDelete = maxDelete
 	// reset MaxDelete for fs/operations, bisync handles this parameter specially
 	ci.MaxDelete = -1
 	opt.DryRun = ci.DryRun
+	return nil
 }
 
 func (opt *Options) setDryRun(ctx context.Context) context.Context {

cmd/bisync/deltas.go

@@ -290,8 +290,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 	handled := bilib.Names{}
 	renameSkipped := bilib.Names{}
 	deletedonboth := bilib.Names{}
-	skippedDirs1 := newFileList()
-	skippedDirs2 := newFileList()
+	skipped1 := newFileList()
+	skipped2 := newFileList()
 	b.renames = renames{}
 
 	ctxMove := b.opt.setDryRun(ctx)
@@ -361,7 +361,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 	}
 
 	// if there are potential conflicts to check, check them all here (outside the loop) in one fell swoop
-	matches, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)
+	matches, matches1, matches2, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)
 
 	for _, file := range ds1.sort() {
 		alias := b.aliases.Alias(file)
@@ -388,8 +388,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 				// if files are identical, leave them alone instead of renaming
 				if (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) {
 					fs.Infof(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)
-					b.march.ls1.getPut(file, skippedDirs1)
-					b.march.ls2.getPut(file, skippedDirs2)
+					b.march.ls1.getPut(file, skipped1)
+					b.march.ls2.getPut(file, skipped2)
 					b.debugFn(file, func() {
 						b.debug(file, fmt.Sprintf("deltas dir: %s, ls1 has name?: %v, ls2 has name?: %v", file, b.march.ls1.has(b.DebugName), b.march.ls2.has(b.DebugName)))
 					})
@@ -415,10 +415,14 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 								b.indent("Path1", p2, "Queue copy to Path2")
 								copy1to2.Add(b.march.ls1.getTryAlias(file, alias))
 							}
-						} else {
+						} else if b.sameCompared(b.march.ls1, matches1, b.march.ls1.getTryAlias(file, alias), b.fs1) &&
+							b.sameCompared(b.march.ls2, matches2, b.march.ls2.getTryAlias(file, alias), b.fs2) {
 							fs.Infof(nil, "Files are equal! Skipping: %s", file)
-							renameSkipped.Add(file)
-							renameSkipped.Add(alias)
+							b.march.ls1.getPut(b.march.ls1.getTryAlias(file, alias), skipped1)
+							b.march.ls2.getPut(b.march.ls2.getTryAlias(file, alias), skipped2)
+						} else {
+							// the snapshot isn't the state we found equal, so leave the prior listing for the next run
+							fs.Infof(nil, "Files are equal but changed after the listing was made, so will check again next run: %s", file)
 						}
 					} else {
 						fs.Debugf(nil, "Files are NOT equal: %s", file)
@@ -530,8 +534,8 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (result
 	queues.copy2to1 = copy2to1
 	queues.renameSkipped = renameSkipped
 	queues.deletedonboth = deletedonboth
-	queues.skippedDirs1 = skippedDirs1
-	queues.skippedDirs2 = skippedDirs2
+	queues.skipped1 = skipped1
+	queues.skipped2 = skipped2
 
 	return
 }
@@ -632,3 +636,12 @@ func (b *bisyncRun) updateAliases(ctx context.Context, ds1, ds2 *deltaSet) {
 	addAliases(delMap1, fullMap2)
 	addAliases(delMap2, fullMap1)
 }
+
+// sameCompared reports whether file has the same values in the listing
+// snapshot and in the checked list, in every field bisync compares.
+func (b *bisyncRun) sameCompared(snapshot, checked *fileList, file string, f fs.Info) bool {
+	if !checked.has(file) {
+		return false
+	}
+	return len(b.fileInfoDiffs(file, file, snapshot, checked, f, f, snapshot.hash, snapshot.hash)) == 0
+}

cmd/bisync/help.go

@@ -41,6 +41,7 @@ func RcHelp() string {
 - path1 (required) - (string) a remote directory string e.g. ||drive:path1||
 - path2 (required) - (string) a remote directory string e.g. ||drive:path2||
 - dryRun - (bool) dry-run mode
+- maxDelete - (int) Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: {MAXDELETE}%)
 `+GenerateParams()+`
 See [bisync command help](https://rclone.org/commands/rclone_bisync/)
 and [full bisync description](https://rclone.org/bisync/)

cmd/bisync/listing.go

@@ -129,12 +129,6 @@ func (ls *fileList) put(file string, size int64, modtime time.Time, hash, id str
 	fi := ls.get(file)
 	if fi != nil {
 		fi.size = size
-		// if already have higher precision of same time, avoid overwriting it
-		if fi.time != modtime {
-			if modtime.Before(fi.time) && fi.time.Sub(modtime) < time.Second {
-				modtime = fi.time
-			}
-		}
 		fi.time = modtime
 		fi.hash = hash
 		fi.id = id
@@ -186,29 +180,35 @@ func (ls *fileList) getHash(file string) string {
 }
 
 func (b *bisyncRun) fileInfoEqual(file1, file2 string, ls1, ls2 *fileList) bool {
-	equal := true
 	if ls1.isDir(file1) && ls2.isDir(file2) {
-		return equal
+		return true
 	}
+	diffs := b.fileInfoDiffs(file1, file2, ls1, ls2, b.fs1, b.fs2, b.opt.Compare.HashType1, b.opt.Compare.HashType2)
+	for _, diff := range diffs {
+		b.indent("ERROR", file1, diff)
+	}
+	return len(diffs) == 0
+}
+
+// fileInfoDiffs describes each field bisync compares where file1 in ls1
+// (on f1, with hash type ht1) differs from file2 in ls2 (on f2, with ht2).
+func (b *bisyncRun) fileInfoDiffs(file1, file2 string, ls1, ls2 *fileList, f1, f2 fs.Info, ht1, ht2 hash.Type) (diffs []string) {
 	if b.opt.Compare.Size {
 		if sizeDiffers(ls1.getSize(file1), ls2.getSize(file2)) {
-			b.indent("ERROR", file1, fmt.Sprintf("Size not equal in listing. Path1: %v, Path2: %v", ls1.getSize(file1), ls2.getSize(file2)))
-			equal = false
+			diffs = append(diffs, fmt.Sprintf("Size not equal in listing. Path1: %v, Path2: %v", ls1.getSize(file1), ls2.getSize(file2)))
 		}
 	}
 	if b.opt.Compare.Modtime {
-		if timeDiffers(b.fctx, ls1.getTime(file1), ls2.getTime(file2), b.fs1, b.fs2) {
-			b.indent("ERROR", file1, fmt.Sprintf("Modtime not equal in listing. Path1: %v, Path2: %v", ls1.getTime(file1), ls2.getTime(file2)))
-			equal = false
+		if timeDiffers(b.fctx, ls1.getTime(file1), ls2.getTime(file2), f1, f2) {
+			diffs = append(diffs, fmt.Sprintf("Modtime not equal in listing. Path1: %v, Path2: %v", ls1.getTime(file1), ls2.getTime(file2)))
 		}
 	}
 	if b.opt.Compare.Checksum && !b.queueOpt.ignoreListingChecksum {
-		if b.hashDiffers(ls1.getHash(file1), ls2.getHash(file2), b.opt.Compare.HashType1, b.opt.Compare.HashType2, ls1.getSize(file1), ls2.getSize(file2)) {
-			b.indent("ERROR", file1, fmt.Sprintf("Checksum not equal in listing. Path1: %v, Path2: %v", ls1.getHash(file1), ls2.getHash(file2)))
-			equal = false
+		if b.hashDiffers(ls1.getHash(file1), ls2.getHash(file2), ht1, ht2, ls1.getSize(file1), ls2.getSize(file2)) {
+			diffs = append(diffs, fmt.Sprintf("Checksum not equal in listing. Path1: %v, Path2: %v", ls1.getHash(file1), ls2.getHash(file2)))
 		}
 	}
-	return equal
+	return diffs
 }
 
 // also returns false if not found
@@ -581,6 +581,11 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
 	// account for "deltaOthers" we handled separately
 	if queues.deletedonboth.NotEmpty() {
 		for file := range queues.deletedonboth {
+			// a conflict rename can reuse the name of a file deleted on
+			// both sides, in which case it was just copied and must stay
+			if queues.copy1to2.Has(file) || queues.copy2to1.Has(file) {
+				continue
+			}
 			srcList.remove(file)
 			dstList.remove(file)
 		}
@@ -627,9 +632,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
 		}
 	}
 
-	// recheck the ones we skipped because they were equal
-	// we never got their info because they were never synced.
-	// TODO: add flag to skip this? (since it re-lists)
+	// recheck the ones we didn't rename due to --dry-run
 	if queues.renameSkipped.NotEmpty() {
 		skippedList := queues.renameSkipped.ToList()
 		for _, file := range skippedList {
@@ -638,7 +641,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
 			}
 		}
 	}
-	// skipped dirs -- nothing to recheck, just add them
+	// skipped dirs and equal files -- nothing to recheck, just add them from the snapshot
 	// (they are not necessarily there already, if they are new)
 	path1List := srcList
 	path2List := dstList
@@ -646,11 +649,11 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
 		path1List = dstList
 		path2List = srcList
 	}
-	if !queues.skippedDirs1.empty() {
-		queues.skippedDirs1.getPutAll(path1List)
+	if !queues.skipped1.empty() {
+		queues.skipped1.getPutAll(path1List)
 	}
-	if !queues.skippedDirs2.empty() {
-		queues.skippedDirs2.getPutAll(path2List)
+	if !queues.skipped2.empty() {
+		queues.skipped2.getPutAll(path2List)
 	}
 
 	if filterRecheck.HaveFilesFrom() {

cmd/bisync/listing_internal_test.go

@@ -0,0 +1,114 @@
+package bisync
+
+import (
+	"context"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileListPut(t *testing.T) {
+	old := time.Date(2026, 9, 29, 12, 0, 1, 555555555, time.UTC)
+	for _, test := range []struct {
+		name    string
+		modtime time.Time
+	}{
+		{"slightly older", old.Add(-124 * time.Microsecond)},
+		{"slightly newer", old.Add(124 * time.Microsecond)},
+		{"same second, truncated", old.Truncate(time.Second)},
+		{"same second, rounded up", old.Round(time.Second)},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			ls := newFileList()
+			ls.put("file", 1, old, "", "-", "-")
+			ls.put("file", 2, test.modtime, "", "-", "-")
+			assert.True(t, test.modtime.Equal(ls.getTime("file")), "got %v, want %v", ls.getTime("file"), test.modtime)
+			assert.EqualValues(t, 2, ls.getSize("file"))
+		})
+	}
+}
+
+// Test a file whose new version is less than a second older than the old one
+func TestBisyncCloseModtimes(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping test on non local remote")
+	}
+	ctx := context.Background()
+	unchanged := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)
+	newRun := func(t *testing.T) (path1, path2 string, run func(resync bool) error) {
+		// t.TempDir includes the test name, which makes the listing names too long
+		dir, err := os.MkdirTemp("", "bisync")
+		require.NoError(t, err)
+		t.Cleanup(func() { _ = os.RemoveAll(dir) })
+		path1, path2 = filepath.Join(dir, "path1"), filepath.Join(dir, "path2")
+		workdir := filepath.Join(dir, "workdir")
+		for _, path := range []string{path1, path2} {
+			require.NoError(t, os.Mkdir(path, 0o755))
+			require.NoError(t, os.WriteFile(filepath.Join(path, "file1.txt"), []byte("unchanged"), 0o644))
+			require.NoError(t, os.Chtimes(filepath.Join(path, "file1.txt"), unchanged, unchanged))
+		}
+		run = func(resync bool) error {
+			f1, err := fs.NewFs(ctx, path1)
+			require.NoError(t, err)
+			f2, err := fs.NewFs(ctx, path2)
+			require.NoError(t, err)
+			return Bisync(ctx, f1, f2, &Options{Workdir: workdir, Resync: resync, CheckSync: CheckSyncTrue, MaxDelete: DefaultMaxDelete})
+		}
+		return path1, path2, run
+	}
+	writeFile := func(t *testing.T, path, content string, modtime time.Time) {
+		file := filepath.Join(path, "file0.txt")
+		require.NoError(t, os.WriteFile(file, []byte(content), 0o644))
+		require.NoError(t, os.Chtimes(file, modtime, modtime))
+	}
+	// listingTime returns the modtime recorded for file0.txt in the listing for path (1 or 2)
+	listingTime := func(t *testing.T, path1 string, path int) time.Time {
+		listings, err := filepath.Glob(filepath.Join(filepath.Dir(path1), "workdir", "*.path"+string(rune('0'+path))+".lst"))
+		require.NoError(t, err)
+		require.Len(t, listings, 1)
+		data, err := os.ReadFile(listings[0])
+		require.NoError(t, err)
+		for line := range strings.SplitSeq(string(data), "\n") {
+			if strings.HasSuffix(line, `"file0.txt"`) {
+				fields := strings.Fields(line)
+				modtime, err := time.Parse(timeFormat, fields[4])
+				require.NoError(t, err)
+				return modtime
+			}
+		}
+		t.Fatal("file0.txt not in listing")
+		return time.Time{}
+	}
+	// fileTime returns file0.txt's modtime as the filesystem stored it (Windows keeps 100ns)
+	fileTime := func(t *testing.T, path string) time.Time {
+		info, err := os.Stat(filepath.Join(path, "file0.txt"))
+		require.NoError(t, err)
+		return info.ModTime()
+	}
+	modtime := time.Date(2026, 9, 29, 12, 0, 1, 555555555, time.UTC)
+
+	t.Run("resync", func(t *testing.T) {
+		path1, path2, run := newRun(t)
+		writeFile(t, path1, "path1", modtime)
+		writeFile(t, path2, "path2, newer", modtime.Add(124*time.Microsecond))
+		require.NoError(t, run(true))
+		assert.True(t, fileTime(t, path2).Equal(listingTime(t, path1, 2)), "Path2 listing has %v, file has %v", listingTime(t, path1, 2), fileTime(t, path2))
+	})
+
+	t.Run("older edit", func(t *testing.T) {
+		path1, path2, run := newRun(t)
+		writeFile(t, path1, "original", modtime)
+		require.NoError(t, run(true))
+		writeFile(t, path1, "edited", modtime.Add(-200*time.Microsecond))
+		require.NoError(t, run(false))
+		assert.True(t, fileTime(t, path1).Equal(listingTime(t, path1, 1)), "Path1 listing has %v, file has %v", listingTime(t, path1, 1), fileTime(t, path1))
+		assert.True(t, fileTime(t, path2).Equal(listingTime(t, path1, 2)), "Path2 listing has %v, file has %v", listingTime(t, path1, 2), fileTime(t, path2))
+	})
+}

cmd/bisync/operations.go

@@ -63,15 +63,20 @@ type bisyncRun struct {
 type queues struct {
 	copy1to2      bilib.Names
 	copy2to1      bilib.Names
-	renameSkipped bilib.Names // not renamed because it was equal
-	skippedDirs1  *fileList
-	skippedDirs2  *fileList
+	renameSkipped bilib.Names // not renamed due to --dry-run
+	skipped1      *fileList   // dirs and equal files not renamed, from the Path1 snapshot
+	skipped2      *fileList   // dirs and equal files not renamed, from the Path2 snapshot
 	deletedonboth bilib.Names
 }
 
 // Bisync handles lock file, performs bisync run and checks exit status
 func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 	opt := *optArg // ensure that input is never changed
+	// applyContext changes the config, so give it a copy
+	ctx, _ = fs.AddConfig(ctx)
+	if err = opt.applyContext(ctx); err != nil {
+		return err
+	}
 	b := &bisyncRun{
 		fs1:       fs1,
 		fs2:       fs2,

cmd/bisync/rc.go

@@ -65,9 +65,9 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	opt := &Options{}
 	octx, ci := fs.AddConfig(ctx)
 
-	if dryRun, err := in.GetBool("dryRun"); err == nil {
-		ci.DryRun = dryRun
-		opt.DryRun = dryRun
+	if dryRun, err := in.GetBool("dryRun"); err == nil && dryRun {
+		// dryRun can ask for a dry run, but not cancel one from the global config
+		ci.DryRun = true
 	} else if rc.NotErrParamNotFound(err) {
 		return nil, err
 	}
@@ -76,60 +76,58 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 		if maxDelete < 0 || maxDelete > 100 {
 			return nil, rc.NewErrParamInvalid(errors.New("maxDelete must be a percentage between 0 and 100"))
 		}
-		opt.MaxDelete = int(maxDelete)
+		ci.MaxDelete = maxDelete
 	} else if rc.NotErrParamNotFound(err) {
 		return nil, err
 	}
 
 	if opt.Resync, err = in.GetBool("resync"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("resync", "optional parameter is missing. using default value: %v", opt.Resync)
+		return nil, err
 	}
 	if opt.CheckAccess, err = in.GetBool("checkAccess"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("checkAccess", "optional parameter is missing. using default value: %v", opt.CheckAccess)
+		return nil, err
 	}
 	if opt.Force, err = in.GetBool("force"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("force", "optional parameter is missing. using default value: %v", opt.Force)
+		return nil, err
 	}
 	if opt.MaxDeleteRenamesAware, err = in.GetBool("maxDeleteRenamesAware"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("maxDeleteRenamesAware", "optional parameter is missing. using default value: %v", opt.MaxDeleteRenamesAware)
+		return nil, err
 	}
 	if opt.CreateEmptySrcDirs, err = in.GetBool("createEmptySrcDirs"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("createEmptySrcDirs", "optional parameter is missing. using default value: %v", opt.CreateEmptySrcDirs)
+		return nil, err
 	}
 	if opt.RemoveEmptyDirs, err = in.GetBool("removeEmptyDirs"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("removeEmptyDirs", "optional parameter is missing. using default value: %v", opt.RemoveEmptyDirs)
+		return nil, err
 	}
 	if opt.NoCleanup, err = in.GetBool("noCleanup"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("noCleanup", "optional parameter is missing. using default value: %v", opt.NoCleanup)
+		return nil, err
 	}
 	if opt.IgnoreListingChecksum, err = in.GetBool("ignoreListingChecksum"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("ignoreListingChecksum", "optional parameter is missing. using default value: %v", opt.IgnoreListingChecksum)
+		return nil, err
 	}
 	if opt.Resilient, err = in.GetBool("resilient"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("resilient", "optional parameter is missing. using default value: %v", opt.Resilient)
+		return nil, err
 	}
 	if opt.CheckFilename, err = in.GetString("checkFilename"); rc.NotErrParamNotFound(err) {
-		opt.CheckFilename = DefaultCheckFilename
-		fs.Debugf("checkFilename", "optional parameter is missing. using default value: %v", opt.CheckFilename)
+		return nil, err
 	}
 	if opt.FiltersFile, err = in.GetString("filtersFile"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("filtersFile", "optional parameter is missing. using default value: %v", opt.FiltersFile)
+		return nil, err
 	}
 	if opt.Workdir, err = in.GetString("workdir"); rc.NotErrParamNotFound(err) {
-		// "" sets correct default later
-		fs.Debugf("workdir", "optional parameter is missing. using default value: %v", opt.Workdir)
+		return nil, err
 	}
-	if opt.BackupDir1, err = in.GetString("backupDir1"); rc.NotErrParamNotFound(err) {
-		// we accept an alternate capitalization here for backward compatibility.
-		if opt.BackupDir1, err = in.GetString("backupdir1"); rc.NotErrParamNotFound(err) {
-			fs.Debugf("backupDir1", "optional parameter is missing. using default value: %v", opt.BackupDir1)
-		}
+	if opt.BackupDir1, err = in.GetString("backupDir1"); rc.IsErrParamNotFound(err) {
+		opt.BackupDir1, err = in.GetString("backupdir1") // the name before v1.74
 	}
-	if opt.BackupDir2, err = in.GetString("backupDir2"); rc.NotErrParamNotFound(err) {
-		// we accept an alternate capitalization here for backward compatibility.
-		if opt.BackupDir2, err = in.GetString("backupdir2"); rc.NotErrParamNotFound(err) {
-			fs.Debugf("backupDir2", "optional parameter is missing. using default value: %v", opt.BackupDir2)
-		}
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
+	}
+	if opt.BackupDir2, err = in.GetString("backupDir2"); rc.IsErrParamNotFound(err) {
+		opt.BackupDir2, err = in.GetString("backupdir2") // the name before v1.74
+	}
+	if rc.NotErrParamNotFound(err) {
+		return nil, err
 	}
 	if err = setEnum(in, "checkSync", "true", opt.CheckSync.Set); err != nil {
 		return nil, err
@@ -144,26 +142,25 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 		return nil, err
 	}
 	if opt.ConflictSuffixFlag, err = in.GetString("conflictSuffix"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("conflictSuffix", "optional parameter is missing. using default value: %v", opt.ConflictSuffixFlag)
+		return nil, err
 	}
 	if opt.Recover, err = in.GetBool("recover"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("recover", "optional parameter is missing. using default value: %v", opt.Recover)
+		return nil, err
 	}
 	if opt.CompareFlag, err = in.GetString("compare"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("compare", "optional parameter is missing. using default value: %v", opt.CompareFlag)
+		return nil, err
 	}
 	if opt.Compare.NoSlowHash, err = in.GetBool("noSlowHash"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("noSlowHash", "optional parameter is missing. using default value: %v", opt.Compare.NoSlowHash)
+		return nil, err
 	}
 	if opt.Compare.SlowHashSyncOnly, err = in.GetBool("slowHashSyncOnly"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("slowHashSyncOnly", "optional parameter is missing. using default value: %v", opt.Compare.SlowHashSyncOnly)
+		return nil, err
 	}
 	if opt.Compare.DownloadHash, err = in.GetBool("downloadHash"); rc.NotErrParamNotFound(err) {
-		fs.Debugf("downloadHash", "optional parameter is missing. using default value: %v", opt.Compare.DownloadHash)
+		return nil, err
 	}
 	if opt.MaxLock, err = in.GetFsDuration("maxLock"); rc.NotErrParamNotFound(err) {
-		opt.MaxLock = 0
-		fs.Debugf("maxLock", "optional parameter is missing. using default value: %v", opt.MaxLock)
+		return nil, err
 	}
 
 	fs1, err := rc.GetFsNamed(octx, in, "path1")
@@ -200,12 +197,11 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 
 func setEnum(in rc.Params, name string, defaultVal string, set func(s string) error) error {
 	v, err := in.GetString(name)
-	if rc.NotErrParamNotFound(err) || v == "" {
-		v = defaultVal
-		fs.Debugf(name, "optional parameter is missing. using default value: %v", v)
-	}
-	if err := set(v); err != nil {
+	if rc.NotErrParamNotFound(err) {
 		return err
 	}
-	return nil
+	if v == "" {
+		v = defaultVal
+	}
+	return set(v)
 }

cmd/bisync/rc.md

@@ -5,6 +5,8 @@ This takes the following parameters:
 - path1 (required) - (string) a remote directory string e.g. `drive:path1`
 - path2 (required) - (string) a remote directory string e.g. `drive:path2`
 - dryRun - (bool) dry-run mode
+- maxDelete - (int) Safety check on maximum percentage of deleted files
+allowed. If exceeded, the bisync run will abort. (default: 50%)
 - backupDir1 - (string) --backup-dir for Path1. Must be a non-overlapping path on
 the same remote.  
 - backupDir2 - (string) --backup-dir for Path2. Must be a non-overlapping path on
@@ -34,6 +36,8 @@ unavailable. (warning: may be slow and use lots of data!)
 using with --verbose  
 - ignoreListingChecksum - (bool) Do not use checksums for listings (add --ignore-
 checksum to additionally skip post-copy checksum checks)  
+- maxDeleteRenamesAware - (bool) Exclude tracked renames from the --max-delete
+safety check (requires --track-renames)  
 - maxLock - (Duration) Consider lock files older than this to be expired
 (default: 0 (never expire)) (minimum: 2m)  
 - noCleanup - (bool) Retain working files (useful for troubleshooting and

cmd/bisync/rc_test.go

@@ -0,0 +1,266 @@
+package bisync_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/rc"
+	"github.com/rclone/rclone/fs/rc/jobs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/spf13/pflag"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// newRcBisync is newRcBisyncCtx with the rc jobs run in a plain context
+func newRcBisync(t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {
+	return newRcBisyncCtx(context.Background(), t, n)
+}
+
+// newRcBisyncCtx creates Path1 with n files, resyncs it to Path2, and returns
+// the paths and a function which runs sync/bisync on them as an rc job in
+// ctx, whose config stands in for the flags the rcd was started with
+func newRcBisyncCtx(ctx context.Context, t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {
+	if *fstest.RemoteName != "" {
+		t.Skip("Skipping test on non local remote")
+	}
+	dir := t.TempDir()
+	path1 = filepath.Join(dir, "path1")
+	path2 = filepath.Join(dir, "path2")
+	workdir := filepath.Join(dir, "workdir")
+	require.NoError(t, os.Mkdir(path1, 0o755))
+	require.NoError(t, os.Mkdir(path2, 0o755))
+	for i := range n {
+		require.NoError(t, os.WriteFile(filepath.Join(path1, fmt.Sprintf("file%d.txt", i)), []byte("content"), 0o644))
+	}
+	call := rc.Calls.Get("sync/bisync")
+	require.NotNil(t, call)
+	runIn := func(ctx context.Context, in rc.Params) error {
+		in["path1"] = path1
+		in["path2"] = path2
+		if _, ok := in["workdir"]; !ok {
+			in["workdir"] = workdir
+		}
+		_, _, err := jobs.NewJob(ctx, call.Fn, in)
+		return err
+	}
+	require.NoError(t, runIn(context.Background(), rc.Params{"resync": true}))
+	return path1, path2, func(in rc.Params) error { return runIn(ctx, in) }
+}
+
+func TestRcDryRun(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		in     rc.Params
+		dryRun bool
+	}{
+		{"dryRun", rc.Params{"dryRun": true}, true},
+		{"dry_run", rc.Params{"dry_run": true}, true},
+		{"_config", rc.Params{"_config": rc.Params{"DryRun": true}}, true},
+		{"dry_run and dryRun false", rc.Params{"dry_run": true, "dryRun": false}, true},
+		{"_config and dryRun false", rc.Params{"_config": rc.Params{"DryRun": true}, "dryRun": false}, true},
+		{"dryRun false", rc.Params{"dryRun": false}, false},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			path1, path2, run := newRcBisync(t, 1)
+			require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
+			require.NoError(t, run(test.in))
+			if test.dryRun {
+				assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
+			} else {
+				assert.FileExists(t, filepath.Join(path2, "new.txt"))
+			}
+		})
+	}
+}
+
+func TestRcMaxDelete(t *testing.T) {
+	for _, test := range []struct {
+		name    string
+		in      rc.Params
+		wantErr string
+	}{
+		{"default", rc.Params{}, ""},
+		{"maxDelete", rc.Params{"maxDelete": 5}, "too many deletes"},
+		{"max_delete allows", rc.Params{"max_delete": 20}, ""},
+		{"max_delete aborts", rc.Params{"max_delete": 5}, "too many deletes"},
+		{"_config MaxDelete aborts", rc.Params{"_config": rc.Params{"MaxDelete": 5}}, "too many deletes"},
+		{"max_delete over 100", rc.Params{"max_delete": 500}, "--max-delete"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			path1, path2, run := newRcBisync(t, 10)
+			require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))
+			err := run(test.in)
+			if test.wantErr != "" {
+				assert.ErrorContains(t, err, test.wantErr)
+				assert.FileExists(t, filepath.Join(path2, "file0.txt"))
+			} else {
+				require.NoError(t, err)
+				assert.NoFileExists(t, filepath.Join(path2, "file0.txt"))
+			}
+		})
+	}
+}
+
+// Test a --max-delete the rcd was started with applies, as on the command line
+func TestRcMaxDeleteFromRcd(t *testing.T) {
+	for _, test := range []struct {
+		maxDelete int64
+		wantErr   string
+	}{
+		{5, "too many deletes"},
+		{500, "--max-delete"},
+	} {
+		t.Run(fmt.Sprint(test.maxDelete), func(t *testing.T) {
+			ctx, ci := fs.AddConfig(context.Background())
+			ci.MaxDelete = test.maxDelete
+			path1, path2, run := newRcBisyncCtx(ctx, t, 10)
+			require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))
+			assert.ErrorContains(t, run(rc.Params{}), test.wantErr)
+			assert.FileExists(t, filepath.Join(path2, "file0.txt"))
+		})
+	}
+}
+
+func TestRcInvalidParams(t *testing.T) {
+	for _, test := range []struct {
+		key   string
+		value any
+	}{
+		{"force", "yes"},
+		{"maxDeleteRenamesAware", "yes"},
+		{"filtersFile", []string{"filters.txt"}},
+		{"checkSync", 1},
+		{"conflictResolve", "bogus"},
+		{"maxLock", "soon"},
+	} {
+		t.Run(test.key, func(t *testing.T) {
+			path1, path2, run := newRcBisync(t, 1)
+			require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
+			err := run(rc.Params{test.key: test.value})
+			assert.Error(t, err)
+			assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
+		})
+	}
+}
+
+func TestRcBackupDir(t *testing.T) {
+	for _, test := range []struct {
+		key     string
+		changed int // the path whose change overwrites the file on the other path
+	}{
+		{"backupDir1", 2},
+		{"backupdir1", 2},
+		{"backupDir2", 1},
+		{"backupdir2", 1},
+	} {
+		t.Run(test.key, func(t *testing.T) {
+			path1, path2, run := newRcBisync(t, 2)
+			backupDir := filepath.Join(filepath.Dir(path1), "backup")
+			changedPath := map[int]string{1: path1, 2: path2}[test.changed]
+			require.NoError(t, os.WriteFile(filepath.Join(changedPath, "file0.txt"), []byte("changed"), 0o644))
+			require.NoError(t, run(rc.Params{test.key: backupDir}))
+			assert.FileExists(t, filepath.Join(backupDir, "file0.txt"))
+		})
+	}
+}
+
+func TestRcEnums(t *testing.T) {
+	read := func(t *testing.T, path string) string {
+		b, err := os.ReadFile(filepath.Join(path, "file0.txt"))
+		require.NoError(t, err)
+		return string(b)
+	}
+	// changeBoth makes file0.txt differ on both paths, with Path2 an hour newer
+	changeBoth := func(t *testing.T, path1, path2 string) {
+		now := time.Now()
+		for path, content := range map[string]string{path1: "path1", path2: "path2 wins"} {
+			file := filepath.Join(path, "file0.txt")
+			require.NoError(t, os.WriteFile(file, []byte(content), 0o644))
+			mtime := now
+			if path == path1 {
+				mtime = now.Add(-time.Hour)
+			}
+			require.NoError(t, os.Chtimes(file, mtime, mtime))
+		}
+	}
+	names := func(t *testing.T, path string) (names []string) {
+		entries, err := os.ReadDir(path)
+		require.NoError(t, err)
+		for _, e := range entries {
+			names = append(names, e.Name())
+		}
+		return names
+	}
+	t.Run("conflictResolve and conflictLoser", func(t *testing.T) {
+		path1, path2, run := newRcBisync(t, 2)
+		changeBoth(t, path1, path2)
+		require.NoError(t, run(rc.Params{"conflictResolve": "path2", "conflictLoser": "delete"}))
+		for _, path := range []string{path1, path2} {
+			assert.Equal(t, []string{"file0.txt", "file1.txt"}, names(t, path))
+			assert.Equal(t, "path2 wins", read(t, path))
+		}
+	})
+	t.Run("conflict defaults", func(t *testing.T) {
+		path1, path2, run := newRcBisync(t, 2)
+		changeBoth(t, path1, path2)
+		require.NoError(t, run(rc.Params{}))
+		assert.NotContains(t, names(t, path1), "file0.txt", "with no winner both versions are renamed")
+	})
+	for _, test := range []struct {
+		name string
+		in   rc.Params
+		want string
+	}{
+		{"resyncMode default", rc.Params{"resync": true}, "path1"},
+		{"resyncMode path2", rc.Params{"resync": true, "resyncMode": "path2"}, "path2 wins"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			path1, path2, run := newRcBisync(t, 2)
+			changeBoth(t, path1, path2)
+			require.NoError(t, run(test.in))
+			assert.Equal(t, test.want, read(t, path1))
+			assert.Equal(t, test.want, read(t, path2))
+		})
+	}
+	t.Run("checkSync only", func(t *testing.T) {
+		path1, path2, run := newRcBisync(t, 1)
+		require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
+		require.NoError(t, run(rc.Params{"checkSync": "only"}))
+		assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
+	})
+}
+
+// rcParamName returns the rc parameter name for a flag, as help.go does for rc.md
+func rcParamName(flag string) string {
+	words := strings.Split(flag, "-")
+	for i := 1; i < len(words); i++ {
+		words[i] = strings.ToUpper(words[i][:1]) + words[i][1:]
+	}
+	return strings.Join(words, "")
+}
+
+// Test every bisync flag can be set through the rc
+func TestRcFlagParity(t *testing.T) {
+	bisyncCmd, _, err := cmd.Root.Find([]string{"bisync"})
+	require.NoError(t, err)
+	bisyncCmd.Flags().VisitAll(func(flag *pflag.Flag) {
+		if flag.Hidden {
+			return
+		}
+		key := rcParamName(flag.Name)
+		t.Run(key, func(t *testing.T) {
+			_, _, run := newRcBisync(t, 1)
+			// no rc getter accepts a map, so this only fails if rcBisync reads the param
+			err := run(rc.Params{key: rc.Params{"wrong": "type"}})
+			assert.ErrorContains(t, err, key, "no rc parameter for --%s", flag.Name)
+		})
+	})
+}

cmd/bisync/testdata/test_conflict_reuse/golden/_testdir_path1.._testdir_path2.copy1to2.que

@@ -0,0 +1 @@
+"file1.txt.conflict1"