encoder: add ForceNFC and ForceNFD options

See https://github.com/rclone/rclone/issues/8088#issuecomment-3001795870

.github/workflows/build.yml

@@ -23,6 +23,9 @@ jobs:
   build:
     if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
     timeout-minutes: 60
+    defaults:
+      run:
+        shell: bash
     strategy:
       fail-fast: false
       matrix:
@@ -103,7 +106,6 @@ jobs:
           check-latest: true
 
       - name: Set environment variables
-        shell: bash
         run: |
           echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
           echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
@@ -112,7 +114,6 @@ jobs:
           if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
 
       - name: Install Libraries on Linux
-        shell: bash
         run: |
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
@@ -122,7 +123,6 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
-        shell: bash
         run: |
           # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
           # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
@@ -151,7 +151,6 @@ jobs:
         if: matrix.os == 'windows-latest'
 
       - name: Print Go version and environment
-        shell: bash
         run: |
           printf "Using go at: $(which go)\n"
           printf "Go version: $(go version)\n"
@@ -163,29 +162,24 @@ jobs:
           env
 
       - name: Build rclone
-        shell: bash
         run: |
           make
 
       - name: Rclone version
-        shell: bash
         run: |
           rclone version
 
       - name: Run tests
-        shell: bash
         run: |
           make quicktest
         if: matrix.quicktest
 
       - name: Race test
-        shell: bash
         run: |
           make racequicktest
         if: matrix.racequicktest
 
       - name: Run librclone tests
-        shell: bash
         run: |
           make -C librclone/ctest test
           make -C librclone/ctest clean
@@ -193,14 +187,12 @@ jobs:
         if: matrix.librclonetest
 
       - name: Compile all architectures test
-        shell: bash
         run: |
           make
           make compile_all
         if: matrix.compile_all
 
       - name: Deploy built binaries
-        shell: bash
         run: |
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
           make ci_beta
@@ -219,7 +211,6 @@ jobs:
     steps:
       - name: Get runner parameters
         id: get-runner-parameters
-        shell: bash
         run: |
           echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
           echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
@@ -314,7 +305,6 @@ jobs:
           go-version: '>=1.24.0-rc.1'
 
       - name: Set global environment variables
-        shell: bash
         run: |
           echo "VERSION=$(make version)" >> $GITHUB_ENV
 
@@ -333,7 +323,6 @@ jobs:
         run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
 
       - name: arm-v7a Set environment variables
-        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
@@ -347,7 +336,6 @@ jobs:
         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
 
       - name: arm64-v8a Set environment variables
-        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
@@ -360,7 +348,6 @@ jobs:
         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
 
       - name: x86 Set environment variables
-        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
@@ -373,7 +360,6 @@ jobs:
         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
 
       - name: x64 Set environment variables
-        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV

CONTRIBUTING.md

@@ -572,3 +572,19 @@ Then, run `go build -buildmode=plugin -o PLUGIN_NAME.so .` to build the plugin.
 [Go reference](https://godoc.org/github.com/rclone/rclone/lib/plugin)
 
 [Minimal example](https://gist.github.com/terorie/21b517ee347828e899e1913efc1d684f)
+
+## Keeping a backend or command out of tree
+
+Rclone was designed to be modular so it is very easy to keep a backend
+or a command out of the main rclone source tree.
+
+So for example if you had a backend which accessed your proprietary
+systems or a command which was specialised for your needs you could
+add them out of tree.
+
+This may be easier than using a plugin and is supported on all
+platforms not just macOS and Linux.
+
+This is explained further in https://github.com/rclone/rclone_out_of_tree_example
+which has an example of an out of tree backend `ram` (which is a
+renamed version of the `memory` backend).

README.md

@@ -1,20 +1,4 @@
-<div align="center">
-<sup>Special thanks to our sponsor:</sup>
-<br>
-<br>
-<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
-  <div>
-    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
-  </div>
-  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
-  <div>
-    <sup>Visit warp.dev to learn more.</sup>
-  </div>
-</a>
-<br>
-<hr>
-</div>
-<br>
+
 
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)
@@ -55,7 +39,9 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
   * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
+  * FileLu [:page_facing_up:](https://rclone.org/filelu/)
   * Files.com [:page_facing_up:](https://rclone.org/filescom/)
+  * FlashBlade [:page_facing_up:](https://rclone.org/s3/#pure-storage-flashblade)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
@@ -80,7 +66,8 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Magalu Object Storage [:page_facing_up:](https://rclone.org/s3/#magalu)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
-  * Mega [:page_facing_up:](https://rclone.org/mega/)
+  * MEGA [:page_facing_up:](https://rclone.org/mega/)
+  * MEGA S4 Object Storage [:page_facing_up:](https://rclone.org/s3/#mega)
   * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
   * Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
@@ -110,6 +97,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
+  * Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
@@ -124,6 +112,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
   * Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
+  * Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
   * The local filesystem [:page_facing_up:](https://rclone.org/local/)
 
 Please see [the full list of all storage providers and their features](https://rclone.org/overview/)

VERSION

@@ -1 +1 @@
-v1.70.0
+v1.71.0

backend/all/all.go

@@ -14,10 +14,12 @@ import (
 	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
+	_ "github.com/rclone/rclone/backend/doi"
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/dropbox"
 	_ "github.com/rclone/rclone/backend/fichier"
 	_ "github.com/rclone/rclone/backend/filefabric"
+	_ "github.com/rclone/rclone/backend/filelu"
 	_ "github.com/rclone/rclone/backend/filescom"
 	_ "github.com/rclone/rclone/backend/ftp"
 	_ "github.com/rclone/rclone/backend/gofile"

backend/azureblob/azureblob.go

@@ -72,6 +72,7 @@ const (
 	emulatorAccount      = "devstoreaccount1"
 	emulatorAccountKey   = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 	emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
+	sasCopyValidity      = time.Hour // how long SAS should last when doing server side copy
 )
 
 var (
@@ -559,6 +560,11 @@ type Fs struct {
 	pacer         *fs.Pacer                    // To pace and retry the API calls
 	uploadToken   *pacer.TokenDispenser        // control concurrency
 	publicAccess  container.PublicAccessType   // Container Public Access Level
+
+	// user delegation cache
+	userDelegationMu     sync.Mutex
+	userDelegation       *service.UserDelegationCredential
+	userDelegationExpiry time.Time
 }
 
 // Object describes an azure object
@@ -612,6 +618,9 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
 	containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
+	if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
+		containerPath = containerPath[:len(containerPath)-1]
+	}
 	return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
 }
 
@@ -928,6 +937,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 	case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
 		// User with username and password
+		//nolint:staticcheck // this is deprecated due to Azure policy
 		options := azidentity.UsernamePasswordCredentialOptions{
 			ClientOptions: policyClientOptions,
 		}
@@ -980,6 +990,38 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
+	case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
+		// Workload Identity based authentication
+		var options azidentity.ManagedIdentityCredentialOptions
+		options.ID = azidentity.ClientID(opt.MSIClientID)
+
+		msiCred, err := azidentity.NewManagedIdentityCredential(&options)
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
+		}
+
+		getClientAssertions := func(context.Context) (string, error) {
+			token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
+				Scopes: []string{"api://AzureADTokenExchange"},
+			})
+
+			if err != nil {
+				return "", fmt.Errorf("failed to acquire MSI token: %w", err)
+			}
+
+			return token.Token, nil
+		}
+
+		assertOpts := &azidentity.ClientAssertionCredentialOptions{}
+		f.cred, err = azidentity.NewClientAssertionCredential(
+			opt.Tenant,
+			opt.ClientID,
+			getClientAssertions,
+			assertOpts)
+
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
+		}
 	case opt.UseAZ:
 		var options = azidentity.AzureCLICredentialOptions{}
 		f.cred, err = azidentity.NewAzureCLICredential(&options)
@@ -1213,7 +1255,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 					continue
 				}
 				// process directory markers as directories
-				remote = strings.TrimRight(remote, "/")
+				remote, _ = strings.CutSuffix(remote, "/")
 			}
 			remote = remote[len(prefix):]
 			if addContainer {
@@ -1534,7 +1576,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote = strings.TrimRight(remote, "/")
+	remote, _ = strings.CutSuffix(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
 		dir = ""
@@ -1684,6 +1726,38 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.deleteContainer(ctx, container)
 }
 
+// Get a user delegation which is valid for at least sasCopyValidity
+//
+// This value is cached in f
+func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCredential, error) {
+	f.userDelegationMu.Lock()
+	defer f.userDelegationMu.Unlock()
+
+	if f.userDelegation != nil && time.Until(f.userDelegationExpiry) > sasCopyValidity {
+		return f.userDelegation, nil
+	}
+
+	// Validity window
+	start := time.Now().UTC()
+	expiry := start.Add(2 * sasCopyValidity)
+	startStr := start.Format(time.RFC3339)
+	expiryStr := expiry.Format(time.RFC3339)
+
+	// Acquire user delegation key from the service client
+	info := service.KeyInfo{
+		Start:  &startStr,
+		Expiry: &expiryStr,
+	}
+	userDelegationKey, err := f.svc.GetUserDelegationCredential(ctx, info, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get user delegation key: %w", err)
+	}
+
+	f.userDelegation = userDelegationKey
+	f.userDelegationExpiry = expiry
+	return f.userDelegation, nil
+}
+
 // getAuth gets auth to copy o.
 //
 // tokenOK is used to signal that token based auth (Microsoft Entra
@@ -1695,7 +1769,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // URL (not a SAS) and token will be empty.
 //
 // If tokenOK is true it may also return a token for the auth.
-func (o *Object) getAuth(ctx context.Context, tokenOK bool, noAuth bool) (srcURL string, token *string, err error) {
+func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err error) {
 	f := o.fs
 	srcBlobSVC := o.getBlobSVC()
 	srcURL = srcBlobSVC.URL()
@@ -1704,29 +1778,47 @@ func (o *Object) getAuth(ctx context.Context, tokenOK bool, noAuth bool) (srcURL
 	case noAuth:
 		// If same storage account then no auth needed
 	case f.cred != nil:
-		if !tokenOK {
-			return srcURL, token, errors.New("not supported: Microsoft Entra ID")
-		}
-		options := policy.TokenRequestOptions{}
-		accessToken, err := f.cred.GetToken(ctx, options)
+		// Generate a User Delegation SAS URL using Azure AD credentials
+		userDelegationKey, err := f.getUserDelegation(ctx)
 		if err != nil {
-			return srcURL, token, fmt.Errorf("failed to create access token: %w", err)
+			return "", fmt.Errorf("sas creation: %w", err)
 		}
-		token = &accessToken.Token
+
+		// Build the SAS values
+		perms := sas.BlobPermissions{Read: true}
+		container, containerPath := o.split()
+		start := time.Now().UTC()
+		expiry := start.Add(sasCopyValidity)
+		vals := sas.BlobSignatureValues{
+			StartTime:     start,
+			ExpiryTime:    expiry,
+			Permissions:   perms.String(),
+			ContainerName: container,
+			BlobName:      containerPath,
+		}
+
+		// Sign with the delegation key
+		queryParameters, err := vals.SignWithUserDelegation(userDelegationKey)
+		if err != nil {
+			return "", fmt.Errorf("signing SAS with user delegation failed: %w", err)
+		}
+
+		// Append the SAS to the URL
+		srcURL = srcBlobSVC.URL() + "?" + queryParameters.Encode()
 	case f.sharedKeyCred != nil:
 		// Generate a short lived SAS URL if using shared key credentials
-		expiry := time.Now().Add(time.Hour)
+		expiry := time.Now().Add(sasCopyValidity)
 		sasOptions := blob.GetSASURLOptions{}
 		srcURL, err = srcBlobSVC.GetSASURL(sas.BlobPermissions{Read: true}, expiry, &sasOptions)
 		if err != nil {
-			return srcURL, token, fmt.Errorf("failed to create SAS URL: %w", err)
+			return srcURL, fmt.Errorf("failed to create SAS URL: %w", err)
 		}
 	case f.anonymous || f.opt.SASURL != "":
 		// If using a SASURL or anonymous, no need for any extra auth
 	default:
-		return srcURL, token, errors.New("unknown authentication type")
+		return srcURL, errors.New("unknown authentication type")
 	}
-	return srcURL, token, nil
+	return srcURL, nil
 }
 
 // Do multipart parallel copy.
@@ -1747,7 +1839,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
 	o.fs = f
 	o.remote = remote
 
-	srcURL, token, err := src.getAuth(ctx, true, false)
+	srcURL, err := src.getAuth(ctx, false)
 	if err != nil {
 		return nil, fmt.Errorf("multipart copy: %w", err)
 	}
@@ -1768,7 +1860,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
 	var (
 		srcSize  = src.size
 		partSize = int64(chunksize.Calculator(o, src.size, blockblob.MaxBlocks, f.opt.ChunkSize))
-		numParts = (srcSize-1)/partSize + 1
+		numParts = (srcSize + partSize - 1) / partSize
 		blockIDs = make([]string, numParts) // list of blocks for finalize
 		g, gCtx  = errgroup.WithContext(ctx)
 		checker  = newCheckForInvalidBlockOrBlob("copy", o)
@@ -1791,7 +1883,8 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
 					Count:  partSize,
 				},
 				// Specifies the authorization scheme and signature for the copy source.
-				CopySourceAuthorization: token,
+				// We use SAS URLs as this doesn't seem to work always
+				// CopySourceAuthorization: token,
 				// CPKInfo *blob.CPKInfo
 				// CPKScopeInfo *blob.CPKScopeInfo
 			}
@@ -1861,7 +1954,7 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
 	dstBlobSVC := f.getBlobSVC(dstContainer, dstPath)
 
 	// Get the source auth - none needed for same storage account
-	srcURL, _, err := src.getAuth(ctx, false, f == src.fs)
+	srcURL, err := src.getAuth(ctx, f == src.fs)
 	if err != nil {
 		return nil, fmt.Errorf("single part copy: source auth: %w", err)
 	}
@@ -2176,11 +2269,6 @@ func (o *Object) getTags() (tags map[string]string) {
 // getBlobSVC creates a blob client
 func (o *Object) getBlobSVC() *blob.Client {
 	container, directory := o.split()
-	// If we are trying to remove an all / directory marker then
-	// this will have one / too many now.
-	if bucket.IsAllSlashes(o.remote) {
-		directory = strings.TrimSuffix(directory, "/")
-	}
 	return o.fs.getBlobSVC(container, directory)
 }
 
@@ -2863,6 +2951,9 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 			return ui, err
 		}
 	}
+	// if ui.isDirMarker && strings.HasSuffix(containerPath, "//") {
+	// 	containerPath = containerPath[:len(containerPath)-1]
+	// }
 
 	// Update Mod time
 	o.updateMetadataWithModTime(src.ModTime(ctx))

backend/azurefiles/azurefiles.go

@@ -516,6 +516,7 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		}
 	case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
 		// User with username and password
+		//nolint:staticcheck // this is deprecated due to Azure policy
 		options := azidentity.UsernamePasswordCredentialOptions{
 			ClientOptions: policyClientOptions,
 		}
@@ -568,6 +569,38 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
+	case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
+		// Workload Identity based authentication
+		var options azidentity.ManagedIdentityCredentialOptions
+		options.ID = azidentity.ClientID(opt.MSIClientID)
+
+		msiCred, err := azidentity.NewManagedIdentityCredential(&options)
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
+		}
+
+		getClientAssertions := func(context.Context) (string, error) {
+			token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
+				Scopes: []string{"api://AzureADTokenExchange"},
+			})
+
+			if err != nil {
+				return "", fmt.Errorf("failed to acquire MSI token: %w", err)
+			}
+
+			return token.Token, nil
+		}
+
+		assertOpts := &azidentity.ClientAssertionCredentialOptions{}
+		cred, err = azidentity.NewClientAssertionCredential(
+			opt.Tenant,
+			opt.ClientID,
+			getClientAssertions,
+			assertOpts)
+
+		if err != nil {
+			return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
+		}
 	default:
 		return nil, errors.New("no authentication method configured")
 	}
@@ -921,7 +954,7 @@ func (o *Object) setMetadata(resp *file.GetPropertiesResponse) {
 	}
 }
 
-// readMetaData gets the metadata if it hasn't already been fetched
+// getMetadata gets the metadata if it hasn't already been fetched
 func (o *Object) getMetadata(ctx context.Context) error {
 	resp, err := o.fileClient().GetProperties(ctx, nil)
 	if err != nil {

backend/b2/b2.go

@@ -1673,6 +1673,21 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
 			return o.getMetaDataListing(ctx)
 		}
 	}
+
+	// If using versionAt we need to list the find the correct version.
+	if o.fs.opt.VersionAt.IsSet() {
+		info, err := o.getMetaDataListing(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		if info.Action == "hide" {
+			// Rerturn object not found error if the current version is deleted.
+			return nil, fs.ErrorObjectNotFound
+		}
+		return info, nil
+	}
+
 	_, info, err = o.getOrHead(ctx, "HEAD", nil)
 	return info, err
 }
@@ -1883,9 +1898,14 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
 	// length read from the listing.
+	// Additionally, the official examples return S3 headers
+	// instead of native, i.e. no file ID, use ones from listing.
 	if info.Size < 0 {
 		info.Size = o.size
 	}
+	if info.ID == "" {
+		info.ID = o.id
+	}
 	return resp, info, nil
 }
 

backend/b2/b2_internal_test.go

@@ -446,14 +446,14 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 				t.Run("List", func(t *testing.T) {
 					fstest.CheckListing(t, f, test.want)
 				})
-				// b2 NewObject doesn't work with VersionAt
-				//t.Run("NewObject", func(t *testing.T) {
-				//	gotObj, gotErr := f.NewObject(ctx, fileName)
-				//	assert.Equal(t, test.wantErr, gotErr)
-				//	if gotErr == nil {
-				//		assert.Equal(t, test.wantSize, gotObj.Size())
-				//	}
-				//})
+
+				t.Run("NewObject", func(t *testing.T) {
+					gotObj, gotErr := f.NewObject(ctx, fileName)
+					assert.Equal(t, test.wantErr, gotErr)
+					if gotErr == nil {
+						assert.Equal(t, test.wantSize, gotObj.Size())
+					}
+				})
 			})
 		}
 	})

backend/chunker/chunker.go

@@ -1861,6 +1861,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // baseMove chains to the wrapped Move or simulates it by Copy+Delete
 func (f *Fs) baseMove(ctx context.Context, src fs.Object, remote string, delMode int) (fs.Object, error) {
+	ctx, ci := fs.AddConfig(ctx)
+	ci.NameTransform = nil // ensure operations.Move does not double-transform here
 	var (
 		dest fs.Object
 		err  error

backend/combine/combine.go

@@ -858,7 +858,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
 		}
 		return wrappedCallback(entries)
 	}
-	return listP(ctx, dir, wrappedCallback)
+	return listP(ctx, uRemote, wrappedCallback)
 }
 
 // ListR lists the objects and directories of the Fs starting

backend/doi/api/dataversetypes.go

@@ -0,0 +1,38 @@
+// Type definitions specific to Dataverse
+
+package api
+
+// DataverseDatasetResponse is returned by the Dataverse dataset API
+type DataverseDatasetResponse struct {
+	Status string           `json:"status"`
+	Data   DataverseDataset `json:"data"`
+}
+
+// DataverseDataset is the representation of a dataset
+type DataverseDataset struct {
+	LatestVersion DataverseDatasetVersion `json:"latestVersion"`
+}
+
+// DataverseDatasetVersion is the representation of a dataset version
+type DataverseDatasetVersion struct {
+	LastUpdateTime string          `json:"lastUpdateTime"`
+	Files          []DataverseFile `json:"files"`
+}
+
+// DataverseFile is the representation of a file found in a dataset
+type DataverseFile struct {
+	DirectoryLabel string            `json:"directoryLabel"`
+	DataFile       DataverseDataFile `json:"dataFile"`
+}
+
+// DataverseDataFile represents file metadata details
+type DataverseDataFile struct {
+	ID                 int64  `json:"id"`
+	Filename           string `json:"filename"`
+	ContentType        string `json:"contentType"`
+	FileSize           int64  `json:"filesize"`
+	OriginalFileFormat string `json:"originalFileFormat"`
+	OriginalFileSize   int64  `json:"originalFileSize"`
+	OriginalFileName   string `json:"originalFileName"`
+	MD5                string `json:"md5"`
+}

backend/doi/api/inveniotypes.go

@@ -0,0 +1,33 @@
+// Type definitions specific to InvenioRDM
+
+package api
+
+// InvenioRecordResponse is the representation of a record stored in InvenioRDM
+type InvenioRecordResponse struct {
+	Links InvenioRecordResponseLinks `json:"links"`
+}
+
+// InvenioRecordResponseLinks represents a record's links
+type InvenioRecordResponseLinks struct {
+	Self string `json:"self"`
+}
+
+// InvenioFilesResponse is the representation of a record's files
+type InvenioFilesResponse struct {
+	Entries []InvenioFilesResponseEntry `json:"entries"`
+}
+
+// InvenioFilesResponseEntry is the representation of a file entry
+type InvenioFilesResponseEntry struct {
+	Key      string                         `json:"key"`
+	Checksum string                         `json:"checksum"`
+	Size     int64                          `json:"size"`
+	Updated  string                         `json:"updated"`
+	MimeType string                         `json:"mimetype"`
+	Links    InvenioFilesResponseEntryLinks `json:"links"`
+}
+
+// InvenioFilesResponseEntryLinks represents file links details
+type InvenioFilesResponseEntryLinks struct {
+	Content string `json:"content"`
+}

backend/doi/api/types.go

@@ -0,0 +1,26 @@
+// Package api has general type definitions for doi
+package api
+
+// DoiResolverResponse is returned by the DOI resolver API
+//
+// Reference: https://www.doi.org/the-identifier/resources/factsheets/doi-resolution-documentation
+type DoiResolverResponse struct {
+	ResponseCode int                        `json:"responseCode"`
+	Handle       string                     `json:"handle"`
+	Values       []DoiResolverResponseValue `json:"values"`
+}
+
+// DoiResolverResponseValue is a single handle record value
+type DoiResolverResponseValue struct {
+	Index     int                          `json:"index"`
+	Type      string                       `json:"type"`
+	Data      DoiResolverResponseValueData `json:"data"`
+	TTL       int                          `json:"ttl"`
+	Timestamp string                       `json:"timestamp"`
+}
+
+// DoiResolverResponseValueData is the data held in a handle value
+type DoiResolverResponseValueData struct {
+	Format string `json:"format"`
+	Value  any    `json:"value"`
+}

backend/doi/dataverse.go

@@ -0,0 +1,112 @@
+// Implementation for Dataverse
+
+package doi
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/doi/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// Returns true if resolvedURL is likely a DOI hosted on a Dataverse intallation
+func activateDataverse(resolvedURL *url.URL) (isActive bool) {
+	queryValues := resolvedURL.Query()
+	persistentID := queryValues.Get("persistentId")
+	return persistentID != ""
+}
+
+// Resolve the main API endpoint for a DOI hosted on a Dataverse installation
+func resolveDataverseEndpoint(resolvedURL *url.URL) (provider Provider, endpoint *url.URL, err error) {
+	queryValues := resolvedURL.Query()
+	persistentID := queryValues.Get("persistentId")
+
+	query := url.Values{}
+	query.Add("persistentId", persistentID)
+	endpointURL := resolvedURL.ResolveReference(&url.URL{Path: "/api/datasets/:persistentId/", RawQuery: query.Encode()})
+
+	return Dataverse, endpointURL, nil
+}
+
+// dataverseProvider implements the doiProvider interface for Dataverse installations
+type dataverseProvider struct {
+	f *Fs
+}
+
+// ListEntries returns the full list of entries found at the remote, regardless of root
+func (dp *dataverseProvider) ListEntries(ctx context.Context) (entries []*Object, err error) {
+	// Use the cache if populated
+	cachedEntries, found := dp.f.cache.GetMaybe("files")
+	if found {
+		parsedEntries, ok := cachedEntries.([]Object)
+		if ok {
+			for _, entry := range parsedEntries {
+				newEntry := entry
+				entries = append(entries, &newEntry)
+			}
+			return entries, nil
+		}
+	}
+
+	filesURL := dp.f.endpoint
+	var res *http.Response
+	var result api.DataverseDatasetResponse
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       strings.TrimLeft(filesURL.EscapedPath(), "/"),
+		Parameters: filesURL.Query(),
+	}
+	err = dp.f.pacer.Call(func() (bool, error) {
+		res, err = dp.f.srv.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("readDir failed: %w", err)
+	}
+	modTime, modTimeErr := time.Parse(time.RFC3339, result.Data.LatestVersion.LastUpdateTime)
+	if modTimeErr != nil {
+		fs.Logf(dp.f, "error: could not parse last update time %v", modTimeErr)
+		modTime = timeUnset
+	}
+	for _, file := range result.Data.LatestVersion.Files {
+		contentURLPath := fmt.Sprintf("/api/access/datafile/%d", file.DataFile.ID)
+		query := url.Values{}
+		query.Add("format", "original")
+		contentURL := dp.f.endpoint.ResolveReference(&url.URL{Path: contentURLPath, RawQuery: query.Encode()})
+		entry := &Object{
+			fs:          dp.f,
+			remote:      path.Join(file.DirectoryLabel, file.DataFile.Filename),
+			contentURL:  contentURL.String(),
+			size:        file.DataFile.FileSize,
+			modTime:     modTime,
+			md5:         file.DataFile.MD5,
+			contentType: file.DataFile.ContentType,
+		}
+		if file.DataFile.OriginalFileName != "" {
+			entry.remote = path.Join(file.DirectoryLabel, file.DataFile.OriginalFileName)
+			entry.size = file.DataFile.OriginalFileSize
+			entry.contentType = file.DataFile.OriginalFileFormat
+		}
+		entries = append(entries, entry)
+	}
+	// Populate the cache
+	cacheEntries := []Object{}
+	for _, entry := range entries {
+		cacheEntries = append(cacheEntries, *entry)
+	}
+	dp.f.cache.Put("files", cacheEntries)
+	return entries, nil
+}
+
+func newDataverseProvider(f *Fs) doiProvider {
+	return &dataverseProvider{
+		f: f,
+	}
+}

backend/doi/doi.go

@@ -0,0 +1,649 @@
+// Package doi provides a filesystem interface for digital objects identified by DOIs.
+//
+// See: https://www.doi.org/the-identifier/what-is-a-doi/
+package doi
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/doi/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/cache"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+const (
+	// the URL of the DOI resolver
+	//
+	// Reference: https://www.doi.org/the-identifier/resources/factsheets/doi-resolution-documentation
+	doiResolverAPIURL = "https://doi.org/api"
+	minSleep          = 10 * time.Millisecond
+	maxSleep          = 2 * time.Second
+	decayConstant     = 2 // bigger for slower decay, exponential
+)
+
+var (
+	errorReadOnly = errors.New("doi remotes are read only")
+	timeUnset     = time.Unix(0, 0)
+)
+
+func init() {
+	fsi := &fs.RegInfo{
+		Name:        "doi",
+		Description: "DOI datasets",
+		NewFs:       NewFs,
+		CommandHelp: commandHelp,
+		Options: []fs.Option{{
+			Name:     "doi",
+			Help:     "The DOI or the doi.org URL.",
+			Required: true,
+		}, {
+			Name: fs.ConfigProvider,
+			Help: `DOI provider.
+
+The DOI provider can be set when rclone does not automatically recognize a supported DOI provider.`,
+			Examples: []fs.OptionExample{
+				{
+					Value: "auto",
+					Help:  "Auto-detect provider",
+				},
+				{
+					Value: string(Zenodo),
+					Help:  "Zenodo",
+				}, {
+					Value: string(Dataverse),
+					Help:  "Dataverse",
+				}, {
+					Value: string(Invenio),
+					Help:  "Invenio",
+				}},
+			Required: false,
+			Advanced: true,
+		}, {
+			Name: "doi_resolver_api_url",
+			Help: `The URL of the DOI resolver API to use.
+
+The DOI resolver can be set for testing or for cases when the the canonical DOI resolver API cannot be used.
+
+Defaults to "https://doi.org/api".`,
+			Required: false,
+			Advanced: true,
+		}},
+	}
+	fs.Register(fsi)
+}
+
+// Provider defines the type of provider hosting the DOI
+type Provider string
+
+const (
+	// Zenodo provider, see https://zenodo.org
+	Zenodo Provider = "zenodo"
+	// Dataverse provider, see https://dataverse.harvard.edu
+	Dataverse Provider = "dataverse"
+	// Invenio provider, see https://inveniordm.docs.cern.ch
+	Invenio Provider = "invenio"
+)
+
+// Options defines the configuration for this backend
+type Options struct {
+	Doi               string `config:"doi"`                  // The DOI, a digital identifier of an object, usually a dataset
+	Provider          string `config:"provider"`             // The DOI provider
+	DoiResolverAPIURL string `config:"doi_resolver_api_url"` // The URL of the DOI resolver API to use.
+}
+
+// Fs stores the interface to the remote HTTP files
+type Fs struct {
+	name        string         // name of this remote
+	root        string         // the path we are working on
+	provider    Provider       // the DOI provider
+	doiProvider doiProvider    // the interface used to interact with the DOI provider
+	features    *fs.Features   // optional features
+	opt         Options        // options for this backend
+	ci          *fs.ConfigInfo // global config
+	endpoint    *url.URL       // the main API endpoint for this remote
+	endpointURL string         // endpoint as a string
+	srv         *rest.Client   // the connection to the server
+	pacer       *fs.Pacer      // pacer for API calls
+	cache       *cache.Cache   // a cache for the remote metadata
+}
+
+// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
+type Object struct {
+	fs          *Fs       // what this object is part of
+	remote      string    // the remote path
+	contentURL  string    // the URL where the contents of the file can be downloaded
+	size        int64     // size of the object
+	modTime     time.Time // modification time of the object
+	contentType string    // content type of the object
+	md5         string    // MD5 hash of the object content
+}
+
+// doiProvider is the interface used to list objects in a DOI
+type doiProvider interface {
+	// ListEntries returns the full list of entries found at the remote, regardless of root
+	ListEntries(ctx context.Context) (entries []*Object, err error)
+}
+
+// Parse the input string as a DOI
+// Examples:
+// 10.1000/182 -> 10.1000/182
+// https://doi.org/10.1000/182 -> 10.1000/182
+// doi:10.1000/182 -> 10.1000/182
+func parseDoi(doi string) string {
+	doiURL, err := url.Parse(doi)
+	if err != nil {
+		return doi
+	}
+	if doiURL.Scheme == "doi" {
+		return strings.TrimLeft(strings.TrimPrefix(doi, "doi:"), "/")
+	}
+	if strings.HasSuffix(doiURL.Hostname(), "doi.org") {
+		return strings.TrimLeft(doiURL.Path, "/")
+	}
+	return doi
+}
+
+// Resolve a DOI to a URL
+// Reference: https://www.doi.org/the-identifier/resources/factsheets/doi-resolution-documentation
+func resolveDoiURL(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, opt *Options) (doiURL *url.URL, err error) {
+	resolverURL := opt.DoiResolverAPIURL
+	if resolverURL == "" {
+		resolverURL = doiResolverAPIURL
+	}
+
+	var result api.DoiResolverResponse
+	params := url.Values{}
+	params.Add("index", "1")
+	opts := rest.Opts{
+		Method:     "GET",
+		RootURL:    resolverURL,
+		Path:       "/handles/" + opt.Doi,
+		Parameters: params,
+	}
+	err = pacer.Call(func() (bool, error) {
+		res, err := srv.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	if result.ResponseCode != 1 {
+		return nil, fmt.Errorf("could not resolve DOI (error code %d)", result.ResponseCode)
+	}
+	resolvedURLStr := ""
+	for _, value := range result.Values {
+		if value.Type == "URL" && value.Data.Format == "string" {
+			valueStr, ok := value.Data.Value.(string)
+			if !ok {
+				return nil, fmt.Errorf("could not resolve DOI (incorrect response format)")
+			}
+			resolvedURLStr = valueStr
+		}
+	}
+	resolvedURL, err := url.Parse(resolvedURLStr)
+	if err != nil {
+		return nil, err
+	}
+	return resolvedURL, nil
+}
+
+// Resolve the passed configuration into a provider and enpoint
+func resolveEndpoint(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, opt *Options) (provider Provider, endpoint *url.URL, err error) {
+	resolvedURL, err := resolveDoiURL(ctx, srv, pacer, opt)
+	if err != nil {
+		return "", nil, err
+	}
+
+	switch opt.Provider {
+	case string(Dataverse):
+		return resolveDataverseEndpoint(resolvedURL)
+	case string(Invenio):
+		return resolveInvenioEndpoint(ctx, srv, pacer, resolvedURL)
+	case string(Zenodo):
+		return resolveZenodoEndpoint(ctx, srv, pacer, resolvedURL, opt.Doi)
+	}
+
+	hostname := strings.ToLower(resolvedURL.Hostname())
+	if hostname == "dataverse.harvard.edu" || activateDataverse(resolvedURL) {
+		return resolveDataverseEndpoint(resolvedURL)
+	}
+	if hostname == "zenodo.org" || strings.HasSuffix(hostname, ".zenodo.org") {
+		return resolveZenodoEndpoint(ctx, srv, pacer, resolvedURL, opt.Doi)
+	}
+	if activateInvenio(ctx, srv, pacer, resolvedURL) {
+		return resolveInvenioEndpoint(ctx, srv, pacer, resolvedURL)
+	}
+
+	return "", nil, fmt.Errorf("provider '%s' is not supported", resolvedURL.Hostname())
+}
+
+// Make the http connection from the passed options
+func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err error) {
+	provider, endpoint, err := resolveEndpoint(ctx, f.srv, f.pacer, opt)
+	if err != nil {
+		return false, err
+	}
+
+	// Update f with the new parameters
+	f.srv.SetRoot(endpoint.ResolveReference(&url.URL{Path: "/"}).String())
+	f.endpoint = endpoint
+	f.endpointURL = endpoint.String()
+	f.provider = provider
+	f.opt.Provider = string(provider)
+
+	switch f.provider {
+	case Dataverse:
+		f.doiProvider = newDataverseProvider(f)
+	case Invenio, Zenodo:
+		f.doiProvider = newInvenioProvider(f)
+	default:
+		return false, fmt.Errorf("provider type '%s' not supported", f.provider)
+	}
+
+	// Determine if the root is a file
+	entries, err := f.doiProvider.ListEntries(ctx)
+	if err != nil {
+		return false, err
+	}
+	for _, entry := range entries {
+		if entry.remote == f.root {
+			isFile = true
+			break
+		}
+	}
+	return isFile, nil
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	429, // Too Many Requests.
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+	509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this res and err
+// deserve to be retried. It returns the err as a convenience.
+func shouldRetry(ctx context.Context, res *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(res, retryErrorCodes), err
+}
+
+// NewFs creates a new Fs object from the name and root. It connects to
+// the host specified in the config file.
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	root = strings.Trim(root, "/")
+
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	opt.Doi = parseDoi(opt.Doi)
+
+	client := fshttp.NewClient(ctx)
+	ci := fs.GetConfig(ctx)
+	f := &Fs{
+		name:  name,
+		root:  root,
+		opt:   *opt,
+		ci:    ci,
+		srv:   rest.NewClient(client),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		cache: cache.New(),
+	}
+	f.features = (&fs.Features{
+		CanHaveEmptyDirectories: true,
+	}).Fill(ctx, f)
+
+	isFile, err := f.httpConnection(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	if isFile {
+		// return an error with an fs which points to the parent
+		newRoot := path.Dir(f.root)
+		if newRoot == "." {
+			newRoot = ""
+		}
+		f.root = newRoot
+		return f, fs.ErrorIsFile
+	}
+
+	return f, nil
+}
+
+// Name returns the configured name of the file system
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root for the filesystem
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns the URL for the filesystem
+func (f *Fs) String() string {
+	return fmt.Sprintf("DOI %s", f.opt.Doi)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Precision is the remote http file system's modtime precision, which we have no way of knowing. We estimate at 1s
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Hashes returns hash.HashNone to indicate remote hashing is unavailable
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hash.MD5)
+	// return hash.Set(hash.None)
+}
+
+// Mkdir makes the root directory of the Fs object
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return errorReadOnly
+}
+
+// Remove a remote http file object
+func (o *Object) Remove(ctx context.Context) error {
+	return errorReadOnly
+}
+
+// Rmdir removes the root directory of the Fs object
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	return errorReadOnly
+}
+
+// NewObject creates a new remote http file object
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	entries, err := f.doiProvider.ListEntries(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	remoteFullPath := remote
+	if f.root != "" {
+		remoteFullPath = path.Join(f.root, remote)
+	}
+
+	for _, entry := range entries {
+		if entry.Remote() == remoteFullPath {
+			return entry, nil
+		}
+	}
+
+	return nil, fs.ErrorObjectNotFound
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	fileEntries, err := f.doiProvider.ListEntries(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("error listing %q: %w", dir, err)
+	}
+
+	fullDir := path.Join(f.root, dir)
+	if fullDir != "" {
+		fullDir += "/"
+	}
+
+	dirPaths := map[string]bool{}
+	for _, entry := range fileEntries {
+		// First, filter out files not in `fullDir`
+		if !strings.HasPrefix(entry.remote, fullDir) {
+			continue
+		}
+		// Then, find entries in subfolers
+		remotePath := entry.remote
+		if fullDir != "" {
+			remotePath = strings.TrimLeft(strings.TrimPrefix(remotePath, fullDir), "/")
+		}
+		parts := strings.SplitN(remotePath, "/", 2)
+		if len(parts) == 1 {
+			newEntry := *entry
+			newEntry.remote = path.Join(dir, remotePath)
+			entries = append(entries, &newEntry)
+		} else {
+			dirPaths[path.Join(dir, parts[0])] = true
+		}
+	}
+
+	for dirPath := range dirPaths {
+		entry := fs.NewDir(dirPath, time.Time{})
+		entries = append(entries, entry)
+	}
+
+	return entries, nil
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return nil, errorReadOnly
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return nil, errorReadOnly
+}
+
+// Fs is the filesystem this remote http file object is located within
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// String returns the URL to the remote HTTP file
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote the name of the remote HTTP file, relative to the fs root
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns "" since HTTP (in Go or OpenSSH) doesn't support remote calculation of hashes
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.MD5 {
+		return "", hash.ErrUnsupported
+	}
+	return o.md5, nil
+}
+
+// Size returns the size in bytes of the remote http file
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// ModTime returns the modification time of the remote http file
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// SetModTime sets the modification and access time to the specified time
+//
+// it also updates the info field
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	return errorReadOnly
+}
+
+// Storable returns whether the remote http file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc.)
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open a remote http file object for reading. Seek is supported
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	fs.FixRangeOption(options, o.size)
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: o.contentURL,
+		Options: options,
+	}
+	var res *http.Response
+	err = o.fs.pacer.Call(func() (bool, error) {
+		res, err = o.fs.srv.Call(ctx, &opts)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Open failed: %w", err)
+	}
+
+	// Handle non-compliant redirects
+	if res.Header.Get("Location") != "" {
+		newURL, err := res.Location()
+		if err == nil {
+			opts.RootURL = newURL.String()
+			err = o.fs.pacer.Call(func() (bool, error) {
+				res, err = o.fs.srv.Call(ctx, &opts)
+				return shouldRetry(ctx, res, err)
+			})
+			if err != nil {
+				return nil, fmt.Errorf("Open failed: %w", err)
+			}
+		}
+	}
+
+	return res.Body, nil
+}
+
+// Update in to the object with the modTime given of the given size
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	return errorReadOnly
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.contentType
+}
+
+var commandHelp = []fs.CommandHelp{{
+	Name:  "metadata",
+	Short: "Show metadata about the DOI.",
+	Long: `This command returns a JSON object with some information about the DOI.
+
+    rclone backend medatadata doi: 
+
+It returns a JSON object representing metadata about the DOI.
+`,
+}, {
+	Name:  "set",
+	Short: "Set command for updating the config parameters.",
+	Long: `This set command can be used to update the config parameters
+for a running doi backend.
+
+Usage Examples:
+
+    rclone backend set doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
+    rclone rc backend/command command=set fs=doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
+    rclone rc backend/command command=set fs=doi: -o doi=NEW_DOI
+
+The option keys are named as they are in the config file.
+
+This rebuilds the connection to the doi backend when it is called with
+the new parameters. Only new parameters need be passed as the values
+will default to those currently in use.
+
+It doesn't return anything.
+`,
+}}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "metadata":
+		return f.ShowMetadata(ctx)
+	case "set":
+		newOpt := f.opt
+		err := configstruct.Set(configmap.Simple(opt), &newOpt)
+		if err != nil {
+			return nil, fmt.Errorf("reading config: %w", err)
+		}
+		_, err = f.httpConnection(ctx, &newOpt)
+		if err != nil {
+			return nil, fmt.Errorf("updating session: %w", err)
+		}
+		f.opt = newOpt
+		keys := []string{}
+		for k := range opt {
+			keys = append(keys, k)
+		}
+		fs.Logf(f, "Updated config values: %s", strings.Join(keys, ", "))
+		return nil, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
+// ShowMetadata returns some metadata about the corresponding DOI
+func (f *Fs) ShowMetadata(ctx context.Context) (metadata interface{}, err error) {
+	doiURL, err := url.Parse("https://doi.org/" + f.opt.Doi)
+	if err != nil {
+		return nil, err
+	}
+
+	info := map[string]any{}
+	info["DOI"] = f.opt.Doi
+	info["URL"] = doiURL.String()
+	info["metadataURL"] = f.endpointURL
+	info["provider"] = f.provider
+	return info, nil
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs          = (*Fs)(nil)
+	_ fs.PutStreamer = (*Fs)(nil)
+	_ fs.Commander   = (*Fs)(nil)
+	_ fs.Object      = (*Object)(nil)
+	_ fs.MimeTyper   = (*Object)(nil)
+)

backend/doi/doi_internal_test.go

@@ -0,0 +1,260 @@
+package doi
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"sort"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/backend/doi/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var remoteName = "TestDoi"
+
+func TestParseDoi(t *testing.T) {
+	// 10.1000/182 -> 10.1000/182
+	doi := "10.1000/182"
+	parsed := parseDoi(doi)
+	assert.Equal(t, "10.1000/182", parsed)
+
+	// https://doi.org/10.1000/182 -> 10.1000/182
+	doi = "https://doi.org/10.1000/182"
+	parsed = parseDoi(doi)
+	assert.Equal(t, "10.1000/182", parsed)
+
+	// https://dx.doi.org/10.1000/182 -> 10.1000/182
+	doi = "https://dxdoi.org/10.1000/182"
+	parsed = parseDoi(doi)
+	assert.Equal(t, "10.1000/182", parsed)
+
+	// doi:10.1000/182 -> 10.1000/182
+	doi = "doi:10.1000/182"
+	parsed = parseDoi(doi)
+	assert.Equal(t, "10.1000/182", parsed)
+
+	// doi://10.1000/182 -> 10.1000/182
+	doi = "doi://10.1000/182"
+	parsed = parseDoi(doi)
+	assert.Equal(t, "10.1000/182", parsed)
+}
+
+// prepareMockDoiResolverServer prepares a test server to resolve DOIs
+func prepareMockDoiResolverServer(t *testing.T, resolvedURL string) (doiResolverAPIURL string) {
+	mux := http.NewServeMux()
+
+	// Handle requests for resolving DOIs
+	mux.HandleFunc("GET /api/handles/{handle...}", func(w http.ResponseWriter, r *http.Request) {
+		// Check that we are resolving a DOI
+		handle := strings.TrimPrefix(r.URL.Path, "/api/handles/")
+		assert.NotEmpty(t, handle)
+		index := r.URL.Query().Get("index")
+		assert.Equal(t, "1", index)
+
+		// Return the most basic response
+		result := api.DoiResolverResponse{
+			ResponseCode: 1,
+			Handle:       handle,
+			Values: []api.DoiResolverResponseValue{
+				{
+					Index: 1,
+					Type:  "URL",
+					Data: api.DoiResolverResponseValueData{
+						Format: "string",
+						Value:  resolvedURL,
+					},
+				},
+			},
+		}
+		resultBytes, err := json.Marshal(result)
+		require.NoError(t, err)
+		w.Header().Add("Content-Type", "application/json")
+		_, err = w.Write(resultBytes)
+		require.NoError(t, err)
+	})
+
+	// Make the test server
+	ts := httptest.NewServer(mux)
+
+	// Close the server at the end of the test
+	t.Cleanup(ts.Close)
+
+	return ts.URL + "/api"
+}
+
+func md5Sum(text string) string {
+	hash := md5.Sum([]byte(text))
+	return hex.EncodeToString(hash[:])
+}
+
+// prepareMockZenodoServer prepares a test server that mocks Zenodo.org
+func prepareMockZenodoServer(t *testing.T, files map[string]string) *httptest.Server {
+	mux := http.NewServeMux()
+
+	// Handle requests for a single record
+	mux.HandleFunc("GET /api/records/{recordID...}", func(w http.ResponseWriter, r *http.Request) {
+		// Check that we are returning data about a single record
+		recordID := strings.TrimPrefix(r.URL.Path, "/api/records/")
+		assert.NotEmpty(t, recordID)
+
+		// Return the most basic response
+		selfURL, err := url.Parse("http://" + r.Host)
+		require.NoError(t, err)
+		selfURL = selfURL.JoinPath(r.URL.String())
+		result := api.InvenioRecordResponse{
+			Links: api.InvenioRecordResponseLinks{
+				Self: selfURL.String(),
+			},
+		}
+		resultBytes, err := json.Marshal(result)
+		require.NoError(t, err)
+		w.Header().Add("Content-Type", "application/json")
+		_, err = w.Write(resultBytes)
+		require.NoError(t, err)
+	})
+	// Handle requests for listing files in a record
+	mux.HandleFunc("GET /api/records/{record}/files", func(w http.ResponseWriter, r *http.Request) {
+		// Return the most basic response
+		filesBaseURL, err := url.Parse("http://" + r.Host)
+		require.NoError(t, err)
+		filesBaseURL = filesBaseURL.JoinPath("/api/files/")
+
+		entries := []api.InvenioFilesResponseEntry{}
+		for filename, contents := range files {
+			entries = append(entries,
+				api.InvenioFilesResponseEntry{
+					Key:      filename,
+					Checksum: md5Sum(contents),
+					Size:     int64(len(contents)),
+					Updated:  time.Now().UTC().Format(time.RFC3339),
+					MimeType: "text/plain; charset=utf-8",
+					Links: api.InvenioFilesResponseEntryLinks{
+						Content: filesBaseURL.JoinPath(filename).String(),
+					},
+				},
+			)
+		}
+
+		result := api.InvenioFilesResponse{
+			Entries: entries,
+		}
+		resultBytes, err := json.Marshal(result)
+		require.NoError(t, err)
+		w.Header().Add("Content-Type", "application/json")
+		_, err = w.Write(resultBytes)
+		require.NoError(t, err)
+	})
+	// Handle requests for file contents
+	mux.HandleFunc("/api/files/{file}", func(w http.ResponseWriter, r *http.Request) {
+		// Check that we are returning the contents of a file
+		filename := strings.TrimPrefix(r.URL.Path, "/api/files/")
+		assert.NotEmpty(t, filename)
+		contents, found := files[filename]
+		if !found {
+			w.WriteHeader(404)
+			return
+		}
+
+		// Return the most basic response
+		_, err := w.Write([]byte(contents))
+		require.NoError(t, err)
+	})
+
+	// Make the test server
+	ts := httptest.NewServer(mux)
+
+	// Close the server at the end of the test
+	t.Cleanup(ts.Close)
+
+	return ts
+}
+
+func TestZenodoRemote(t *testing.T) {
+	recordID := "2600782"
+	doi := "10.5281/zenodo.2600782"
+
+	// The files in the dataset
+	files := map[string]string{
+		"README.md": "This is a dataset.",
+		"data.txt":  "Some data",
+	}
+
+	ts := prepareMockZenodoServer(t, files)
+	resolvedURL := ts.URL + "/record/" + recordID
+
+	doiResolverAPIURL := prepareMockDoiResolverServer(t, resolvedURL)
+
+	testConfig := configmap.Simple{
+		"type":                 "doi",
+		"doi":                  doi,
+		"provider":             "zenodo",
+		"doi_resolver_api_url": doiResolverAPIURL,
+	}
+	f, err := NewFs(context.Background(), remoteName, "", testConfig)
+	require.NoError(t, err)
+
+	// Test listing the DOI files
+	entries, err := f.List(context.Background(), "")
+	require.NoError(t, err)
+
+	sort.Sort(entries)
+
+	require.Equal(t, len(files), len(entries))
+
+	e := entries[0]
+	assert.Equal(t, "README.md", e.Remote())
+	assert.Equal(t, int64(18), e.Size())
+	_, ok := e.(*Object)
+	assert.True(t, ok)
+
+	e = entries[1]
+	assert.Equal(t, "data.txt", e.Remote())
+	assert.Equal(t, int64(9), e.Size())
+	_, ok = e.(*Object)
+	assert.True(t, ok)
+
+	// Test reading the DOI files
+	o, err := f.NewObject(context.Background(), "README.md")
+	require.NoError(t, err)
+	assert.Equal(t, int64(18), o.Size())
+	md5Hash, err := o.Hash(context.Background(), hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, "464352b1cab5240e44528a56fda33d9d", md5Hash)
+	fd, err := o.Open(context.Background())
+	require.NoError(t, err)
+	data, err := io.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	assert.Equal(t, []byte(files["README.md"]), data)
+	do, ok := o.(fs.MimeTyper)
+	require.True(t, ok)
+	assert.Equal(t, "text/plain; charset=utf-8", do.MimeType(context.Background()))
+
+	o, err = f.NewObject(context.Background(), "data.txt")
+	require.NoError(t, err)
+	assert.Equal(t, int64(9), o.Size())
+	md5Hash, err = o.Hash(context.Background(), hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, "5b82f8bf4df2bfb0e66ccaa7306fd024", md5Hash)
+	fd, err = o.Open(context.Background())
+	require.NoError(t, err)
+	data, err = io.ReadAll(fd)
+	require.NoError(t, err)
+	require.NoError(t, fd.Close())
+	assert.Equal(t, []byte(files["data.txt"]), data)
+	do, ok = o.(fs.MimeTyper)
+	require.True(t, ok)
+	assert.Equal(t, "text/plain; charset=utf-8", do.MimeType(context.Background()))
+}

backend/doi/doi_test.go

@@ -0,0 +1,16 @@
+// Test DOI filesystem interface
+package doi
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestDoi:",
+		NilObject:  (*Object)(nil),
+	})
+}

backend/doi/invenio.go

@@ -0,0 +1,164 @@
+// Implementation for InvenioRDM
+
+package doi
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/doi/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+var invenioRecordRegex = regexp.MustCompile(`\/records?\/(.+)`)
+
+// Returns true if resolvedURL is likely a DOI hosted on an InvenioRDM intallation
+func activateInvenio(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL) (isActive bool) {
+	_, _, err := resolveInvenioEndpoint(ctx, srv, pacer, resolvedURL)
+	return err == nil
+}
+
+// Resolve the main API endpoint for a DOI hosted on an InvenioRDM installation
+func resolveInvenioEndpoint(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL) (provider Provider, endpoint *url.URL, err error) {
+	var res *http.Response
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: resolvedURL.String(),
+	}
+	err = pacer.Call(func() (bool, error) {
+		res, err = srv.Call(ctx, &opts)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return "", nil, err
+	}
+
+	// First, attempt to grab the API URL from the headers
+	var linksetURL *url.URL
+	links := parseLinkHeader(res.Header.Get("Link"))
+	for _, link := range links {
+		if link.Rel == "linkset" && link.Type == "application/linkset+json" {
+			parsed, err := url.Parse(link.Href)
+			if err == nil {
+				linksetURL = parsed
+				break
+			}
+		}
+	}
+
+	if linksetURL != nil {
+		endpoint, err = checkInvenioAPIURL(ctx, srv, pacer, linksetURL)
+		if err == nil {
+			return Invenio, endpoint, nil
+		}
+		fs.Logf(nil, "using linkset URL failed: %s", err.Error())
+	}
+
+	// If there is no linkset header, try to grab the record ID from the URL
+	recordID := ""
+	resURL := res.Request.URL
+	match := invenioRecordRegex.FindStringSubmatch(resURL.EscapedPath())
+	if match != nil {
+		recordID = match[1]
+		guessedURL := res.Request.URL.ResolveReference(&url.URL{
+			Path: "/api/records/" + recordID,
+		})
+		endpoint, err = checkInvenioAPIURL(ctx, srv, pacer, guessedURL)
+		if err == nil {
+			return Invenio, endpoint, nil
+		}
+		fs.Logf(nil, "guessing the URL failed: %s", err.Error())
+	}
+
+	return "", nil, fmt.Errorf("could not resolve the Invenio API endpoint for '%s'", resolvedURL.String())
+}
+
+func checkInvenioAPIURL(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL) (endpoint *url.URL, err error) {
+	var result api.InvenioRecordResponse
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: resolvedURL.String(),
+	}
+	err = pacer.Call(func() (bool, error) {
+		res, err := srv.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	if result.Links.Self == "" {
+		return nil, fmt.Errorf("could not parse API response from '%s'", resolvedURL.String())
+	}
+	return url.Parse(result.Links.Self)
+}
+
+// invenioProvider implements the doiProvider interface for InvenioRDM installations
+type invenioProvider struct {
+	f *Fs
+}
+
+// ListEntries returns the full list of entries found at the remote, regardless of root
+func (ip *invenioProvider) ListEntries(ctx context.Context) (entries []*Object, err error) {
+	// Use the cache if populated
+	cachedEntries, found := ip.f.cache.GetMaybe("files")
+	if found {
+		parsedEntries, ok := cachedEntries.([]Object)
+		if ok {
+			for _, entry := range parsedEntries {
+				newEntry := entry
+				entries = append(entries, &newEntry)
+			}
+			return entries, nil
+		}
+	}
+
+	filesURL := ip.f.endpoint.JoinPath("files")
+	var result api.InvenioFilesResponse
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   strings.TrimLeft(filesURL.EscapedPath(), "/"),
+	}
+	err = ip.f.pacer.Call(func() (bool, error) {
+		res, err := ip.f.srv.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("readDir failed: %w", err)
+	}
+	for _, file := range result.Entries {
+		modTime, modTimeErr := time.Parse(time.RFC3339, file.Updated)
+		if modTimeErr != nil {
+			fs.Logf(ip.f, "error: could not parse last update time %v", modTimeErr)
+			modTime = timeUnset
+		}
+		entry := &Object{
+			fs:          ip.f,
+			remote:      file.Key,
+			contentURL:  file.Links.Content,
+			size:        file.Size,
+			modTime:     modTime,
+			contentType: file.MimeType,
+			md5:         strings.TrimPrefix(file.Checksum, "md5:"),
+		}
+		entries = append(entries, entry)
+	}
+	// Populate the cache
+	cacheEntries := []Object{}
+	for _, entry := range entries {
+		cacheEntries = append(cacheEntries, *entry)
+	}
+	ip.f.cache.Put("files", cacheEntries)
+	return entries, nil
+}
+
+func newInvenioProvider(f *Fs) doiProvider {
+	return &invenioProvider{
+		f: f,
+	}
+}

backend/doi/link_header.go

@@ -0,0 +1,75 @@
+package doi
+
+import (
+	"regexp"
+	"strings"
+)
+
+var linkRegex = regexp.MustCompile(`^<(.+)>$`)
+var valueRegex = regexp.MustCompile(`^"(.+)"$`)
+
+// headerLink represents a link as presented in HTTP headers
+// MDN Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link
+type headerLink struct {
+	Href   string
+	Rel    string
+	Type   string
+	Extras map[string]string
+}
+
+func parseLinkHeader(header string) (links []headerLink) {
+	for _, link := range strings.Split(header, ",") {
+		link = strings.TrimSpace(link)
+		parsed := parseLink(link)
+		if parsed != nil {
+			links = append(links, *parsed)
+		}
+	}
+	return links
+}
+
+func parseLink(link string) (parsedLink *headerLink) {
+	var parts []string
+	for _, part := range strings.Split(link, ";") {
+		parts = append(parts, strings.TrimSpace(part))
+	}
+
+	match := linkRegex.FindStringSubmatch(parts[0])
+	if match == nil {
+		return nil
+	}
+
+	result := &headerLink{
+		Href:   match[1],
+		Extras: map[string]string{},
+	}
+
+	for _, keyValue := range parts[1:] {
+		parsed := parseKeyValue(keyValue)
+		if parsed != nil {
+			key, value := parsed[0], parsed[1]
+			switch strings.ToLower(key) {
+			case "rel":
+				result.Rel = value
+			case "type":
+				result.Type = value
+			default:
+				result.Extras[key] = value
+			}
+		}
+	}
+	return result
+}
+
+func parseKeyValue(keyValue string) []string {
+	parts := strings.SplitN(keyValue, "=", 2)
+	if parts[0] == "" || len(parts) < 2 {
+		return nil
+	}
+	match := valueRegex.FindStringSubmatch(parts[1])
+	if match != nil {
+		parts[1] = match[1]
+		return parts
+	}
+	return parts
+}

backend/doi/link_header_internal_test.go

@@ -0,0 +1,44 @@
+package doi
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseLinkHeader(t *testing.T) {
+	header := "<https://zenodo.org/api/records/15063252> ; rel=\"linkset\" ; type=\"application/linkset+json\""
+	links := parseLinkHeader(header)
+	expected := headerLink{
+		Href:   "https://zenodo.org/api/records/15063252",
+		Rel:    "linkset",
+		Type:   "application/linkset+json",
+		Extras: map[string]string{},
+	}
+	assert.Contains(t, links, expected)
+
+	header = "<https://api.example.com/issues?page=2>; rel=\"prev\", <https://api.example.com/issues?page=4>; rel=\"next\", <https://api.example.com/issues?page=10>; rel=\"last\", <https://api.example.com/issues?page=1>; rel=\"first\""
+	links = parseLinkHeader(header)
+	expectedList := []headerLink{{
+		Href:   "https://api.example.com/issues?page=2",
+		Rel:    "prev",
+		Type:   "",
+		Extras: map[string]string{},
+	}, {
+		Href:   "https://api.example.com/issues?page=4",
+		Rel:    "next",
+		Type:   "",
+		Extras: map[string]string{},
+	}, {
+		Href:   "https://api.example.com/issues?page=10",
+		Rel:    "last",
+		Type:   "",
+		Extras: map[string]string{},
+	}, {
+		Href:   "https://api.example.com/issues?page=1",
+		Rel:    "first",
+		Type:   "",
+		Extras: map[string]string{},
+	}}
+	assert.Equal(t, links, expectedList)
+}

backend/doi/zenodo.go

@@ -0,0 +1,47 @@
+// Implementation for Zenodo
+
+package doi
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"regexp"
+
+	"github.com/rclone/rclone/backend/doi/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+var zenodoRecordRegex = regexp.MustCompile(`zenodo[.](.+)`)
+
+// Resolve the main API endpoint for a DOI hosted on Zenodo
+func resolveZenodoEndpoint(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL, doi string) (provider Provider, endpoint *url.URL, err error) {
+	match := zenodoRecordRegex.FindStringSubmatch(doi)
+	if match == nil {
+		return "", nil, fmt.Errorf("could not derive API endpoint URL from '%s'", resolvedURL.String())
+	}
+
+	recordID := match[1]
+	endpointURL := resolvedURL.ResolveReference(&url.URL{Path: "/api/records/" + recordID})
+
+	var result api.InvenioRecordResponse
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: endpointURL.String(),
+	}
+	err = pacer.Call(func() (bool, error) {
+		res, err := srv.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, res, err)
+	})
+	if err != nil {
+		return "", nil, err
+	}
+
+	endpointURL, err = url.Parse(result.Links.Self)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return Zenodo, endpointURL, nil
+}

backend/drive/drive.go

@@ -1745,7 +1745,7 @@ func (f *Fs) createDir(ctx context.Context, pathID, leaf string, metadata fs.Met
 	}
 	var updateMetadata updateMetadataFn
 	if len(metadata) > 0 {
-		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true)
+		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true, true)
 		if err != nil {
 			return nil, fmt.Errorf("create dir: failed to update metadata: %w", err)
 		}
@@ -1776,7 +1776,7 @@ func (f *Fs) updateDir(ctx context.Context, dirID string, metadata fs.Metadata)
 	}
 	dirID = actualID(dirID)
 	updateInfo := &drive.File{}
-	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true)
+	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true, true)
 	if err != nil {
 		return nil, fmt.Errorf("update dir: failed to update metadata from source object: %w", err)
 	}

backend/drive/metadata.go

@@ -507,7 +507,7 @@ type updateMetadataFn func(context.Context, *drive.File) error
 //
 // It returns a callback which should be called to finish the updates
 // after the data is uploaded.
-func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) {
+func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update, isFolder bool) (callback updateMetadataFn, err error) {
 	callbackFns := []updateMetadataFn{}
 	callback = func(ctx context.Context, info *drive.File) error {
 		for _, fn := range callbackFns {
@@ -532,7 +532,9 @@ func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs
 		}
 		switch k {
 		case "copy-requires-writer-permission":
-			if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
+			if isFolder {
+				fs.Debugf(f, "Ignoring %s=%s as can't set on folders", k, v)
+			} else if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
 				return nil, err
 			}
 		case "writers-can-share":
@@ -629,7 +631,7 @@ func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, opti
 	if err != nil {
 		return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
 	}
-	callback, err = f.updateMetadata(ctx, updateInfo, meta, update)
+	callback, err = f.updateMetadata(ctx, updateInfo, meta, update, false)
 	if err != nil {
 		return nil, fmt.Errorf("failed to update metadata from source object: %w", err)
 	}

backend/filelu/api/types.go

@@ -0,0 +1,81 @@
+// Package api defines types for interacting with the FileLu API.
+package api
+
+import "encoding/json"
+
+// CreateFolderResponse represents the response for creating a folder.
+type CreateFolderResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Result struct {
+		FldID interface{} `json:"fld_id"`
+	} `json:"result"`
+}
+
+// DeleteFolderResponse represents the response for deleting a folder.
+type DeleteFolderResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+}
+
+// FolderListResponse represents the response for listing folders.
+type FolderListResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Result struct {
+		Files []struct {
+			Name     string      `json:"name"`
+			FldID    json.Number `json:"fld_id"`
+			Path     string      `json:"path"`
+			FileCode string      `json:"file_code"`
+			Size     int64       `json:"size"`
+		} `json:"files"`
+		Folders []struct {
+			Name  string      `json:"name"`
+			FldID json.Number `json:"fld_id"`
+			Path  string      `json:"path"`
+		} `json:"folders"`
+	} `json:"result"`
+}
+
+// FileDirectLinkResponse represents the response for a direct link to a file.
+type FileDirectLinkResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Result struct {
+		URL  string `json:"url"`
+		Size int64  `json:"size"`
+	} `json:"result"`
+}
+
+// FileInfoResponse represents the response for file information.
+type FileInfoResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+	Result []struct {
+		Size     string `json:"size"`
+		Name     string `json:"name"`
+		FileCode string `json:"filecode"`
+		Hash     string `json:"hash"`
+		Status   int    `json:"status"`
+	} `json:"result"`
+}
+
+// DeleteFileResponse represents the response for deleting a file.
+type DeleteFileResponse struct {
+	Status int    `json:"status"`
+	Msg    string `json:"msg"`
+}
+
+// AccountInfoResponse represents the response for account information.
+type AccountInfoResponse struct {
+	Status int    `json:"status"` // HTTP status code of the response.
+	Msg    string `json:"msg"`    // Message describing the response.
+	Result struct {
+		PremiumExpire string `json:"premium_expire"` // Expiration date of premium access.
+		Email         string `json:"email"`          // User's email address.
+		UType         string `json:"utype"`          // User type (e.g., premium or free).
+		Storage       string `json:"storage"`        // Total storage available to the user.
+		StorageUsed   string `json:"storage_used"`   // Amount of storage used.
+	} `json:"result"` // Nested result structure containing account details.
+}

backend/filelu/filelu.go

@@ -0,0 +1,366 @@
+// Package filelu provides an interface to the FileLu storage system.
+package filelu
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// Register the backend with Rclone
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "filelu",
+		Description: "FileLu Cloud Storage",
+		NewFs:       NewFs,
+		Options: []fs.Option{{
+			Name:      "key",
+			Help:      "Your FileLu Rclone key from My Account",
+			Required:  true,
+			Sensitive: true,
+		},
+			{
+				Name:     config.ConfigEncoding,
+				Help:     config.ConfigEncodingHelp,
+				Advanced: true,
+				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
+					encoder.EncodeSlash |
+					encoder.EncodeLtGt |
+					encoder.EncodeExclamation |
+					encoder.EncodeDoubleQuote |
+					encoder.EncodeSingleQuote |
+					encoder.EncodeBackQuote |
+					encoder.EncodeQuestion |
+					encoder.EncodeDollar |
+					encoder.EncodeColon |
+					encoder.EncodeAsterisk |
+					encoder.EncodePipe |
+					encoder.EncodeHash |
+					encoder.EncodePercent |
+					encoder.EncodeBackSlash |
+					encoder.EncodeCrLf |
+					encoder.EncodeDel |
+					encoder.EncodeCtl |
+					encoder.EncodeLeftSpace |
+					encoder.EncodeLeftPeriod |
+					encoder.EncodeLeftTilde |
+					encoder.EncodeLeftCrLfHtVt |
+					encoder.EncodeRightPeriod |
+					encoder.EncodeRightCrLfHtVt |
+					encoder.EncodeSquareBracket |
+					encoder.EncodeSemicolon |
+					encoder.EncodeRightSpace |
+					encoder.EncodeInvalidUtf8 |
+					encoder.EncodeDot),
+			},
+		}})
+}
+
+// Options defines the configuration for the FileLu backend
+type Options struct {
+	Key string               `config:"key"`
+	Enc encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents the FileLu file system
+type Fs struct {
+	name       string
+	root       string
+	opt        Options
+	features   *fs.Features
+	endpoint   string
+	pacer      *pacer.Pacer
+	srv        *rest.Client
+	client     *http.Client
+	targetFile string
+}
+
+// NewFs creates a new Fs object for FileLu
+func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config: %w", err)
+	}
+
+	if opt.Key == "" {
+		return nil, fmt.Errorf("FileLu Rclone Key is required")
+	}
+
+	client := fshttp.NewClient(ctx)
+
+	if strings.TrimSpace(root) == "" {
+		root = ""
+	}
+	root = strings.Trim(root, "/")
+
+	filename := ""
+
+	f := &Fs{
+		name:       name,
+		opt:        *opt,
+		endpoint:   "https://filelu.com/rclone",
+		client:     client,
+		srv:        rest.NewClient(client).SetRoot("https://filelu.com/rclone"),
+		pacer:      pacer.New(),
+		targetFile: filename,
+		root:       root,
+	}
+
+	f.features = (&fs.Features{
+		CanHaveEmptyDirectories: true,
+		WriteMetadata:           false,
+		SlowHash:                true,
+	}).Fill(ctx, f)
+
+	rootContainer, rootDirectory := rootSplit(f.root)
+	if rootContainer != "" && rootDirectory != "" {
+		// Check to see if the (container,directory) is actually an existing file
+		oldRoot := f.root
+		newRoot, leaf := path.Split(oldRoot)
+		f.root = strings.Trim(newRoot, "/")
+		_, err := f.NewObject(ctx, leaf)
+		if err != nil {
+			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
+				// File doesn't exist or is a directory so return old f
+				f.root = strings.Trim(oldRoot, "/")
+				return f, nil
+			}
+			return nil, err
+		}
+		// return an error with an fs which points to the parent
+		return f, fs.ErrorIsFile
+	}
+
+	return f, nil
+}
+
+// Mkdir to create directory on remote server.
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	fullPath := path.Clean(f.root + "/" + dir)
+	_, err := f.createFolder(ctx, fullPath)
+	return err
+}
+
+// About provides usage statistics for the remote
+func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
+	accountInfo, err := f.getAccountInfo(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	totalStorage, err := parseStorageToBytes(accountInfo.Result.Storage)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse total storage: %w", err)
+	}
+
+	usedStorage, err := parseStorageToBytes(accountInfo.Result.StorageUsed)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse used storage: %w", err)
+	}
+
+	return &fs.Usage{
+		Total: fs.NewUsageValue(totalStorage), // Total bytes available
+		Used:  fs.NewUsageValue(usedStorage),  // Total bytes used
+		Free:  fs.NewUsageValue(totalStorage - usedStorage),
+	}, nil
+}
+
+// Purge deletes the directory and all its contents
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	fullPath := path.Join(f.root, dir)
+	if fullPath != "" {
+		fullPath = "/" + strings.Trim(fullPath, "/")
+	}
+	return f.deleteFolder(ctx, fullPath)
+}
+
+// List returns a list of files and folders
+// List returns a list of files and folders for the given directory
+func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
+	// Compose full path for API call
+	fullPath := path.Join(f.root, dir)
+	fullPath = "/" + strings.Trim(fullPath, "/")
+	if fullPath == "/" {
+		fullPath = ""
+	}
+
+	var entries fs.DirEntries
+	result, err := f.getFolderList(ctx, fullPath)
+	if err != nil {
+		return nil, err
+	}
+
+	fldMap := map[string]bool{}
+	for _, folder := range result.Result.Folders {
+		fldMap[folder.FldID.String()] = true
+		if f.root == "" && dir == "" && strings.Contains(folder.Path, "/") {
+			continue
+		}
+
+		paths := strings.Split(folder.Path, fullPath+"/")
+		remote := paths[0]
+		if len(paths) > 1 {
+			remote = paths[1]
+		}
+
+		if strings.Contains(remote, "/") {
+			continue
+		}
+
+		pathsWithoutRoot := strings.Split(folder.Path, "/"+f.root+"/")
+		remotePathWithoutRoot := pathsWithoutRoot[0]
+		if len(pathsWithoutRoot) > 1 {
+			remotePathWithoutRoot = pathsWithoutRoot[1]
+		}
+		remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
+		entries = append(entries, fs.NewDir(remotePathWithoutRoot, time.Now()))
+	}
+	for _, file := range result.Result.Files {
+		if _, ok := fldMap[file.FldID.String()]; ok {
+			continue
+		}
+		remote := path.Join(dir, file.Name)
+		// trim leading slashes
+		remote = strings.TrimPrefix(remote, "/")
+		obj := &Object{
+			fs:      f,
+			remote:  remote,
+			size:    file.Size,
+			modTime: time.Now(),
+		}
+		entries = append(entries, obj)
+	}
+	return entries, nil
+}
+
+// Put uploads a file directly to the destination folder in the FileLu storage system.
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	if src.Size() == 0 {
+		return nil, fs.ErrorCantUploadEmptyFiles
+	}
+
+	err := f.uploadFile(ctx, in, src.Remote())
+	if err != nil {
+		return nil, err
+	}
+
+	newObject := &Object{
+		fs:      f,
+		remote:  src.Remote(),
+		size:    src.Size(),
+		modTime: src.ModTime(ctx),
+	}
+	fs.Infof(f, "Put: Successfully uploaded new file %q", src.Remote())
+	return newObject, nil
+}
+
+// Move moves the file to the specified location
+func (f *Fs) Move(ctx context.Context, src fs.Object, destinationPath string) (fs.Object, error) {
+
+	if strings.HasPrefix(destinationPath, "/") || strings.Contains(destinationPath, ":\\") {
+		dir := path.Dir(destinationPath)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			return nil, fmt.Errorf("failed to create destination directory: %w", err)
+		}
+
+		reader, err := src.Open(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("failed to open source file: %w", err)
+		}
+		defer func() {
+			if err := reader.Close(); err != nil {
+				fs.Logf(nil, "Failed to close file body: %v", err)
+			}
+		}()
+
+		dest, err := os.Create(destinationPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create destination file: %w", err)
+		}
+		defer func() {
+			if err := dest.Close(); err != nil {
+				fs.Logf(nil, "Failed to close file body: %v", err)
+			}
+		}()
+
+		if _, err := io.Copy(dest, reader); err != nil {
+			return nil, fmt.Errorf("failed to copy file content: %w", err)
+		}
+
+		if err := src.Remove(ctx); err != nil {
+			return nil, fmt.Errorf("failed to remove source file: %w", err)
+		}
+
+		return nil, nil
+	}
+
+	reader, err := src.Open(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open source object: %w", err)
+	}
+	defer func() {
+		if err := reader.Close(); err != nil {
+			fs.Logf(nil, "Failed to close file body: %v", err)
+		}
+	}()
+
+	err = f.uploadFile(ctx, reader, destinationPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to upload file to destination: %w", err)
+	}
+
+	if err := src.Remove(ctx); err != nil {
+		return nil, fmt.Errorf("failed to delete source file: %w", err)
+	}
+
+	return &Object{
+		fs:      f,
+		remote:  destinationPath,
+		size:    src.Size(),
+		modTime: src.ModTime(ctx),
+	}, nil
+}
+
+// Rmdir removes a directory
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	fullPath := path.Join(f.root, dir)
+	if fullPath != "" {
+		fullPath = "/" + strings.Trim(fullPath, "/")
+	}
+
+	// Step 1: Check if folder is empty
+	listResp, err := f.getFolderList(ctx, fullPath)
+	if err != nil {
+		return err
+	}
+	if len(listResp.Result.Files) > 0 || len(listResp.Result.Folders) > 0 {
+		return fmt.Errorf("Rmdir: directory %q is not empty", fullPath)
+	}
+
+	// Step 2: Delete the folder
+	return f.deleteFolder(ctx, fullPath)
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs      = (*Fs)(nil)
+	_ fs.Purger  = (*Fs)(nil)
+	_ fs.Abouter = (*Fs)(nil)
+	_ fs.Mover   = (*Fs)(nil)
+	_ fs.Object  = (*Object)(nil)
+)

backend/filelu/filelu_client.go

@@ -0,0 +1,324 @@
+package filelu
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"github.com/rclone/rclone/backend/filelu/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// createFolder creates a folder at the specified path.
+func (f *Fs) createFolder(ctx context.Context, dirPath string) (*api.CreateFolderResponse, error) {
+	encodedDir := f.fromStandardPath(dirPath)
+	apiURL := fmt.Sprintf("%s/folder/create?folder_path=%s&key=%s",
+		f.endpoint,
+		url.QueryEscape(encodedDir),
+		url.QueryEscape(f.opt.Key), // assuming f.opt.Key is the correct field
+	)
+
+	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+
+	var resp *http.Response
+	result := api.CreateFolderResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		var innerErr error
+		resp, innerErr = f.client.Do(req)
+		return fserrors.ShouldRetry(innerErr), innerErr
+	})
+	if err != nil {
+		return nil, fmt.Errorf("request failed: %w", err)
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			fs.Logf(nil, "Failed to close response body: %v", err)
+		}
+	}()
+
+	err = json.NewDecoder(resp.Body).Decode(&result)
+	if err != nil {
+		return nil, fmt.Errorf("error decoding response: %w", err)
+	}
+	if result.Status != 200 {
+		return nil, fmt.Errorf("error: %s", result.Msg)
+	}
+
+	fs.Infof(f, "Successfully created folder %q with ID %v", dirPath, result.Result.FldID)
+	return &result, nil
+}
+
+// getFolderList List both files and folders in a directory.
+func (f *Fs) getFolderList(ctx context.Context, path string) (*api.FolderListResponse, error) {
+	encodedDir := f.fromStandardPath(path)
+	apiURL := fmt.Sprintf("%s/folder/list?folder_path=%s&key=%s",
+		f.endpoint,
+		url.QueryEscape(encodedDir),
+		url.QueryEscape(f.opt.Key),
+	)
+
+	var body []byte
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+		if err != nil {
+			return false, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to list directory: %w", err)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			return false, fmt.Errorf("error reading response body: %w", err)
+		}
+
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	var response api.FolderListResponse
+	if err := json.NewDecoder(bytes.NewReader(body)).Decode(&response); err != nil {
+		return nil, fmt.Errorf("error decoding response: %w", err)
+	}
+	if response.Status != 200 {
+		if strings.Contains(response.Msg, "Folder not found") {
+			return nil, fs.ErrorDirNotFound
+		}
+		return nil, fmt.Errorf("API error: %s", response.Msg)
+	}
+
+	for index := range response.Result.Folders {
+		response.Result.Folders[index].Path = f.toStandardPath(response.Result.Folders[index].Path)
+	}
+
+	for index := range response.Result.Files {
+		response.Result.Files[index].Name = f.toStandardPath(response.Result.Files[index].Name)
+	}
+
+	return &response, nil
+
+}
+
+// deleteFolder deletes a folder at the specified path.
+func (f *Fs) deleteFolder(ctx context.Context, fullPath string) error {
+	fullPath = f.fromStandardPath(fullPath)
+	deleteURL := fmt.Sprintf("%s/folder/delete?folder_path=%s&key=%s",
+		f.endpoint,
+		url.QueryEscape(fullPath),
+		url.QueryEscape(f.opt.Key),
+	)
+
+	delResp := api.DeleteFolderResponse{}
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", deleteURL, nil)
+		if err != nil {
+			return false, err
+		}
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return fserrors.ShouldRetry(err), err
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return false, err
+		}
+
+		if err := json.Unmarshal(body, &delResp); err != nil {
+			return false, fmt.Errorf("error decoding delete response: %w", err)
+		}
+		if delResp.Status != 200 {
+			return false, fmt.Errorf("delete error: %s", delResp.Msg)
+		}
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+
+	fs.Infof(f, "Rmdir: successfully deleted %q", fullPath)
+	return nil
+}
+
+// getDirectLink of file from FileLu to download.
+func (f *Fs) getDirectLink(ctx context.Context, filePath string) (string, int64, error) {
+	filePath = f.fromStandardPath(filePath)
+	apiURL := fmt.Sprintf("%s/file/direct_link?file_path=%s&key=%s",
+		f.endpoint,
+		url.QueryEscape(filePath),
+		url.QueryEscape(f.opt.Key),
+	)
+
+	result := api.FileDirectLinkResponse{}
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+		if err != nil {
+			return false, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to fetch direct link: %w", err)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+			return false, fmt.Errorf("error decoding response: %w", err)
+		}
+
+		if result.Status != 200 {
+			return false, fmt.Errorf("API error: %s", result.Msg)
+		}
+
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+	if err != nil {
+		return "", 0, err
+	}
+
+	return result.Result.URL, result.Result.Size, nil
+}
+
+// deleteFile deletes a file based on filePath
+func (f *Fs) deleteFile(ctx context.Context, filePath string) error {
+	filePath = f.fromStandardPath(filePath)
+	apiURL := fmt.Sprintf("%s/file/remove?file_path=%s&key=%s",
+		f.endpoint,
+		url.QueryEscape(filePath),
+		url.QueryEscape(f.opt.Key),
+	)
+
+	result := api.DeleteFileResponse{}
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+		if err != nil {
+			return false, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to fetch direct link: %w", err)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+			return false, fmt.Errorf("error decoding response: %w", err)
+		}
+
+		if result.Status != 200 {
+			return false, fmt.Errorf("API error: %s", result.Msg)
+		}
+
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+	return err
+}
+
+// getAccountInfo retrieves account information
+func (f *Fs) getAccountInfo(ctx context.Context) (*api.AccountInfoResponse, error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/account/info",
+		Parameters: url.Values{
+			"key": {f.opt.Key},
+		},
+	}
+
+	var result api.AccountInfoResponse
+	err := f.pacer.Call(func() (bool, error) {
+		_, callErr := f.srv.CallJSON(ctx, &opts, nil, &result)
+		return fserrors.ShouldRetry(callErr), callErr
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	if result.Status != 200 {
+		return nil, fmt.Errorf("error: %s", result.Msg)
+	}
+
+	return &result, nil
+}
+
+// getFileInfo retrieves file information based on file code
+func (f *Fs) getFileInfo(ctx context.Context, fileCode string) (*api.FileInfoResponse, error) {
+	u, _ := url.Parse(f.endpoint + "/file/info2")
+	q := u.Query()
+	q.Set("file_code", fileCode) // raw path — Go handles escaping properly here
+	q.Set("key", f.opt.Key)
+	u.RawQuery = q.Encode()
+
+	apiURL := f.endpoint + "/file/info2?" + u.RawQuery
+
+	var body []byte
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+		if err != nil {
+			return false, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to fetch file info: %w", err)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			return false, fmt.Errorf("error reading response body: %w", err)
+		}
+
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	result := api.FileInfoResponse{}
+
+	if err := json.NewDecoder(bytes.NewReader(body)).Decode(&result); err != nil {
+		return nil, fmt.Errorf("error decoding response: %w", err)
+	}
+
+	if result.Status != 200 || len(result.Result) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	return &result, nil
+}

backend/filelu/filelu_file_uploader.go

@@ -0,0 +1,193 @@
+package filelu
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// uploadFile uploads a file to FileLu
+func (f *Fs) uploadFile(ctx context.Context, fileContent io.Reader, fileFullPath string) error {
+	directory := path.Dir(fileFullPath)
+	fileName := path.Base(fileFullPath)
+	if directory == "." {
+		directory = ""
+	}
+	destinationFolderPath := path.Join(f.root, directory)
+	if destinationFolderPath != "" {
+		destinationFolderPath = "/" + strings.Trim(destinationFolderPath, "/")
+	}
+
+	existingEntries, err := f.List(ctx, path.Dir(fileFullPath))
+	if err != nil {
+		if errors.Is(err, fs.ErrorDirNotFound) {
+			err = f.Mkdir(ctx, path.Dir(fileFullPath))
+			if err != nil {
+				return fmt.Errorf("failed to create directory: %w", err)
+			}
+		} else {
+			return fmt.Errorf("failed to list existing files: %w", err)
+		}
+	}
+
+	for _, entry := range existingEntries {
+		if entry.Remote() == fileFullPath {
+			_, ok := entry.(fs.Object)
+			if !ok {
+				continue
+			}
+
+			// If the file exists but is different, remove it
+			filePath := "/" + strings.Trim(destinationFolderPath+"/"+fileName, "/")
+			err = f.deleteFile(ctx, filePath)
+			if err != nil {
+				return fmt.Errorf("failed to delete existing file: %w", err)
+			}
+		}
+	}
+
+	uploadURL, sessID, err := f.getUploadServer(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to retrieve upload server: %w", err)
+	}
+
+	// Since the fileCode isn't used, just handle the error
+	if _, err := f.uploadFileWithDestination(ctx, uploadURL, sessID, fileName, fileContent, destinationFolderPath); err != nil {
+		return fmt.Errorf("failed to upload file: %w", err)
+	}
+
+	return nil
+}
+
+// getUploadServer gets the upload server URL with proper key authentication
+func (f *Fs) getUploadServer(ctx context.Context) (string, string, error) {
+	apiURL := fmt.Sprintf("%s/upload/server?key=%s", f.endpoint, url.QueryEscape(f.opt.Key))
+
+	var result struct {
+		Status int    `json:"status"`
+		SessID string `json:"sess_id"`
+		Result string `json:"result"`
+		Msg    string `json:"msg"`
+	}
+
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+		if err != nil {
+			return false, fmt.Errorf("failed to create request: %w", err)
+		}
+
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to get upload server: %w", err)
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+			return false, fmt.Errorf("error decoding response: %w", err)
+		}
+
+		if result.Status != 200 {
+			return false, fmt.Errorf("API error: %s", result.Msg)
+		}
+
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+
+	if err != nil {
+		return "", "", err
+	}
+
+	return result.Result, result.SessID, nil
+}
+
+// uploadFileWithDestination uploads a file directly to a specified folder using file content reader.
+func (f *Fs) uploadFileWithDestination(ctx context.Context, uploadURL, sessID, fileName string, fileContent io.Reader, dirPath string) (string, error) {
+	destinationPath := f.fromStandardPath(dirPath)
+	encodedFileName := f.fromStandardPath(fileName)
+	pr, pw := io.Pipe()
+	writer := multipart.NewWriter(pw)
+	isDeletionRequired := false
+	go func() {
+		defer func() {
+			if err := pw.Close(); err != nil {
+				fs.Logf(nil, "Failed to close: %v", err)
+			}
+		}()
+		_ = writer.WriteField("sess_id", sessID)
+		_ = writer.WriteField("utype", "prem")
+		_ = writer.WriteField("fld_path", destinationPath)
+
+		part, err := writer.CreateFormFile("file_0", encodedFileName)
+		if err != nil {
+			pw.CloseWithError(fmt.Errorf("failed to create form file: %w", err))
+			return
+		}
+
+		if _, err := io.Copy(part, fileContent); err != nil {
+			isDeletionRequired = true
+			pw.CloseWithError(fmt.Errorf("failed to copy file content: %w", err))
+			return
+		}
+
+		if err := writer.Close(); err != nil {
+			pw.CloseWithError(fmt.Errorf("failed to close writer: %w", err))
+		}
+	}()
+
+	var fileCode string
+	err := f.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "POST", uploadURL, pr)
+		if err != nil {
+			return false, fmt.Errorf("failed to create upload request: %w", err)
+		}
+		req.Header.Set("Content-Type", writer.FormDataContentType())
+
+		resp, err := f.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to send upload request: %w", err)
+		}
+		defer respBodyClose(resp.Body)
+
+		var result []struct {
+			FileCode   string `json:"file_code"`
+			FileStatus string `json:"file_status"`
+		}
+		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+			return false, fmt.Errorf("failed to parse upload response: %w", err)
+		}
+
+		if len(result) == 0 || result[0].FileStatus != "OK" {
+			return false, fmt.Errorf("upload failed with status: %s", result[0].FileStatus)
+		}
+
+		fileCode = result[0].FileCode
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+
+	if err != nil && isDeletionRequired {
+		// Attempt to delete the file if upload fails
+		_ = f.deleteFile(ctx, destinationPath+"/"+fileName)
+	}
+
+	return fileCode, err
+}
+
+// respBodyClose to check body response.
+func respBodyClose(responseBody io.Closer) {
+	if cerr := responseBody.Close(); cerr != nil {
+		fmt.Printf("Error closing response body: %v\n", cerr)
+	}
+}

backend/filelu/filelu_helper.go

@@ -0,0 +1,112 @@
+package filelu
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// errFileNotFound represent file not found error
+var errFileNotFound error = errors.New("file not found")
+
+// getFileCode retrieves the file code for a given file path
+func (f *Fs) getFileCode(ctx context.Context, filePath string) (string, error) {
+	// Prepare parent directory
+	parentDir := path.Dir(filePath)
+
+	// Call List to get all the files
+	result, err := f.getFolderList(ctx, parentDir)
+	if err != nil {
+		return "", err
+	}
+
+	for _, file := range result.Result.Files {
+		filePathFromServer := parentDir + "/" + file.Name
+		if parentDir == "/" {
+			filePathFromServer = "/" + file.Name
+		}
+		if filePath == filePathFromServer {
+			return file.FileCode, nil
+		}
+	}
+
+	return "", errFileNotFound
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+func (f *Fs) fromStandardPath(remote string) string {
+	return f.opt.Enc.FromStandardPath(remote)
+}
+
+func (f *Fs) toStandardPath(remote string) string {
+	return f.opt.Enc.ToStandardPath(remote)
+}
+
+// Hashes returns an empty hash set, indicating no hash support
+func (f *Fs) Hashes() hash.Set {
+	return hash.NewHashSet() // Properly creates an empty hash set
+}
+
+// Name returns the remote name
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root returns the root path
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// Precision returns the precision of the remote
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+func (f *Fs) String() string {
+	return fmt.Sprintf("FileLu root '%s'", f.root)
+}
+
+// isFileCode checks if a string looks like a file code
+func isFileCode(s string) bool {
+	if len(s) != 12 {
+		return false
+	}
+	for _, c := range s {
+		if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
+			return false
+		}
+	}
+	return true
+}
+
+func shouldRetry(err error) bool {
+	return fserrors.ShouldRetry(err)
+}
+
+func shouldRetryHTTP(code int) bool {
+	return code == 429 || code >= 500
+}
+
+func rootSplit(absPath string) (bucket, bucketPath string) {
+	// No bucket
+	if absPath == "" {
+		return "", ""
+	}
+	slash := strings.IndexRune(absPath, '/')
+	// Bucket but no path
+	if slash < 0 {
+		return absPath, ""
+	}
+	return absPath[:slash], absPath[slash+1:]
+}

backend/filelu/filelu_object.go

@@ -0,0 +1,259 @@
+package filelu
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Object describes a FileLu object
+type Object struct {
+	fs      *Fs
+	remote  string
+	size    int64
+	modTime time.Time
+}
+
+// NewObject creates a new Object for the given remote path
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	var filePath string
+	filePath = path.Join(f.root, remote)
+	filePath = "/" + strings.Trim(filePath, "/")
+
+	// Get File code
+	fileCode, err := f.getFileCode(ctx, filePath)
+	if err != nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	// Get File info
+	fileInfos, err := f.getFileInfo(ctx, fileCode)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get file info: %w", err)
+	}
+
+	fileInfo := fileInfos.Result[0]
+	size, _ := strconv.ParseInt(fileInfo.Size, 10, 64)
+
+	returnedRemote := remote
+	return &Object{
+		fs:      f,
+		remote:  returnedRemote,
+		size:    size,
+		modTime: time.Now(),
+	}, nil
+}
+
+// Open opens the object for reading
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	filePath := path.Join(o.fs.root, o.remote)
+	// Get direct link
+	directLink, size, err := o.fs.getDirectLink(ctx, filePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get direct link: %w", err)
+	}
+
+	o.size = size
+
+	// Offset and Count for range download
+	var offset int64
+	var count int64
+	fs.FixRangeOption(options, o.size)
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			offset, count = x.Decode(o.size)
+			if count < 0 {
+				count = o.size - offset
+			}
+		case *fs.SeekOption:
+			offset = x.Offset
+			count = o.size
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	var reader io.ReadCloser
+	err = o.fs.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", directLink, nil)
+		if err != nil {
+			return false, fmt.Errorf("failed to create download request: %w", err)
+		}
+
+		resp, err := o.fs.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), fmt.Errorf("failed to download file: %w", err)
+		}
+
+		if resp.StatusCode != http.StatusOK {
+			defer func() {
+				if err := resp.Body.Close(); err != nil {
+					fs.Logf(nil, "Failed to close response body: %v", err)
+				}
+			}()
+			return false, fmt.Errorf("failed to download file: HTTP %d", resp.StatusCode)
+		}
+
+		// Wrap the response body to handle offset and count
+		currentContents, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return false, fmt.Errorf("failed to read response body: %w", err)
+		}
+
+		if offset > 0 {
+			if offset > int64(len(currentContents)) {
+				return false, fmt.Errorf("offset %d exceeds file size %d", offset, len(currentContents))
+			}
+			currentContents = currentContents[offset:]
+		}
+		if count > 0 && count < int64(len(currentContents)) {
+			currentContents = currentContents[:count]
+		}
+		reader = io.NopCloser(bytes.NewReader(currentContents))
+
+		return false, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return reader, nil
+}
+
+// Update updates the object with new data
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	if src.Size() <= 0 {
+		return fs.ErrorCantUploadEmptyFiles
+	}
+
+	err := o.fs.uploadFile(ctx, in, o.remote)
+	if err != nil {
+		return fmt.Errorf("failed to upload file: %w", err)
+	}
+	o.size = src.Size()
+	return nil
+}
+
+// Remove deletes the object from FileLu
+func (o *Object) Remove(ctx context.Context) error {
+	fullPath := "/" + strings.Trim(path.Join(o.fs.root, o.remote), "/")
+
+	err := o.fs.deleteFile(ctx, fullPath)
+	if err != nil {
+		return err
+	}
+	fs.Infof(o.fs, "Successfully deleted file: %s", fullPath)
+	return nil
+}
+
+// Hash returns the MD5 hash of an object
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hash.MD5 {
+		return "", hash.ErrUnsupported
+	}
+
+	var fileCode string
+	if isFileCode(o.fs.root) {
+		fileCode = o.fs.root
+	} else {
+		matches := regexp.MustCompile(`\((.*?)\)`).FindAllStringSubmatch(o.remote, -1)
+		for _, match := range matches {
+			if len(match) > 1 && len(match[1]) == 12 {
+				fileCode = match[1]
+				break
+			}
+		}
+	}
+	if fileCode == "" {
+		return "", fmt.Errorf("no valid file code found in the remote path")
+	}
+
+	apiURL := fmt.Sprintf("%s/file/info?file_code=%s&key=%s",
+		o.fs.endpoint, url.QueryEscape(fileCode), url.QueryEscape(o.fs.opt.Key))
+
+	var result struct {
+		Status int    `json:"status"`
+		Msg    string `json:"msg"`
+		Result []struct {
+			Hash string `json:"hash"`
+		} `json:"result"`
+	}
+	err := o.fs.pacer.Call(func() (bool, error) {
+		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
+		if err != nil {
+			return false, err
+		}
+		resp, err := o.fs.client.Do(req)
+		if err != nil {
+			return shouldRetry(err), err
+		}
+		defer func() {
+			if err := resp.Body.Close(); err != nil {
+				fs.Logf(nil, "Failed to close response body: %v", err)
+			}
+		}()
+
+		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
+			return false, err
+		}
+		return shouldRetryHTTP(resp.StatusCode), nil
+	})
+	if err != nil {
+		return "", err
+	}
+	if result.Status != 200 || len(result.Result) == 0 {
+		return "", fmt.Errorf("error: unable to fetch hash: %s", result.Msg)
+	}
+
+	return result.Result[0].Hash, nil
+}
+
+// String returns a string representation of the object
+func (o *Object) String() string {
+	return o.remote
+}
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Size returns the size of the object
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// ModTime returns the modification time of the object
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// SetModTime sets the modification time of the object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+// Storable indicates whether the object is storable
+func (o *Object) Storable() bool {
+	return true
+}

backend/filelu/filelu_test.go

@@ -0,0 +1,16 @@
+package filelu_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests for the FileLu backend
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:      "TestFileLu:",
+		NilObject:       nil,
+		SkipInvalidUTF8: true,
+	})
+}

backend/filelu/utils.go

@@ -0,0 +1,15 @@
+package filelu
+
+import (
+	"fmt"
+)
+
+// parseStorageToBytes converts a storage string (e.g., "10") to bytes
+func parseStorageToBytes(storage string) (int64, error) {
+	var gb float64
+	_, err := fmt.Sscanf(storage, "%f", &gb)
+	if err != nil {
+		return 0, fmt.Errorf("failed to parse storage: %w", err)
+	}
+	return int64(gb * 1024 * 1024 * 1024), nil
+}

backend/ftp/ftp.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net"
 	"net/textproto"
+	"net/url"
 	"path"
 	"runtime"
 	"strings"
@@ -185,6 +186,14 @@ Supports the format user:pass@host:port, user@host:port, host:port.
 Example:
 		
     myUser:myPass@localhost:9005
+`,
+			Advanced: true,
+		}, {
+			Name:    "http_proxy",
+			Default: "",
+			Help: `URL for HTTP CONNECT proxy
+
+Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
 `,
 			Advanced: true,
 		}, {
@@ -248,6 +257,7 @@ type Options struct {
 	AskPassword       bool                 `config:"ask_password"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 	SocksProxy        string               `config:"socks_proxy"`
+	HTTPProxy         string               `config:"http_proxy"`
 	NoCheckUpload     bool                 `config:"no_check_upload"`
 }
 
@@ -266,6 +276,7 @@ type Fs struct {
 	pool     []*ftp.ServerConn
 	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
+	proxyURL *url.URL  // address of HTTP proxy read from environment
 	pacer    *fs.Pacer // pacer for FTP connections
 	fGetTime bool      // true if the ftp library accepts GetTime
 	fSetTime bool      // true if the ftp library accepts SetTime
@@ -413,11 +424,26 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	dial := func(network, address string) (conn net.Conn, err error) {
 		fs.Debugf(f, "dial(%q,%q)", network, address)
 		defer func() {
-			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
+			if err != nil {
+				fs.Debugf(f, "> dial: conn=%v, err=%v", conn, err)
+			} else {
+				fs.Debugf(f, "> dial: conn=%s->%s, err=%v", conn.LocalAddr(), conn.RemoteAddr(), err)
+			}
 		}()
 		baseDialer := fshttp.NewDialer(ctx)
 		if f.opt.SocksProxy != "" {
 			conn, err = proxy.SOCKS5Dial(network, address, f.opt.SocksProxy, baseDialer)
+		} else if f.proxyURL != nil {
+			// We need to make the onward connection to f.opt.Host. However the FTP
+			// library sets the host to the proxy IP after using EPSV or PASV so we need
+			// to correct that here.
+			var dialPort string
+			_, dialPort, err = net.SplitHostPort(address)
+			if err != nil {
+				return nil, err
+			}
+			dialAddress := net.JoinHostPort(f.opt.Host, dialPort)
+			conn, err = proxy.HTTPConnectDial(network, dialAddress, f.proxyURL, baseDialer)
 		} else {
 			conn, err = baseDialer.Dial(network, address)
 		}
@@ -631,6 +657,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		CanHaveEmptyDirectories: true,
 		PartialUploads:          true,
 	}).Fill(ctx, f)
+	// get proxy URL if set
+	if opt.HTTPProxy != "" {
+		proxyURL, err := url.Parse(opt.HTTPProxy)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse HTTP Proxy URL: %w", err)
+		}
+		f.proxyURL = proxyURL
+	}
 	// set the pool drainer timer going
 	if f.opt.IdleTimeout > 0 {
 		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })

backend/ftp/ftp_internal_test.go

@@ -52,7 +52,7 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
 		ci.Timeout = saveTimeout
 	}()
 	ci.LowLevelRetries = 1
-	ci.Timeout = idleTimeout
+	ci.Timeout = fs.Duration(idleTimeout)
 
 	upload := func(concurrency int, shutTimeout time.Duration) (obj fs.Object, err error) {
 		fixFs := deriveFs(ctx, t, f, settings{

backend/gofile/api/types.go

@@ -194,33 +194,9 @@ type DeleteResponse struct {
 	Data map[string]Error
 }
 
-// Server is an upload server
-type Server struct {
-	Name string `json:"name"`
-	Zone string `json:"zone"`
-}
-
-// String returns a string representation of the Server
-func (s *Server) String() string {
-	return fmt.Sprintf("%s (%s)", s.Name, s.Zone)
-}
-
-// Root returns the root URL for the server
-func (s *Server) Root() string {
-	return fmt.Sprintf("https://%s.gofile.io/", s.Name)
-}
-
-// URL returns the upload URL for the server
-func (s *Server) URL() string {
-	return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)
-}
-
-// ServersResponse is the output from /servers
-type ServersResponse struct {
-	Error
-	Data struct {
-		Servers []Server `json:"servers"`
-	} `json:"data"`
+// DirectUploadURL returns the direct upload URL for Gofile
+func DirectUploadURL() string {
+	return "https://upload.gofile.io/uploadfile"
 }
 
 // UploadResponse is returned by POST /contents/uploadfile

backend/gofile/gofile.go

@@ -8,13 +8,11 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"path"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/gofile/api"
@@ -37,10 +35,8 @@ const (
 	maxSleep       = 20 * time.Second
 	decayConstant  = 1 // bigger for slower decay, exponential
 	rootURL        = "https://api.gofile.io"
-	serversExpiry  = 60 * time.Second // check for new upload servers this often
-	serversActive  = 2                // choose this many closest upload servers to use
-	rateLimitSleep = 5 * time.Second  // penalise a goroutine by this long for making a rate limit error
-	maxDepth       = 4                // in ListR recursive list this deep (maximum is 16)
+	rateLimitSleep = 5 * time.Second // penalise a goroutine by this long for making a rate limit error
+	maxDepth       = 4               // in ListR recursive list this deep (maximum is 16)
 )
 
 /*
@@ -128,16 +124,13 @@ type Options struct {
 
 // Fs represents a remote gofile
 type Fs struct {
-	name           string             // name of this remote
-	root           string             // the path we are working on
-	opt            Options            // parsed options
-	features       *fs.Features       // optional features
-	srv            *rest.Client       // the connection to the server
-	dirCache       *dircache.DirCache // Map of directory path to directory id
-	pacer          *fs.Pacer          // pacer for API calls
-	serversMu      *sync.Mutex        // protect the servers info below
-	servers        []api.Server       // upload servers we can use
-	serversChecked time.Time          // time the servers were refreshed
+	name     string             // name of this remote
+	root     string             // the path we are working on
+	opt      Options            // parsed options
+	features *fs.Features       // optional features
+	srv      *rest.Client       // the connection to the server
+	dirCache *dircache.DirCache // Map of directory path to directory id
+	pacer    *fs.Pacer          // pacer for API calls
 }
 
 // Object describes a gofile object
@@ -311,12 +304,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	client := fshttp.NewClient(ctx)
 
 	f := &Fs{
-		name:      name,
-		root:      root,
-		opt:       *opt,
-		srv:       rest.NewClient(client).SetRoot(rootURL),
-		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		serversMu: new(sync.Mutex),
+		name:  name,
+		root:  root,
+		opt:   *opt,
+		srv:   rest.NewClient(client).SetRoot(rootURL),
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:          false,
@@ -435,98 +427,6 @@ func (f *Fs) readRootFolderID(ctx context.Context, m configmap.Mapper) (err erro
 	return nil
 }
 
-// Find the top n servers measured by response time
-func (f *Fs) bestServers(ctx context.Context, servers []api.Server, n int) (newServers []api.Server) {
-	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))
-	defer cancel()
-
-	if n > len(servers) {
-		n = len(servers)
-	}
-	results := make(chan int, len(servers))
-
-	// Test how long the servers take to respond
-	for i := range servers {
-		i := i // for closure
-		go func() {
-			opts := rest.Opts{
-				Method:  "GET",
-				RootURL: servers[i].Root(),
-			}
-			var result api.UploadServerStatus
-			start := time.Now()
-			_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
-			ping := time.Since(start)
-			err = result.Err(err)
-			if err != nil {
-				results <- -1 // send a -ve number on error
-				return
-			}
-			fs.Debugf(nil, "Upload server %v responded in %v", &servers[i], ping)
-			results <- i
-		}()
-	}
-
-	// Wait for n servers to respond
-	newServers = make([]api.Server, 0, n)
-	for range servers {
-		i := <-results
-		if i >= 0 {
-			newServers = append(newServers, servers[i])
-		}
-		if len(newServers) >= n {
-			break
-		}
-	}
-	return newServers
-}
-
-// Clear all the upload servers - call on an error
-func (f *Fs) clearServers() {
-	f.serversMu.Lock()
-	defer f.serversMu.Unlock()
-
-	fs.Debugf(f, "Clearing upload servers")
-	f.servers = nil
-}
-
-// Gets an upload server
-func (f *Fs) getServer(ctx context.Context) (server *api.Server, err error) {
-	f.serversMu.Lock()
-	defer f.serversMu.Unlock()
-
-	if len(f.servers) == 0 || time.Since(f.serversChecked) >= serversExpiry {
-		opts := rest.Opts{
-			Method: "GET",
-			Path:   "/servers",
-		}
-		var result api.ServersResponse
-		var resp *http.Response
-		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-			return shouldRetry(ctx, resp, err)
-		})
-		if err = result.Err(err); err != nil {
-			if len(f.servers) == 0 {
-				return nil, fmt.Errorf("failed to read upload servers: %w", err)
-			}
-			fs.Errorf(f, "failed to read new upload servers: %v", err)
-		} else {
-			// Find the top servers measured by response time
-			f.servers = f.bestServers(ctx, result.Data.Servers, serversActive)
-			f.serversChecked = time.Now()
-		}
-	}
-
-	if len(f.servers) == 0 {
-		return nil, errors.New("no upload servers found")
-	}
-
-	// Pick a server at random since we've already found the top ones
-	i := rand.Intn(len(f.servers))
-	return &f.servers[i], nil
-}
-
 // rootSlash returns root with a slash on if it is empty, otherwise empty string
 func (f *Fs) rootSlash() string {
 	if f.root == "" {
@@ -1526,13 +1426,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// Find an upload server
-	server, err := o.fs.getServer(ctx)
-	if err != nil {
-		return err
-	}
-	fs.Debugf(o, "Using upload server %v", server)
-
 	// If the file exists, delete it after a successful upload
 	if o.id != "" {
 		id := o.id
@@ -1561,7 +1454,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		},
 		MultipartContentName: "file",
 		MultipartFileName:    o.fs.opt.Enc.FromStandardName(leaf),
-		RootURL:              server.URL(),
+		RootURL:              api.DirectUploadURL(),
 		Options:              options,
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1569,10 +1462,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return shouldRetry(ctx, resp, err)
 	})
 	if err = result.Err(err); err != nil {
-		if isAPIErr(err, "error-freespace") {
-			fs.Errorf(o, "Upload server out of space - need to retry upload")
-		}
-		o.fs.clearServers()
 		return fmt.Errorf("failed to upload file: %w", err)
 	}
 	return o.setMetaData(&result.Data)

backend/googlecloudstorage/googlecloudstorage.go

@@ -483,6 +483,9 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
 	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
+	if f.opt.DirectoryMarkers && strings.HasSuffix(bucketPath, "//") {
+		bucketPath = bucketPath[:len(bucketPath)-1]
+	}
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -712,7 +715,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 					continue
 				}
 				// process directory markers as directories
-				remote = strings.TrimRight(remote, "/")
+				remote, _ = strings.CutSuffix(remote, "/")
 			}
 			remote = remote[len(prefix):]
 			if addBucket {
@@ -959,7 +962,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote = strings.TrimRight(remote, "/")
+	remote, _ = strings.CutSuffix(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
 		dir = ""

backend/googlephotos/googlephotos.go

@@ -43,6 +43,7 @@ var (
 	errAlbumDelete = errors.New("google photos API does not implement deleting albums")
 	errRemove      = errors.New("google photos API only implements removing files from albums")
 	errOwnAlbums   = errors.New("google photos API only allows uploading to albums rclone created")
+	errReadOnly    = errors.New("can't upload files in read only mode")
 )
 
 const (
@@ -52,19 +53,31 @@ const (
 	listChunks                  = 100 // chunk size to read directory listings
 	albumChunks                 = 50  // chunk size to read album listings
 	minSleep                    = 10 * time.Millisecond
-	scopeReadOnly               = "https://www.googleapis.com/auth/photoslibrary.readonly"
-	scopeReadWrite              = "https://www.googleapis.com/auth/photoslibrary"
-	scopeAccess                 = 2 // position of access scope in list
+	scopeAppendOnly             = "https://www.googleapis.com/auth/photoslibrary.appendonly"
+	scopeReadOnly               = "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata"
+	scopeReadWrite              = "https://www.googleapis.com/auth/photoslibrary.edit.appcreateddata"
 )
 
 var (
+	// scopes needed for read write access
+	scopesReadWrite = []string{
+		"openid",
+		"profile",
+		scopeAppendOnly,
+		scopeReadOnly,
+		scopeReadWrite,
+	}
+
+	// scopes needed for read only access
+	scopesReadOnly = []string{
+		"openid",
+		"profile",
+		scopeReadOnly,
+	}
+
 	// Description of how to auth for this app
 	oauthConfig = &oauthutil.Config{
-		Scopes: []string{
-			"openid",
-			"profile",
-			scopeReadWrite, // this must be at position scopeAccess
-		},
+		Scopes:       scopesReadWrite,
 		AuthURL:      google.Endpoint.AuthURL,
 		TokenURL:     google.Endpoint.TokenURL,
 		ClientID:     rcloneClientID,
@@ -100,9 +113,9 @@ func init() {
 			case "":
 				// Fill in the scopes
 				if opt.ReadOnly {
-					oauthConfig.Scopes[scopeAccess] = scopeReadOnly
+					oauthConfig.Scopes = scopesReadOnly
 				} else {
-					oauthConfig.Scopes[scopeAccess] = scopeReadWrite
+					oauthConfig.Scopes = scopesReadWrite
 				}
 				return oauthutil.ConfigOut("warning", &oauthutil.Options{
 					OAuth2Config: oauthConfig,
@@ -333,7 +346,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	baseClient := fshttp.NewClient(ctx)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
 	if err != nil {
-		return nil, fmt.Errorf("failed to configure Box: %w", err)
+		return nil, fmt.Errorf("failed to configure google photos: %w", err)
 	}
 
 	root = strings.Trim(path.Clean(root), "/")
@@ -1120,6 +1133,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 
 		if !album.IsWriteable {
+			if o.fs.opt.ReadOnly {
+				return errReadOnly
+			}
 			return errOwnAlbums
 		}
 

backend/iclouddrive/api/drive.go

@@ -252,18 +252,14 @@ func (d *DriveService) DownloadFile(ctx context.Context, url string, opt []fs.Op
 	}
 
 	resp, err := d.icloud.srv.Call(ctx, opts)
-	if err != nil {
-		// icloud has some weird http codes
-		if resp.StatusCode == 330 {
-			loc, err := resp.Location()
-			if err == nil {
-				return d.DownloadFile(ctx, loc.String(), opt)
-			}
+	// icloud has some weird http codes
+	if err != nil && resp != nil && resp.StatusCode == 330 {
+		loc, err := resp.Location()
+		if err == nil {
+			return d.DownloadFile(ctx, loc.String(), opt)
 		}
-
-		return resp, err
 	}
-	return d.icloud.srv.Call(ctx, opts)
+	return resp, err
 }
 
 // MoveItemToTrashByItemID moves an item to the trash based on the item ID.

backend/imagekit/imagekit.go

@@ -421,6 +421,9 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // will return the object and the error, otherwise will return
 // nil and the error
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	if src.Size() == 0 {
+		return nil, fs.ErrorCantUploadEmptyFiles
+	}
 	return uploadFile(ctx, f, in, src.Remote(), options...)
 }
 
@@ -659,6 +662,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	if src.Size() == 0 {
+		return fs.ErrorCantUploadEmptyFiles
+	}
 
 	srcRemote := o.Remote()
 
@@ -670,7 +676,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	var resp *client.UploadResult
 
-	err = o.fs.pacer.Call(func() (bool, error) {
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		var res *http.Response
 		res, resp, err = o.fs.ik.Upload(ctx, in, client.UploadParam{
 			FileName:      fileName,
@@ -725,7 +731,7 @@ func uploadFile(ctx context.Context, f *Fs, in io.Reader, srcRemote string, opti
 	UseUniqueFileName := new(bool)
 	*UseUniqueFileName = false
 
-	err := f.pacer.Call(func() (bool, error) {
+	err := f.pacer.CallNoRetry(func() (bool, error) {
 		var res *http.Response
 		var err error
 		res, _, err = f.ik.Upload(ctx, in, client.UploadParam{
@@ -794,35 +800,10 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
 	return metadata, nil
 }
 
-// Copy src to this remote using server-side move operations.
-//
-// This is stored with the remote path given.
-//
-// It returns the destination Object and a possible error.
-//
-// Will only be called if src.Fs().Name() == f.Name()
-//
-// If it isn't possible then return fs.ErrorCantMove
-func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	srcObj, ok := src.(*Object)
-	if !ok {
-		return nil, fs.ErrorCantMove
-	}
-
-	file, err := srcObj.Open(ctx)
-
-	if err != nil {
-		return nil, err
-	}
-
-	return uploadFile(ctx, f, file, remote)
-}
-
 // Check the interfaces are satisfied.
 var (
 	_ fs.Fs           = &Fs{}
 	_ fs.Purger       = &Fs{}
 	_ fs.PublicLinker = &Fs{}
 	_ fs.Object       = &Object{}
-	_ fs.Copier       = &Fs{}
 )

backend/koofr/koofr.go

@@ -461,7 +461,7 @@ func translateErrorsDir(err error) error {
 	return err
 }
 
-// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
+// translateErrorsObject translates Koofr errors to rclone errors (for an object operation)
 func translateErrorsObject(err error) error {
 	switch err := err.(type) {
 	case httpclient.InvalidStatusError:

backend/linkbox/linkbox.go

@@ -617,16 +617,36 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	case 1:
 		// upload file using link from first step
 		var res *http.Response
+		var location string
+
+		// Check to see if we are being redirected
+		opts := &rest.Opts{
+			Method:     "HEAD",
+			RootURL:    getFirstStepResult.Data.SignURL,
+			Options:    options,
+			NoRedirect: true,
+		}
+		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+			res, err = o.fs.srv.Call(ctx, opts)
+			return o.fs.shouldRetry(ctx, res, err)
+		})
+		if res != nil {
+			location = res.Header.Get("Location")
+			if location != "" {
+				// set the URL to the new Location
+				opts.RootURL = location
+				err = nil
+			}
+		}
+		if err != nil {
+			return fmt.Errorf("head upload URL: %w", err)
+		}
 
 		file := io.MultiReader(bytes.NewReader(first10mBytes), in)
 
-		opts := &rest.Opts{
-			Method:        "PUT",
-			RootURL:       getFirstStepResult.Data.SignURL,
-			Options:       options,
-			Body:          file,
-			ContentLength: &size,
-		}
+		opts.Method = "PUT"
+		opts.Body = file
+		opts.ContentLength = &size
 
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			res, err = o.fs.srv.Call(ctx, opts)

backend/local/local.go

@@ -305,6 +305,12 @@ only useful for reading.
 					Help:  "The last status change time.",
 				}},
 			},
+			{
+				Name:     "hashes",
+				Help:     `Comma separated list of supported checksum types.`,
+				Default:  fs.CommaSepList{},
+				Advanced: true,
+			},
 			{
 				Name:     config.ConfigEncoding,
 				Help:     config.ConfigEncodingHelp,
@@ -331,6 +337,7 @@ type Options struct {
 	NoSparse          bool                 `config:"no_sparse"`
 	NoSetModTime      bool                 `config:"no_set_modtime"`
 	TimeType          timeType             `config:"time_type"`
+	Hashes            fs.CommaSepList      `config:"hashes"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 	NoClone           bool                 `config:"no_clone"`
 }
@@ -1021,6 +1028,19 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
+	if len(f.opt.Hashes) > 0 {
+		// Return only configured hashes.
+		// Note: Could have used hash.SupportOnly to limit supported hashes for all hash related features.
+		var supported hash.Set
+		for _, hashName := range f.opt.Hashes {
+			var ht hash.Type
+			if err := ht.Set(hashName); err != nil {
+				fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
+			}
+			supported.Add(ht)
+		}
+		return supported
+	}
 	return hash.Supported()
 }
 
@@ -1090,6 +1110,10 @@ func (o *Object) Remote() string {
 
 // Hash returns the requested hash of a file as a lowercase hex string
 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
+	if r == hash.None {
+		return "", nil
+	}
+
 	// Check that the underlying file hasn't changed
 	o.fs.objectMetaMu.RLock()
 	oldtime := o.modTime
@@ -1197,7 +1221,15 @@ func (o *Object) Storable() bool {
 	o.fs.objectMetaMu.RLock()
 	mode := o.mode
 	o.fs.objectMetaMu.RUnlock()
-	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
+
+	// On Windows items with os.ModeIrregular are likely Junction
+	// points so we treat them as symlinks for the purpose of ignoring them.
+	// https://github.com/golang/go/issues/73827
+	symlinkFlag := os.ModeSymlink
+	if runtime.GOOS == "windows" {
+		symlinkFlag |= os.ModeIrregular
+	}
+	if mode&symlinkFlag != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}

backend/local/local_internal_test.go

@@ -204,6 +204,23 @@ func TestSymlinkError(t *testing.T) {
 	assert.Equal(t, errLinksAndCopyLinks, err)
 }
 
+func TestHashWithTypeNone(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	const filePath = "file.txt"
+	r.WriteFile(filePath, "content", time.Now())
+	f := r.Flocal.(*Fs)
+
+	// Get the object
+	o, err := f.NewObject(ctx, filePath)
+	require.NoError(t, err)
+
+	// Test the hash is as we expect
+	h, err := o.Hash(ctx, hash.None)
+	require.Empty(t, h)
+	require.NoError(t, err)
+}
+
 // Test hashes on updating an object
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()

backend/mailru/mailru.go

@@ -634,7 +634,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 	return
 }
 
-// itemToEntry converts API item to rclone directory entry
+// itemToDirEntry converts API item to rclone directory entry
 // The dirSize return value is:
 //
 //	<0 - for a file or in case of error

backend/mega/mega.go

@@ -17,9 +17,11 @@ Improvements:
 
 import (
 	"context"
+	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"path"
 	"slices"
 	"strings"
@@ -216,7 +218,25 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	defer megaCacheMu.Unlock()
 	srv := megaCache[opt.User]
 	if srv == nil {
-		srv = mega.New().SetClient(fshttp.NewClient(ctx))
+		// srv = mega.New().SetClient(fshttp.NewClient(ctx))
+
+		// Workaround for Mega's use of insecure cipher suites which are no longer supported by default since Go 1.22.
+		// Relevant issues:
+		// https://github.com/rclone/rclone/issues/8565
+		// https://github.com/meganz/webclient/issues/103
+		clt := fshttp.NewClient(ctx)
+		clt.Transport = fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
+			var ids []uint16
+			// Read default ciphers
+			for _, cs := range tls.CipherSuites() {
+				ids = append(ids, cs.ID)
+			}
+			// Insecure but Mega uses TLS_RSA_WITH_AES_128_GCM_SHA256 for storage endpoints
+			// (e.g. https://gfs302n114.userstorage.mega.co.nz) as of June 18, 2025.
+			t.TLSClientConfig.CipherSuites = append(ids, tls.TLS_RSA_WITH_AES_128_GCM_SHA256)
+		})
+		srv = mega.New().SetClient(clt)
+
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
 		srv.SetHTTPS(opt.UseHTTPS)
 		srv.SetLogger(func(format string, v ...any) {

backend/onedrive/metadata.go

@@ -749,6 +749,8 @@ func (o *Object) fetchMetadataForCreate(ctx context.Context, src fs.ObjectInfo,
 
 // Fetch metadata and update updateInfo if --metadata is in use
 // modtime will still be set when there is no metadata to set
+//
+// May return info=nil and err=nil if there was no metadata to update.
 func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *Object) (info *api.Item, err error) {
 	meta, err := fs.GetMetadataOptions(ctx, f, src, options)
 	if err != nil {
@@ -768,6 +770,8 @@ func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, opti
 }
 
 // updateMetadata calls Get, Set, and Write
+//
+// May return info=nil and err=nil if there was no metadata to update.
 func (o *Object) updateMetadata(ctx context.Context, meta fs.Metadata) (info *api.Item, err error) {
 	_, err = o.meta.Get(ctx) // refresh permissions
 	if err != nil {

backend/onedrive/onedrive.go

@@ -56,6 +56,7 @@ const (
 	driveTypeSharepoint         = "documentLibrary"
 	defaultChunkSize            = 10 * fs.Mebi
 	chunkSizeMultiple           = 320 * fs.Kibi
+	maxSinglePartSize           = 4 * fs.Mebi
 
 	regionGlobal = "global"
 	regionUS     = "us"
@@ -138,6 +139,21 @@ func init() {
 					Help:  "Azure and Office 365 operated by Vnet Group in China",
 				},
 			},
+		}, {
+			Name: "upload_cutoff",
+			Help: `Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+
+This is disabled by default as uploading using single part uploads
+causes rclone to use twice the storage on Onedrive business as when
+rclone sets the modification time after the upload Onedrive creates a
+new version.
+
+See: https://github.com/rclone/rclone/issues/1716
+`,
+			Default:  fs.SizeSuffix(-1),
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
@@ -746,6 +762,7 @@ Examples:
 // Options defines the configuration for this backend
 type Options struct {
 	Region                  string               `config:"region"`
+	UploadCutoff            fs.SizeSuffix        `config:"upload_cutoff"`
 	ChunkSize               fs.SizeSuffix        `config:"chunk_size"`
 	DriveID                 string               `config:"drive_id"`
 	DriveType               string               `config:"drive_type"`
@@ -1022,6 +1039,13 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
+func checkUploadCutoff(cs fs.SizeSuffix) error {
+	if cs > maxSinglePartSize {
+		return fmt.Errorf("%v is greater than %v", cs, maxSinglePartSize)
+	}
+	return nil
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -1035,6 +1059,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("onedrive: chunk size: %w", err)
 	}
+	err = checkUploadCutoff(opt.UploadCutoff)
+	if err != nil {
+		return nil, fmt.Errorf("onedrive: upload cutoff: %w", err)
+	}
 
 	if opt.DriveID == "" || opt.DriveType == "" {
 		return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
@@ -1754,7 +1782,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	if err != nil {
 		return nil, err
 	}
-	err = dstObj.setMetaData(info)
+	if info != nil {
+		err = dstObj.setMetaData(info)
+	}
 	return dstObj, err
 }
 
@@ -1834,7 +1864,9 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	err = dstObj.setMetaData(info)
+	if info != nil {
+		err = dstObj.setMetaData(info)
+	}
 	return dstObj, err
 }
 
@@ -2469,6 +2501,10 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 				return false, nil
 			}
 			return true, fmt.Errorf("retry this chunk skipping %d bytes: %w", skip, err)
+		} else if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound {
+			fs.Debugf(o, "Received 404 error: assuming eventual consistency problem with session - retrying chunk: %v", err)
+			time.Sleep(5 * time.Second) // a little delay to help things along
+			return true, err
 		}
 		if err != nil {
 			return shouldRetry(ctx, resp, err)
@@ -2563,8 +2599,8 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 // This function will set modtime and metadata after uploading, which will create a new version for the remote file
 func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
 	size := src.Size()
-	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
-		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
+	if size < 0 || size > int64(maxSinglePartSize) {
+		return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
 	}
 
 	fs.Debugf(o, "Starting singlepart upload")
@@ -2597,7 +2633,10 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.Obje
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch and update metadata: %w", err)
 	}
-	return info, o.setMetaData(info)
+	if info != nil {
+		err = o.setMetaData(info)
+	}
+	return info, err
 }
 
 // Update the object with the contents of the io.Reader, modTime and size
@@ -2617,9 +2656,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	size := src.Size()
 
 	var info *api.Item
-	if size > 0 {
+	if size > 0 && size >= int64(o.fs.opt.UploadCutoff) {
 		info, err = o.uploadMultipart(ctx, in, src, options...)
-	} else if size == 0 {
+	} else if size >= 0 {
 		info, err = o.uploadSinglepart(ctx, in, src, options...)
 	} else {
 		return errors.New("unknown-sized upload not supported")

backend/pcloud/pcloud.go

@@ -378,12 +378,20 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
-// OpenWriterAt opens with a handle for random access writes
+// XOpenWriterAt opens with a handle for random access writes
 //
 // Pass in the remote desired and the size if known.
 //
-// It truncates any existing object
-func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
+// It truncates any existing object.
+//
+// OpenWriterAt disabled because it seems to have been disabled at pcloud
+// PUT /file_open?flags=XXX&folderid=XXX&name=XXX HTTP/1.1
+//
+//	{
+//	        "result": 2003,
+//	        "error": "Access denied. You do not have permissions to perform this operation."
+//	}
+func (f *Fs) XOpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
 	client, err := f.newSingleConnClient(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("create client: %w", err)

backend/pikpak/helper.go

@@ -155,6 +155,7 @@ func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error)
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
 		if err == nil && !info.Links.ApplicationOctetStream.Valid() {
+			time.Sleep(5 * time.Second)
 			return true, errors.New("no link")
 		}
 		return f.shouldRetry(ctx, resp, err)

backend/pikpak/multipart.go

@@ -0,0 +1,333 @@
+package pikpak
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/rclone/rclone/backend/pikpak/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/chunksize"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/atexit"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	bufferSize           = 1024 * 1024     // default size of the pages used in the reader
+	bufferCacheSize      = 64              // max number of buffers to keep in cache
+	bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
+)
+
+// bufferPool is a global pool of buffers
+var (
+	bufferPool     *pool.Pool
+	bufferPoolOnce sync.Once
+)
+
+// get a buffer pool
+func getPool() *pool.Pool {
+	bufferPoolOnce.Do(func() {
+		ci := fs.GetConfig(context.Background())
+		// Initialise the buffer pool when used
+		bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
+	})
+	return bufferPool
+}
+
+// NewRW gets a pool.RW using the multipart pool
+func NewRW() *pool.RW {
+	return pool.NewRW(getPool())
+}
+
+// Upload does a multipart upload in parallel
+func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
+	// make concurrency machinery
+	tokens := pacer.NewTokenDispenser(w.con)
+
+	uploadCtx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	defer atexit.OnError(&err, func() {
+		cancel()
+		fs.Debugf(w.o, "multipart upload: Cancelling...")
+		errCancel := w.Abort(ctx)
+		if errCancel != nil {
+			fs.Debugf(w.o, "multipart upload: failed to cancel: %v", errCancel)
+		}
+	})()
+
+	var (
+		g, gCtx   = errgroup.WithContext(uploadCtx)
+		finished  = false
+		off       int64
+		size      = w.size
+		chunkSize = w.chunkSize
+	)
+
+	// Do the accounting manually
+	in, acc := accounting.UnWrapAccounting(w.in)
+
+	for partNum := int64(0); !finished; partNum++ {
+		// Get a block of memory from the pool and token which limits concurrency.
+		tokens.Get()
+		rw := NewRW()
+		if acc != nil {
+			rw.SetAccounting(acc.AccountRead)
+		}
+
+		free := func() {
+			// return the memory and token
+			_ = rw.Close() // Can't return an error
+			tokens.Put()
+		}
+
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in uploading all the other parts.
+		if gCtx.Err() != nil {
+			free()
+			break
+		}
+
+		// Read the chunk
+		var n int64
+		n, err = io.CopyN(rw, in, chunkSize)
+		if err == io.EOF {
+			if n == 0 && partNum != 0 { // end if no data and if not first chunk
+				free()
+				break
+			}
+			finished = true
+		} else if err != nil {
+			free()
+			return fmt.Errorf("multipart upload: failed to read source: %w", err)
+		}
+
+		partNum := partNum
+		partOff := off
+		off += n
+		g.Go(func() (err error) {
+			defer free()
+			fs.Debugf(w.o, "multipart upload: starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(partOff), fs.SizeSuffix(size))
+			_, err = w.WriteChunk(gCtx, int32(partNum), rw)
+			return err
+		})
+	}
+
+	err = g.Wait()
+	if err != nil {
+		return err
+	}
+
+	err = w.Close(ctx)
+	if err != nil {
+		return fmt.Errorf("multipart upload: failed to finalise: %w", err)
+	}
+
+	return nil
+}
+
+var warnStreamUpload sync.Once
+
+// state of ChunkWriter
+type pikpakChunkWriter struct {
+	chunkSize      int64
+	size           int64
+	con            int
+	f              *Fs
+	o              *Object
+	in             io.Reader
+	mu             sync.Mutex
+	completedParts []types.CompletedPart
+	client         *s3.Client
+	mOut           *s3.CreateMultipartUploadOutput
+}
+
+func (f *Fs) newChunkWriter(ctx context.Context, remote string, size int64, p *api.ResumableParams, in io.Reader, options ...fs.OpenOption) (w *pikpakChunkWriter, err error) {
+	// Temporary Object under construction
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+
+	// calculate size of parts
+	chunkSize := f.opt.ChunkSize
+
+	// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
+	// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
+	// 48 GiB which seems like a not too unreasonable limit.
+	if size == -1 {
+		warnStreamUpload.Do(func() {
+			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				f.opt.ChunkSize, fs.SizeSuffix(int64(chunkSize)*int64(maxUploadParts)))
+		})
+	} else {
+		chunkSize = chunksize.Calculator(o, size, maxUploadParts, chunkSize)
+	}
+
+	client, err := f.newS3Client(ctx, p)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create upload client: %w", err)
+	}
+	w = &pikpakChunkWriter{
+		chunkSize:      int64(chunkSize),
+		size:           size,
+		con:            max(1, f.opt.UploadConcurrency),
+		f:              f,
+		o:              o,
+		in:             in,
+		completedParts: make([]types.CompletedPart, 0),
+		client:         client,
+	}
+
+	req := &s3.CreateMultipartUploadInput{
+		Bucket: &p.Bucket,
+		Key:    &p.Key,
+	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			req.CacheControl = aws.String(value)
+		case "content-disposition":
+			req.ContentDisposition = aws.String(value)
+		case "content-encoding":
+			req.ContentEncoding = aws.String(value)
+		case "content-type":
+			req.ContentType = aws.String(value)
+		}
+	}
+	err = w.f.pacer.Call(func() (bool, error) {
+		w.mOut, err = w.client.CreateMultipartUpload(ctx, req)
+		return w.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("create multipart upload failed: %w", err)
+	}
+	fs.Debugf(w.o, "multipart upload: %q initiated", *w.mOut.UploadId)
+	return
+}
+
+// shouldRetry returns a boolean as to whether this err
+// deserve to be retried. It returns the err as a convenience
+func (w *pikpakChunkWriter) shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+	if fserrors.ShouldRetry(err) {
+		return true, err
+	}
+	return false, err
+}
+
+// add a part number and etag to the completed parts
+func (w *pikpakChunkWriter) addCompletedPart(part types.CompletedPart) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.completedParts = append(w.completedParts, part)
+}
+
+// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
+func (w *pikpakChunkWriter) WriteChunk(ctx context.Context, chunkNumber int32, reader io.ReadSeeker) (currentChunkSize int64, err error) {
+	if chunkNumber < 0 {
+		err := fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
+		return -1, err
+	}
+
+	partNumber := chunkNumber + 1
+	var res *s3.UploadPartOutput
+	err = w.f.pacer.Call(func() (bool, error) {
+		// Discover the size by seeking to the end
+		currentChunkSize, err = reader.Seek(0, io.SeekEnd)
+		if err != nil {
+			return false, err
+		}
+		// rewind the reader on retry and after reading md5
+		_, err := reader.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		res, err = w.client.UploadPart(ctx, &s3.UploadPartInput{
+			Bucket:     w.mOut.Bucket,
+			Key:        w.mOut.Key,
+			UploadId:   w.mOut.UploadId,
+			PartNumber: &partNumber,
+			Body:       reader,
+		})
+		if err != nil {
+			if chunkNumber <= 8 {
+				return w.shouldRetry(ctx, err)
+			}
+			// retry all chunks once have done the first few
+			return true, err
+		}
+		return false, nil
+	})
+	if err != nil {
+		return -1, fmt.Errorf("failed to upload chunk %d with %v bytes: %w", partNumber, currentChunkSize, err)
+	}
+
+	w.addCompletedPart(types.CompletedPart{
+		PartNumber: &partNumber,
+		ETag:       res.ETag,
+	})
+
+	fs.Debugf(w.o, "multipart upload: wrote chunk %d with %v bytes", partNumber, currentChunkSize)
+	return currentChunkSize, err
+}
+
+// Abort the multipart upload
+func (w *pikpakChunkWriter) Abort(ctx context.Context) (err error) {
+	// Abort the upload session
+	err = w.f.pacer.Call(func() (bool, error) {
+		_, err = w.client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
+			Bucket:   w.mOut.Bucket,
+			Key:      w.mOut.Key,
+			UploadId: w.mOut.UploadId,
+		})
+		return w.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to abort multipart upload %q: %w", *w.mOut.UploadId, err)
+	}
+	fs.Debugf(w.o, "multipart upload: %q aborted", *w.mOut.UploadId)
+	return
+}
+
+// Close and finalise the multipart upload
+func (w *pikpakChunkWriter) Close(ctx context.Context) (err error) {
+	// sort the completed parts by part number
+	sort.Slice(w.completedParts, func(i, j int) bool {
+		return *w.completedParts[i].PartNumber < *w.completedParts[j].PartNumber
+	})
+	// Finalise the upload session
+	err = w.f.pacer.Call(func() (bool, error) {
+		_, err = w.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
+			Bucket:   w.mOut.Bucket,
+			Key:      w.mOut.Key,
+			UploadId: w.mOut.UploadId,
+			MultipartUpload: &types.CompletedMultipartUpload{
+				Parts: w.completedParts,
+			},
+		})
+		return w.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to complete multipart upload: %w", err)
+	}
+	fs.Debugf(w.o, "multipart upload: %q finished", *w.mOut.UploadId)
+	return
+}

backend/pikpak/pikpak.go

@@ -41,12 +41,10 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
-	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/rclone/rclone/backend/pikpak/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
@@ -66,17 +64,22 @@ import (
 
 // Constants
 const (
-	clientID                 = "YUMx5nI8ZU8Ap8pm"
-	clientVersion            = "2.0.0"
-	packageName              = "mypikpak.com"
-	defaultUserAgent         = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
-	minSleep                 = 100 * time.Millisecond
-	maxSleep                 = 2 * time.Second
-	taskWaitTime             = 500 * time.Millisecond
-	decayConstant            = 2 // bigger for slower decay, exponential
-	rootURL                  = "https://api-drive.mypikpak.com"
-	minChunkSize             = fs.SizeSuffix(manager.MinUploadPartSize)
-	defaultUploadConcurrency = manager.DefaultUploadConcurrency
+	clientID         = "YUMx5nI8ZU8Ap8pm"
+	clientVersion    = "2.0.0"
+	packageName      = "mypikpak.com"
+	defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
+	minSleep         = 100 * time.Millisecond
+	maxSleep         = 2 * time.Second
+	taskWaitTime     = 500 * time.Millisecond
+	decayConstant    = 2 // bigger for slower decay, exponential
+	rootURL          = "https://api-drive.mypikpak.com"
+
+	maxUploadParts      = 10000                          // Part number must be an integer between 1 and 10000, inclusive.
+	defaultChunkSize    = fs.SizeSuffix(1024 * 1024 * 5) // Part size should be in [100KB, 5GB]
+	minChunkSize        = 100 * fs.Kibi
+	maxChunkSize        = 5 * fs.Gibi
+	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
+	maxUploadCutoff     = 5 * fs.Gibi // maximum allowed size for singlepart uploads
 )
 
 // Globals
@@ -223,6 +226,14 @@ Fill in for rclone to use a non root folder as its starting point.
 			Help:     "Files bigger than this will be cached on disk to calculate hash if required.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
+		}, {
+			Name: "upload_cutoff",
+			Help: `Cutoff for switching to chunked upload.
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5 GiB.`,
+			Default:  defaultUploadCutoff,
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Chunk size for multipart uploads.
@@ -241,7 +252,7 @@ large file of known size to stay below the 10,000 chunks limit.
 
 Increasing the chunk size decreases the accuracy of the progress
 statistics displayed with "-P" flag.`,
-			Default:  minChunkSize,
+			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
 			Name: "upload_concurrency",
@@ -257,7 +268,7 @@ in memory.
 If you are uploading small numbers of large files over high-speed links
 and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
-			Default:  defaultUploadConcurrency,
+			Default:  4,
 			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
@@ -294,6 +305,7 @@ type Options struct {
 	NoMediaLink         bool                 `config:"no_media_link"`
 	HashMemoryThreshold fs.SizeSuffix        `config:"hash_memory_limit"`
 	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
+	UploadCutoff        fs.SizeSuffix        `config:"upload_cutoff"`
 	UploadConcurrency   int                  `config:"upload_concurrency"`
 	Enc                 encoder.MultiEncoder `config:"encoding"`
 }
@@ -467,6 +479,11 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 			// when a zero-byte file was uploaded with an invalid captcha token
 			f.rst.captcha.Invalidate()
 			return true, err
+		} else if strings.Contains(apiErr.Reason, "idx.shub.mypikpak.com") && apiErr.Code == 500 {
+			// internal server error: Post "http://idx.shub.mypikpak.com": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
+			// This typically happens when trying to retrieve a gcid for which no record exists.
+			// No retry is needed in this case.
+			return false, err
 		}
 	}
 
@@ -524,6 +541,39 @@ func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
 	return nil
 }
 
+func checkUploadChunkSize(cs fs.SizeSuffix) error {
+	if cs < minChunkSize {
+		return fmt.Errorf("%s is less than %s", cs, minChunkSize)
+	}
+	if cs > maxChunkSize {
+		return fmt.Errorf("%s is greater than %s", cs, maxChunkSize)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadChunkSize(cs)
+	if err == nil {
+		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	}
+	return
+}
+
+func checkUploadCutoff(cs fs.SizeSuffix) error {
+	if cs > maxUploadCutoff {
+		return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadCutoff(cs)
+	if err == nil {
+		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
+	}
+	return
+}
+
 // newFs partially constructs Fs from the path
 //
 // It constructs a valid Fs but doesn't attempt to figure out whether
@@ -531,11 +581,17 @@ func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
 func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
-	if err := configstruct.Set(m, opt); err != nil {
+	err := configstruct.Set(m, opt)
+	if err != nil {
 		return nil, err
 	}
-	if opt.ChunkSize < minChunkSize {
-		return nil, fmt.Errorf("chunk size must be at least %s", minChunkSize)
+	err = checkUploadChunkSize(opt.ChunkSize)
+	if err != nil {
+		return nil, fmt.Errorf("pikpak: chunk size: %w", err)
+	}
+	err = checkUploadCutoff(opt.UploadCutoff)
+	if err != nil {
+		return nil, fmt.Errorf("pikpak: upload cutoff: %w", err)
 	}
 
 	root := parsePath(path)
@@ -1260,9 +1316,7 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
 	return
 }
 
-func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable) (err error) {
-	p := resumable.Params
-
+func (f *Fs) newS3Client(ctx context.Context, p *api.ResumableParams) (s3Client *s3.Client, err error) {
 	// Create a credentials provider
 	creds := credentials.NewStaticCredentialsProvider(p.AccessKeyID, p.AccessKeySecret, p.SecurityToken)
 
@@ -1272,22 +1326,64 @@ func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, s
 	if err != nil {
 		return
 	}
+	ci := fs.GetConfig(ctx)
+	cfg.RetryMaxAttempts = ci.LowLevelRetries
+	cfg.HTTPClient = getClient(ctx, &f.opt)
 
 	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
 		o.BaseEndpoint = aws.String("https://mypikpak.com/")
+		o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+		o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
 	})
-	partSize := chunksize.Calculator(name, size, int(manager.MaxUploadParts), f.opt.ChunkSize)
+	return client, nil
+}
 
-	// Create an uploader with custom options
-	uploader := manager.NewUploader(client, func(u *manager.Uploader) {
-		u.PartSize = int64(partSize)
-		u.Concurrency = f.opt.UploadConcurrency
-	})
-	// Perform an upload
-	_, err = uploader.Upload(ctx, &s3.PutObjectInput{
+func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable, options ...fs.OpenOption) (err error) {
+	p := resumable.Params
+
+	if size < 0 || size >= int64(f.opt.UploadCutoff) {
+		mu, err := f.newChunkWriter(ctx, name, size, p, in, options...)
+		if err != nil {
+			return fmt.Errorf("multipart upload failed to initialise: %w", err)
+		}
+		return mu.Upload(ctx)
+	}
+
+	// upload singlepart
+	client, err := f.newS3Client(ctx, p)
+	if err != nil {
+		return fmt.Errorf("failed to create upload client: %w", err)
+	}
+	req := &s3.PutObjectInput{
 		Bucket: &p.Bucket,
 		Key:    &p.Key,
-		Body:   in,
+		Body:   io.NopCloser(in),
+	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			req.CacheControl = aws.String(value)
+		case "content-disposition":
+			req.ContentDisposition = aws.String(value)
+		case "content-encoding":
+			req.ContentEncoding = aws.String(value)
+		case "content-type":
+			req.ContentType = aws.String(value)
+		}
+	}
+	var s3opts = []func(*s3.Options){}
+	// Can't retry single part uploads as only have an io.Reader
+	s3opts = append(s3opts, func(o *s3.Options) {
+		o.RetryMaxAttempts = 1
+	})
+	err = f.pacer.CallNoRetry(func() (bool, error) {
+		_, err = client.PutObject(ctx, req, s3opts...)
+		return f.shouldRetry(ctx, nil, err)
 	})
 	return
 }
@@ -1345,7 +1441,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, gcid string,
 	if uploadType == api.UploadTypeForm && new.Form != nil {
 		err = f.uploadByForm(ctx, in, req.Name, size, new.Form, options...)
 	} else if uploadType == api.UploadTypeResumable && new.Resumable != nil {
-		err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable)
+		err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable, options...)
 	} else {
 		err = fmt.Errorf("no method available for uploading: %+v", new)
 	}

backend/pikpak/pikpak_test.go

@@ -1,10 +1,10 @@
 // Test PikPak filesystem interface
-package pikpak_test
+package pikpak
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/backend/pikpak"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -12,6 +12,23 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestPikPak:",
-		NilObject:  (*pikpak.Object)(nil),
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+			MaxChunkSize: maxChunkSize,
+		},
 	})
 }
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadChunkSize(cs)
+}
+
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadCutoff(cs)
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)

backend/s3/s3.go

@@ -101,6 +101,12 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Dreamhost",
 		Help:  "Dreamhost DreamObjects",
+	}, {
+		Value: "Exaba",
+		Help:  "Exaba Object Storage",
+	}, {
+		Value: "FlashBlade",
+		Help:  "Pure Storage FlashBlade Object Storage",
 	}, {
 		Value: "GCS",
 		Help:  "Google Cloud Storage",
@@ -131,6 +137,9 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Magalu",
 		Help:  "Magalu Object Storage",
+	}, {
+		Value: "Mega",
+		Help:  "MEGA S4 Object Storage",
 	}, {
 		Value: "Minio",
 		Help:  "Minio Object Storage",
@@ -176,6 +185,9 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Qiniu",
 		Help:  "Qiniu Object Storage (Kodo)",
+	}, {
+		Value: "Zata",
+		Help:  "Zata (S3 compatible Gateway)",
 	}, {
 		Value: "Other",
 		Help:  "Any other S3 compatible provider",
@@ -481,6 +493,14 @@ func init() {
 				Value: "ap-northeast-1",
 				Help:  "Northeast Asia Region 1.\nNeeds location constraint ap-northeast-1.",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region where you can connect with.\n",
+			Provider: "Zata",
+			Examples: []fs.OptionExample{{
+				Value: "us-east-1",
+				Help:  "Indore, Madhya Pradesh, India",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region where your bucket will be created and your data stored.\n",
@@ -567,7 +587,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive",
+			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega,Zata",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -1003,6 +1023,12 @@ func init() {
 				Value: "us-iad-1.linodeobjects.com",
 				Help:  "Washington, DC, (USA), us-iad-1",
 			}},
+		}, {
+			// Lyve Cloud endpoints
+			Name:     "endpoint",
+			Help:     "Endpoint for Lyve Cloud S3 API.\nRequired when using an S3 clone. Please type in your LyveCloud endpoint.\nExamples:\n- s3.us-west-1.{account_name}.lyve.seagate.com (US West 1 - California)\n- s3.eu-west-1.{account_name}.lyve.seagate.com (EU West 1 - Ireland)",
+			Provider: "LyveCloud",
+			Required: true,
 		}, {
 			// Magalu endpoints: https://docs.magalu.cloud/docs/object-storage/how-to/copy-url
 			Name:     "endpoint",
@@ -1365,6 +1391,14 @@ func init() {
 				Value: "s3-ap-northeast-1.qiniucs.com",
 				Help:  "Northeast Asia Endpoint 1",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Zata Object Storage.",
+			Provider: "Zata",
+			Examples: []fs.OptionExample{{
+				Value: "idr01.zata.ai",
+				Help:  "South Asia Endpoint",
+			}},
 		}, {
 			// Selectel endpoints: https://docs.selectel.ru/en/cloud/object-storage/manage/domains/#s3-api-domains
 			Name:     "endpoint",
@@ -1377,7 +1411,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox,Zata",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1426,18 +1460,6 @@ func init() {
 				Value:    "localhost:8333",
 				Help:     "SeaweedFS S3 localhost",
 				Provider: "SeaweedFS",
-			}, {
-				Value:    "s3.us-east-1.lyvecloud.seagate.com",
-				Help:     "Seagate Lyve Cloud US East 1 (Virginia)",
-				Provider: "LyveCloud",
-			}, {
-				Value:    "s3.us-west-1.lyvecloud.seagate.com",
-				Help:     "Seagate Lyve Cloud US West 1 (California)",
-				Provider: "LyveCloud",
-			}, {
-				Value:    "s3.ap-southeast-1.lyvecloud.seagate.com",
-				Help:     "Seagate Lyve Cloud AP Southeast 1 (Singapore)",
-				Provider: "LyveCloud",
 			}, {
 				Value:    "oos.eu-west-2.outscale.com",
 				Help:     "Outscale EU West 2 (Paris)",
@@ -1526,6 +1548,22 @@ func init() {
 				Value:    "s3.ir-tbz-sh1.arvanstorage.ir",
 				Help:     "ArvanCloud Tabriz Iran (Shahriar) endpoint",
 				Provider: "ArvanCloud",
+			}, {
+				Value:    "s3.eu-central-1.s4.mega.io",
+				Help:     "Mega S4 eu-central-1 (Amsterdam)",
+				Provider: "Mega",
+			}, {
+				Value:    "s3.eu-central-2.s4.mega.io",
+				Help:     "Mega S4 eu-central-2 (Bettembourg)",
+				Provider: "Mega",
+			}, {
+				Value:    "s3.ca-central-1.s4.mega.io",
+				Help:     "Mega S4 ca-central-1 (Montreal)",
+				Provider: "Mega",
+			}, {
+				Value:    "s3.ca-west-1.s4.mega.io",
+				Help:     "Mega S4 ca-west-1 (Vancouver)",
+				Provider: "Mega",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -1908,7 +1946,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox",
+			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,FlashBlade,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox,Mega",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1923,7 +1961,7 @@ doesn't copy the ACL from the source but rather writes a fresh one.
 If the acl is an empty string then no X-Amz-Acl: header is added and
 the default (private) will be used.
 `,
-			Provider: "!Storj,Selectel,Synology,Cloudflare",
+			Provider: "!Storj,Selectel,Synology,Cloudflare,FlashBlade,Mega",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
 				Help:     "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -1981,6 +2019,7 @@ isn't set then "acl" is used instead.
 If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl:
 header is added and the default (private) will be used.
 `,
+			Provider: "!Storj,Selectel,Synology,Cloudflare,FlashBlade",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "private",
@@ -2596,7 +2635,7 @@ The parameter should be a date, "2006-01-02", datetime "2006-01-02
 Note that when using this no file write operations are permitted,
 so you can't upload files or delete them.
 
-See [the time option docs](/docs/#time-option) for valid formats.
+See [the time option docs](/docs/#time-options) for valid formats.
 `,
 			Default:  fs.Time{},
 			Advanced: true,
@@ -3116,6 +3155,9 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
 	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
+	if f.opt.DirectoryMarkers && strings.HasSuffix(bucketPath, "//") {
+		bucketPath = bucketPath[:len(bucketPath)-1]
+	}
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -3495,6 +3537,9 @@ func setQuirks(opt *Options) {
 	case "Dreamhost":
 		urlEncodeListings = false
 		useAlreadyExists = false // untested
+	case "FlashBlade":
+		mightGzip = false        // Never auto gzips objects
+		virtualHostStyle = false // supports vhost but defaults to paths
 	case "IBMCOS":
 		listObjectsV2 = false // untested
 		virtualHostStyle = false
@@ -3527,6 +3572,14 @@ func setQuirks(opt *Options) {
 		urlEncodeListings = false
 		useMultipartEtag = false
 		useAlreadyExists = false
+	case "Mega":
+		listObjectsV2 = true
+		virtualHostStyle = false
+		urlEncodeListings = true
+		useMultipartEtag = false
+		useAlreadyExists = false
+		// Multipart server side copies not supported
+		opt.CopyCutoff = math.MaxInt64
 	case "Minio":
 		virtualHostStyle = false
 	case "Netease":
@@ -3597,6 +3650,13 @@ func setQuirks(opt *Options) {
 		urlEncodeListings = false
 		virtualHostStyle = false
 		useAlreadyExists = false // untested
+	case "Zata":
+		useMultipartEtag = false
+		mightGzip = false
+		useUnsignedPayload = false
+		useAlreadyExists = false
+	case "Exaba":
+		virtualHostStyle = false
 	case "GCS":
 		// Google break request Signature by mutating accept-encoding HTTP header
 		// https://github.com/rclone/rclone/issues/6670
@@ -4425,7 +4485,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 			remote = remote[len(opt.prefix):]
 			if isDirectory {
 				// process directory markers as directories
-				remote = strings.TrimRight(remote, "/")
+				remote, _ = strings.CutSuffix(remote, "/")
 			}
 			if opt.addBucket {
 				remote = bucket.Join(opt.bucket, remote)
@@ -4740,7 +4800,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote = strings.TrimRight(remote, "/")
+	remote, _ = strings.CutSuffix(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
 		dir = ""

backend/seafile/seafile.go

@@ -111,7 +111,8 @@ func init() {
 				encoder.EncodeSlash |
 				encoder.EncodeBackSlash |
 				encoder.EncodeDoubleQuote |
-				encoder.EncodeInvalidUtf8),
+				encoder.EncodeInvalidUtf8 |
+				encoder.EncodeDot),
 		}},
 	})
 }

backend/sftp/sftp.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"io"
 	iofs "io/fs"
+	"net/url"
 	"os"
 	"path"
 	"regexp"
@@ -221,15 +222,45 @@ E.g. the second example above should be rewritten as:
 					Help:  "Windows Command Prompt",
 				},
 			},
+		}, {
+			Name:     "hashes",
+			Help:     `Comma separated list of supported checksum types.`,
+			Default:  fs.CommaSepList{},
+			Advanced: true,
 		}, {
 			Name:     "md5sum_command",
 			Default:  "",
-			Help:     "The command used to read md5 hashes.\n\nLeave blank for autodetect.",
+			Help:     "The command used to read MD5 hashes.\n\nLeave blank for autodetect.",
 			Advanced: true,
 		}, {
 			Name:     "sha1sum_command",
 			Default:  "",
-			Help:     "The command used to read sha1 hashes.\n\nLeave blank for autodetect.",
+			Help:     "The command used to read SHA-1 hashes.\n\nLeave blank for autodetect.",
+			Advanced: true,
+		}, {
+			Name:     "crc32sum_command",
+			Default:  "",
+			Help:     "The command used to read CRC-32 hashes.\n\nLeave blank for autodetect.",
+			Advanced: true,
+		}, {
+			Name:     "sha256sum_command",
+			Default:  "",
+			Help:     "The command used to read SHA-256 hashes.\n\nLeave blank for autodetect.",
+			Advanced: true,
+		}, {
+			Name:     "blake3sum_command",
+			Default:  "",
+			Help:     "The command used to read BLAKE3 hashes.\n\nLeave blank for autodetect.",
+			Advanced: true,
+		}, {
+			Name:     "xxh3sum_command",
+			Default:  "",
+			Help:     "The command used to read XXH3 hashes.\n\nLeave blank for autodetect.",
+			Advanced: true,
+		}, {
+			Name:     "xxh128sum_command",
+			Default:  "",
+			Help:     "The command used to read XXH128 hashes.\n\nLeave blank for autodetect.",
 			Advanced: true,
 		}, {
 			Name:     "skip_links",
@@ -482,6 +513,14 @@ Example:
 	myUser:myPass@localhost:9005
 	`,
 			Advanced: true,
+		}, {
+			Name:    "http_proxy",
+			Default: "",
+			Help: `URL for HTTP CONNECT proxy
+
+Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
+`,
+			Advanced: true,
 		}, {
 			Name:    "copy_is_hardlink",
 			Default: false,
@@ -526,8 +565,14 @@ type Options struct {
 	PathOverride            string          `config:"path_override"`
 	SetModTime              bool            `config:"set_modtime"`
 	ShellType               string          `config:"shell_type"`
+	Hashes                  fs.CommaSepList `config:"hashes"`
 	Md5sumCommand           string          `config:"md5sum_command"`
 	Sha1sumCommand          string          `config:"sha1sum_command"`
+	Crc32sumCommand         string          `config:"crc32sum_command"`
+	Sha256sumCommand        string          `config:"sha256sum_command"`
+	Blake3sumCommand        string          `config:"blake3sum_command"`
+	Xxh3sumCommand          string          `config:"xxh3sum_command"`
+	Xxh128sumCommand        string          `config:"xxh128sum_command"`
 	SkipLinks               bool            `config:"skip_links"`
 	Subsystem               string          `config:"subsystem"`
 	ServerCommand           string          `config:"server_command"`
@@ -545,6 +590,7 @@ type Options struct {
 	HostKeyAlgorithms       fs.SpaceSepList `config:"host_key_algorithms"`
 	SSH                     fs.SpaceSepList `config:"ssh"`
 	SocksProxy              string          `config:"socks_proxy"`
+	HTTPProxy               string          `config:"http_proxy"`
 	CopyIsHardlink          bool            `config:"copy_is_hardlink"`
 }
 
@@ -570,17 +616,23 @@ type Fs struct {
 	savedpswd    string
 	sessions     atomic.Int32 // count in use sessions
 	tokens       *pacer.TokenDispenser
+	proxyURL     *url.URL // address of HTTP proxy read from environment
 }
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
 type Object struct {
-	fs      *Fs
-	remote  string
-	size    int64       // size of the object
-	modTime uint32      // modification time of the object as unix time
-	mode    os.FileMode // mode bits from the file
-	md5sum  *string     // Cached MD5 checksum
-	sha1sum *string     // Cached SHA1 checksum
+	fs        *Fs
+	remote    string
+	size      int64       // size of the object
+	modTime   uint32      // modification time of the object as unix time
+	mode      os.FileMode // mode bits from the file
+	md5sum    *string     // Cached MD5 checksum
+	sha1sum   *string     // Cached SHA-1 checksum
+	crc32sum  *string     // Cached CRC-32 checksum
+	sha256sum *string     // Cached SHA-256 checksum
+	blake3sum *string     // Cached BLAKE3 checksum
+	xxh3sum   *string     // Cached XXH3 checksum
+	xxh128sum *string     // Cached XXH128 checksum
 }
 
 // conn encapsulates an ssh client and corresponding sftp client
@@ -867,11 +919,20 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt.Port = "22"
 	}
 
+	// get proxy URL if set
+	if opt.HTTPProxy != "" {
+		proxyURL, err := url.Parse(opt.HTTPProxy)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse HTTP Proxy URL: %w", err)
+		}
+		f.proxyURL = proxyURL
+	}
+
 	sshConfig := &ssh.ClientConfig{
 		User:            opt.User,
 		Auth:            []ssh.AuthMethod{},
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		Timeout:         f.ci.ConnectTimeout,
+		Timeout:         time.Duration(f.ci.ConnectTimeout),
 		ClientVersion:   "SSH-2.0-" + f.ci.UserAgent,
 	}
 
@@ -1623,14 +1684,112 @@ func (f *Fs) Hashes() hash.Set {
 		return *f.cachedHashes
 	}
 
-	hashSet := hash.NewHashSet()
-	f.cachedHashes = &hashSet
+	hashTypesSupported := hash.NewHashSet()
+	f.cachedHashes = &hashTypesSupported
 
 	if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
-		return hashSet
+		return hashTypesSupported
+	}
+
+	hashTypes := hash.NewHashSet()
+	if len(f.opt.Hashes) > 0 {
+		for _, hashName := range f.opt.Hashes {
+			var hashType hash.Type
+			if err := hashType.Set(hashName); err != nil {
+				fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
+			}
+			hashTypes.Add(hashType)
+		}
+	} else {
+		hashTypes.Add(hash.MD5, hash.SHA1)
+	}
+
+	hashCommands := map[hash.Type]struct {
+		option       *string
+		emptyHash    string
+		hashCommands []struct{ hashFile, hashEmpty string }
+	}{
+		hash.MD5: {
+			&f.opt.Md5sumCommand,
+			"d41d8cd98f00b204e9800998ecf8427e",
+			[]struct{ hashFile, hashEmpty string }{
+				{"md5sum", "md5sum"},
+				{"md5 -r", "md5 -r"},
+				{"rclone md5sum", "rclone md5sum"},
+			},
+		},
+		hash.SHA1: {
+			&f.opt.Sha1sumCommand,
+			"da39a3ee5e6b4b0d3255bfef95601890afd80709",
+			[]struct{ hashFile, hashEmpty string }{
+				{"sha1sum", "sha1sum"},
+				{"sha1 -r", "sha1 -r"},
+				{"rclone sha1sum", "rclone sha1sum"},
+			},
+		},
+		hash.CRC32: {
+			&f.opt.Sha1sumCommand,
+			"00000000",
+			[]struct{ hashFile, hashEmpty string }{
+				{"crc32", "crc32"},
+				{"rclone hashsum crc32", "rclone hashsum crc32"},
+			},
+		},
+		hash.SHA256: {
+			&f.opt.Sha256sumCommand,
+			"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+			[]struct{ hashFile, hashEmpty string }{
+				{"sha256sum", "sha1sum"},
+				{"sha256 -r", "sha1 -r"},
+				{"rclone hashsum sha256", "rclone hashsum sha256"},
+			},
+		},
+		hash.BLAKE3: {
+			&f.opt.Blake3sumCommand,
+			"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
+			[]struct{ hashFile, hashEmpty string }{
+				{"b3sum", "b3sum"},
+				{"rclone hashsum blake3", "rclone hashsum blake3"},
+			},
+		},
+		hash.XXH3: {
+			&f.opt.Xxh3sumCommand,
+			"2d06800538d394c2",
+			[]struct{ hashFile, hashEmpty string }{
+				// The xxhsum tool uses a non-standard prefix "XXH3_" preceding the hash output for the 64-bit variant
+				// of XXH3, to avoid confusion with the older 64-bit algorithm XXH64. This was introduced in version
+				// 0.8.3 released Dec 30, 2024. Older versions only supported the alternative BSD style output format,
+				// otherwise optional with argument --tag. We are currently not expecting these output formats and can
+				// therefore not use the "xxhsum -H3" command or its xxh3sum alias directly.
+				//{"xxh3sum", "xxh3sum"},
+				//{"xxhsum -H3", "xxhsum -H3"},
+				{"rclone hashsum xxh3", "rclone hashsum xxh3"},
+			},
+		},
+		hash.XXH128: {
+			&f.opt.Xxh128sumCommand,
+			"99aa06d3014798d86001c324468d497f",
+			[]struct{ hashFile, hashEmpty string }{
+				{"xxh128sum", "xxh128sum"},
+				{"xxhsum -H2", "xxhsum -H2"},
+				{"rclone hashsum xxh128", "rclone hashsum xxh128"},
+			},
+		},
+	}
+	if f.shellType == "powershell" {
+		for _, hashType := range []hash.Type{hash.MD5, hash.SHA1, hash.SHA256} {
+			if entry, ok := hashCommands[hashType]; ok {
+				entry.hashCommands = append(hashCommands[hashType].hashCommands, struct {
+					hashFile, hashEmpty string
+				}{
+					fmt.Sprintf("&{param($Path);Get-FileHash -Algorithm %v -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower())  ${Path}\"}}", hashType),
+					fmt.Sprintf("Get-FileHash -Algorithm %v -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", hashType),
+				})
+				hashCommands[hashType] = entry
+			}
+		}
 	}
 
-	// look for a hash command which works
 	checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
 		if *hashCommand == hashCommandNotSupported {
 			return false
@@ -1659,55 +1818,25 @@ func (f *Fs) Hashes() hash.Set {
 	}
 
 	changed := false
-	md5Commands := []struct {
-		hashFile, hashEmpty string
-	}{
-		{"md5sum", "md5sum"},
-		{"md5 -r", "md5 -r"},
-		{"rclone md5sum", "rclone md5sum"},
+	for _, hashType := range hashTypes.Array() {
+		if entry, ok := hashCommands[hashType]; ok {
+			if works := checkHash(hashType, entry.hashCommands, entry.emptyHash, entry.option, &changed); works {
+				hashTypesSupported.Add(hashType)
+			}
+		}
 	}
-	sha1Commands := []struct {
-		hashFile, hashEmpty string
-	}{
-		{"sha1sum", "sha1sum"},
-		{"sha1 -r", "sha1 -r"},
-		{"rclone sha1sum", "rclone sha1sum"},
-	}
-	if f.shellType == "powershell" {
-		md5Commands = append(md5Commands, struct {
-			hashFile, hashEmpty string
-		}{
-			"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower())  ${Path}\"}}",
-			"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
-		})
-
-		sha1Commands = append(sha1Commands, struct {
-			hashFile, hashEmpty string
-		}{
-			"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower())  ${Path}\"}}",
-			"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
-		})
-	}
-
-	md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
-	sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
 
 	if changed {
 		// Save permanently in config to avoid the extra work next time
-		fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand)
-		f.m.Set("md5sum_command", f.opt.Md5sumCommand)
-		fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand)
-		f.m.Set("sha1sum_command", f.opt.Sha1sumCommand)
+		for _, hashType := range hashTypes.Array() {
+			if entry, ok := hashCommands[hashType]; ok {
+				fs.Debugf(f, "Setting hash command for %v to %q (set %vsum_command to override)", hashType, *entry.option, hashType)
+				f.m.Set(fmt.Sprintf("%vsum_command", hashType), *entry.option)
+			}
+		}
 	}
 
-	if sha1Works {
-		hashSet.Add(hash.SHA1)
-	}
-	if md5Works {
-		hashSet.Add(hash.MD5)
-	}
-
-	return hashSet
+	return hashTypesSupported
 }
 
 // About gets usage stats
@@ -1842,17 +1971,43 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	_ = o.fs.Hashes()
 
 	var hashCmd string
-	if r == hash.MD5 {
+	switch r {
+	case hash.MD5:
 		if o.md5sum != nil {
 			return *o.md5sum, nil
 		}
 		hashCmd = o.fs.opt.Md5sumCommand
-	} else if r == hash.SHA1 {
+	case hash.SHA1:
 		if o.sha1sum != nil {
 			return *o.sha1sum, nil
 		}
 		hashCmd = o.fs.opt.Sha1sumCommand
-	} else {
+	case hash.CRC32:
+		if o.crc32sum != nil {
+			return *o.crc32sum, nil
+		}
+		hashCmd = o.fs.opt.Crc32sumCommand
+	case hash.SHA256:
+		if o.sha256sum != nil {
+			return *o.sha256sum, nil
+		}
+		hashCmd = o.fs.opt.Sha256sumCommand
+	case hash.BLAKE3:
+		if o.blake3sum != nil {
+			return *o.blake3sum, nil
+		}
+		hashCmd = o.fs.opt.Blake3sumCommand
+	case hash.XXH3:
+		if o.xxh3sum != nil {
+			return *o.xxh3sum, nil
+		}
+		hashCmd = o.fs.opt.Xxh3sumCommand
+	case hash.XXH128:
+		if o.xxh128sum != nil {
+			return *o.xxh128sum, nil
+		}
+		hashCmd = o.fs.opt.Xxh128sumCommand
+	default:
 		return "", hash.ErrUnsupported
 	}
 	if hashCmd == "" || hashCmd == hashCommandNotSupported {
@@ -1869,10 +2024,21 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	}
 	hashString := parseHash(outBytes)
 	fs.Debugf(o, "Parsed hash: %s", hashString)
-	if r == hash.MD5 {
+	switch r {
+	case hash.MD5:
 		o.md5sum = &hashString
-	} else if r == hash.SHA1 {
+	case hash.SHA1:
 		o.sha1sum = &hashString
+	case hash.CRC32:
+		o.crc32sum = &hashString
+	case hash.SHA256:
+		o.sha256sum = &hashString
+	case hash.BLAKE3:
+		o.blake3sum = &hashString
+	case hash.XXH3:
+		o.xxh3sum = &hashString
+	case hash.XXH128:
+		o.xxh128sum = &hashString
 	}
 	return hashString, nil
 }
@@ -1937,7 +2103,7 @@ func (f *Fs) remoteShellPath(remote string) string {
 }
 
 // Converts a byte array from the SSH session returned by
-// an invocation of md5sum/sha1sum to a hash string
+// an invocation of hash command to a hash string
 // as expected by the rest of this application
 func parseHash(bytes []byte) string {
 	// For strings with backslash *sum writes a leading \
@@ -2166,6 +2332,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Clear the hash cache since we are about to update the object
 	o.md5sum = nil
 	o.sha1sum = nil
+	o.crc32sum = nil
+	o.sha256sum = nil
+	o.blake3sum = nil
+	o.xxh3sum = nil
+	o.xxh128sum = nil
 	c, err := o.fs.getSftpConnection(ctx)
 	if err != nil {
 		return fmt.Errorf("Update: %w", err)

backend/sftp/ssh_internal.go

@@ -31,6 +31,8 @@ func (f *Fs) newSSHClientInternal(ctx context.Context, network, addr string, ssh
 	)
 	if f.opt.SocksProxy != "" {
 		conn, err = proxy.SOCKS5Dial(network, addr, f.opt.SocksProxy, baseDialer)
+	} else if f.proxyURL != nil {
+		conn, err = proxy.HTTPConnectDial(network, addr, f.proxyURL, baseDialer)
 	} else {
 		conn, err = baseDialer.Dial(network, addr)
 	}

backend/smb/connpool.go

@@ -38,7 +38,7 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 
 	d := &smb2.Dialer{}
 	if f.opt.UseKerberos {
-		cl, err := getKerberosClient()
+		cl, err := createKerberosClient(f.opt.KerberosCCache)
 		if err != nil {
 			return nil, err
 		}

backend/smb/kerberos.go

@@ -14,65 +14,81 @@ import (
 )
 
 var (
-	kerberosClient *client.Client
-	kerberosErr    error
-	kerberosOnce   sync.Once
+	kerberosClient sync.Map // map[string]*client.Client
+	kerberosErr    sync.Map // map[string]error
 )
 
-// getKerberosClient returns a Kerberos client that can be used to authenticate.
-func getKerberosClient() (*client.Client, error) {
-	if kerberosClient == nil || kerberosErr == nil {
-		kerberosOnce.Do(func() {
-			kerberosClient, kerberosErr = createKerberosClient()
-		})
+func resolveCcachePath(ccachePath string) (string, error) {
+	if ccachePath == "" {
+		ccachePath = os.Getenv("KRB5CCNAME")
 	}
 
-	return kerberosClient, kerberosErr
-}
-
-// createKerberosClient creates a new Kerberos client.
-func createKerberosClient() (*client.Client, error) {
-	cfgPath := os.Getenv("KRB5_CONFIG")
-	if cfgPath == "" {
-		cfgPath = "/etc/krb5.conf"
-	}
-
-	cfg, err := config.Load(cfgPath)
-	if err != nil {
-		return nil, err
-	}
-
-	// Determine the ccache location from the environment, falling back to the
-	// default location.
-	ccachePath := os.Getenv("KRB5CCNAME")
 	switch {
 	case strings.Contains(ccachePath, ":"):
 		parts := strings.SplitN(ccachePath, ":", 2)
-		switch parts[0] {
+		prefix, path := parts[0], parts[1]
+		switch prefix {
 		case "FILE":
-			ccachePath = parts[1]
+			return path, nil
 		case "DIR":
-			primary, err := os.ReadFile(filepath.Join(parts[1], "primary"))
+			primary, err := os.ReadFile(filepath.Join(path, "primary"))
 			if err != nil {
-				return nil, err
+				return "", err
 			}
-			ccachePath = filepath.Join(parts[1], strings.TrimSpace(string(primary)))
+			return filepath.Join(path, strings.TrimSpace(string(primary))), nil
 		default:
-			return nil, fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
+			return "", fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
 		}
 	case ccachePath == "":
 		u, err := user.Current()
 		if err != nil {
-			return nil, err
+			return "", err
 		}
-
-		ccachePath = "/tmp/krb5cc_" + u.Uid
+		return "/tmp/krb5cc_" + u.Uid, nil
+	default:
+		return ccachePath, nil
 	}
+}
 
-	ccache, err := credentials.LoadCCache(ccachePath)
+func loadKerberosConfig() (*config.Config, error) {
+	cfgPath := os.Getenv("KRB5_CONFIG")
+	if cfgPath == "" {
+		cfgPath = "/etc/krb5.conf"
+	}
+	return config.Load(cfgPath)
+}
+
+// createKerberosClient creates a new Kerberos client.
+func createKerberosClient(ccachePath string) (*client.Client, error) {
+	ccachePath, err := resolveCcachePath(ccachePath)
 	if err != nil {
 		return nil, err
 	}
 
-	return client.NewFromCCache(ccache, cfg)
+	// check if we already have a client or an error for this ccache path
+	if errVal, ok := kerberosErr.Load(ccachePath); ok {
+		return nil, errVal.(error)
+	}
+	if clientVal, ok := kerberosClient.Load(ccachePath); ok {
+		return clientVal.(*client.Client), nil
+	}
+
+	// create a new client if not found in the map
+	cfg, err := loadKerberosConfig()
+	if err != nil {
+		kerberosErr.Store(ccachePath, err)
+		return nil, err
+	}
+	ccache, err := credentials.LoadCCache(ccachePath)
+	if err != nil {
+		kerberosErr.Store(ccachePath, err)
+		return nil, err
+	}
+	cl, err := client.NewFromCCache(ccache, cfg)
+	if err != nil {
+		kerberosErr.Store(ccachePath, err)
+		return nil, err
+	}
+	kerberosClient.Store(ccachePath, cl)
+	return cl, nil
 }

backend/smb/kerberos_test.go

@@ -0,0 +1,79 @@
+package smb
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestResolveCcachePath(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	// Setup: files for FILE and DIR modes
+	fileCcache := filepath.Join(tmpDir, "file_ccache")
+	err := os.WriteFile(fileCcache, []byte{}, 0600)
+	assert.NoError(t, err)
+
+	dirCcache := filepath.Join(tmpDir, "dir_ccache")
+	err = os.Mkdir(dirCcache, 0755)
+	assert.NoError(t, err)
+	err = os.WriteFile(filepath.Join(dirCcache, "primary"), []byte("ticket"), 0600)
+	assert.NoError(t, err)
+	dirCcacheTicket := filepath.Join(dirCcache, "ticket")
+	err = os.WriteFile(dirCcacheTicket, []byte{}, 0600)
+	assert.NoError(t, err)
+
+	tests := []struct {
+		name          string
+		ccachePath    string
+		envKRB5CCNAME string
+		expected      string
+		expectError   bool
+	}{
+		{
+			name:          "FILE: prefix from env",
+			ccachePath:    "",
+			envKRB5CCNAME: "FILE:" + fileCcache,
+			expected:      fileCcache,
+		},
+		{
+			name:          "DIR: prefix from env",
+			ccachePath:    "",
+			envKRB5CCNAME: "DIR:" + dirCcache,
+			expected:      dirCcacheTicket,
+		},
+		{
+			name:          "Unsupported prefix",
+			ccachePath:    "",
+			envKRB5CCNAME: "MEMORY:/bad/path",
+			expectError:   true,
+		},
+		{
+			name:       "Direct file path (no prefix)",
+			ccachePath: "/tmp/myccache",
+			expected:   "/tmp/myccache",
+		},
+		{
+			name:          "Default to /tmp/krb5cc_<uid>",
+			ccachePath:    "",
+			envKRB5CCNAME: "",
+			expected:      "/tmp/krb5cc_",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Setenv("KRB5CCNAME", tt.envKRB5CCNAME)
+			result, err := resolveCcachePath(tt.ccachePath)
+
+			if tt.expectError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Contains(t, result, tt.expected)
+			}
+		})
+	}
+}

backend/smb/smb.go

@@ -107,6 +107,20 @@ Set to 0 to keep connections indefinitely.
 			Help:     "Whether the server is configured to be case-insensitive.\n\nAlways true on Windows shares.",
 			Default:  true,
 			Advanced: true,
+		}, {
+			Name: "kerberos_ccache",
+			Help: `Path to the Kerberos credential cache (krb5cc).
+
+Overrides the default KRB5CCNAME environment variable and allows this
+instance of the SMB backend to use a different Kerberos cache file.
+This is useful when mounting multiple SMB with different credentials
+or running in multi-user environments.
+
+Supported formats:
+  - FILE:/path/to/ccache   – Use the specified file.
+  - DIR:/path/to/ccachedir – Use the primary file inside the specified directory.
+  - /path/to/ccache        – Interpreted as a file path.`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -137,6 +151,7 @@ type Options struct {
 	Domain          string      `config:"domain"`
 	SPN             string      `config:"spn"`
 	UseKerberos     bool        `config:"use_kerberos"`
+	KerberosCCache  string      `config:"kerberos_ccache"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`

backend/smb/smb_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/smb"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -18,6 +19,9 @@ func TestIntegration(t *testing.T) {
 }
 
 func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
 	krb5Dir := t.TempDir()
 	t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
 	t.Setenv("KRB5CCNAME", filepath.Join(krb5Dir, "ccache"))
@@ -26,3 +30,24 @@ func TestIntegration2(t *testing.T) {
 		NilObject:  (*smb.Object)(nil),
 	})
 }
+
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+
+	krb5Dir := t.TempDir()
+	t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
+	ccache := filepath.Join(krb5Dir, "ccache")
+	t.Setenv("RCLONE_TEST_CUSTOM_CCACHE_LOCATION", ccache)
+
+	name := "TestSMBKerberosCcache"
+
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":rclone",
+		NilObject:  (*smb.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "kerberos_ccache", Value: ccache},
+		},
+	})
+}

backend/swift/swift.go

@@ -491,8 +491,8 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
 		ApplicationCredentialName:   opt.ApplicationCredentialName,
 		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
 		EndpointType:                swift.EndpointType(opt.EndpointType),
-		ConnectTimeout:              10 * ci.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:                     10 * ci.Timeout,        // Use the timeouts in the transport
+		ConnectTimeout:              time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
+		Timeout:                     time.Duration(10 * ci.Timeout),        // Use the timeouts in the transport
 		Transport:                   fshttp.NewTransport(ctx),
 		FetchUntilEmptyPage:         opt.FetchUntilEmptyPage,
 		PartialPageFetchThreshold:   opt.PartialPageFetchThreshold,

backend/webdav/tus-errors.go

@@ -0,0 +1,40 @@
+package webdav
+
+import (
+	"errors"
+	"fmt"
+)
+
+var (
+	// ErrChunkSize is returned when the chunk size is zero
+	ErrChunkSize = errors.New("tus chunk size must be greater than zero")
+	// ErrNilLogger is returned when the logger is nil
+	ErrNilLogger = errors.New("tus logger can't be nil")
+	// ErrNilStore is returned when the store is nil
+	ErrNilStore = errors.New("tus store can't be nil if resume is enable")
+	// ErrNilUpload is returned when the upload is nil
+	ErrNilUpload = errors.New("tus upload can't be nil")
+	// ErrLargeUpload is returned when the upload body is to large
+	ErrLargeUpload = errors.New("tus upload body is to large")
+	// ErrVersionMismatch is returned when the tus protocol version is mismatching
+	ErrVersionMismatch = errors.New("tus protocol version mismatch")
+	// ErrOffsetMismatch is returned when the tus upload offset is mismatching
+	ErrOffsetMismatch = errors.New("tus upload offset mismatch")
+	// ErrUploadNotFound is returned when the tus upload is not found
+	ErrUploadNotFound = errors.New("tus upload not found")
+	// ErrResumeNotEnabled is returned when the tus resuming is not enabled
+	ErrResumeNotEnabled = errors.New("tus resuming not enabled")
+	// ErrFingerprintNotSet is returned when the tus fingerprint is not set
+	ErrFingerprintNotSet = errors.New("tus fingerprint not set")
+)
+
+// ClientError represents an error state of a client
+type ClientError struct {
+	Code int
+	Body []byte
+}
+
+// Error returns an error string containing the client error code
+func (c ClientError) Error() string {
+	return fmt.Sprintf("unexpected status code: %d", c.Code)
+}

backend/webdav/tus-upload.go

@@ -0,0 +1,88 @@
+package webdav
+
+import (
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"strings"
+)
+
+// Metadata is a typedef for a string to string map to hold metadata
+type Metadata map[string]string
+
+// Upload is a struct containing the file status during upload
+type Upload struct {
+	stream io.ReadSeeker
+	size   int64
+	offset int64
+
+	Fingerprint string
+	Metadata    Metadata
+}
+
+// Updates the Upload information based on offset.
+func (u *Upload) updateProgress(offset int64) {
+	u.offset = offset
+}
+
+// Finished returns whether this upload is finished or not.
+func (u *Upload) Finished() bool {
+	return u.offset >= u.size
+}
+
+// Progress returns the progress in a percentage.
+func (u *Upload) Progress() int64 {
+	return (u.offset * 100) / u.size
+}
+
+// Offset returns the current upload offset.
+func (u *Upload) Offset() int64 {
+	return u.offset
+}
+
+// Size returns the size of the upload body.
+func (u *Upload) Size() int64 {
+	return u.size
+}
+
+// EncodedMetadata encodes the upload metadata.
+func (u *Upload) EncodedMetadata() string {
+	var encoded []string
+
+	for k, v := range u.Metadata {
+		encoded = append(encoded, fmt.Sprintf("%s %s", k, b64encode(v)))
+	}
+
+	return strings.Join(encoded, ",")
+}
+
+func b64encode(s string) string {
+	return base64.StdEncoding.EncodeToString([]byte(s))
+}
+
+// NewUpload creates a new upload from an io.Reader.
+func NewUpload(reader io.Reader, size int64, metadata Metadata, fingerprint string) *Upload {
+	stream, ok := reader.(io.ReadSeeker)
+
+	if !ok {
+		buf := new(bytes.Buffer)
+		_, err := buf.ReadFrom(reader)
+		if err != nil {
+			return nil
+		}
+		stream = bytes.NewReader(buf.Bytes())
+	}
+
+	if metadata == nil {
+		metadata = make(Metadata)
+	}
+
+	return &Upload{
+		stream: stream,
+		size:   size,
+
+		Fingerprint: fingerprint,
+		Metadata:    metadata,
+	}
+}

backend/webdav/tus-uploader.go

@@ -0,0 +1,191 @@
+package webdav
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// Uploader holds all information about a currently running upload
+type Uploader struct {
+	fs                  *Fs
+	url                 string
+	upload              *Upload
+	offset              int64
+	aborted             bool
+	uploadSubs          []chan Upload
+	notifyChan          chan bool
+	overridePatchMethod bool
+}
+
+// NotifyUploadProgress subscribes to progress updates.
+func (u *Uploader) NotifyUploadProgress(c chan Upload) {
+	u.uploadSubs = append(u.uploadSubs, c)
+}
+
+func (f *Fs) shouldRetryChunk(ctx context.Context, resp *http.Response, err error, newOff *int64) (bool, error) {
+	if resp == nil {
+		return true, err
+	}
+
+	switch resp.StatusCode {
+	case 204:
+		if off, err := strconv.ParseInt(resp.Header.Get("Upload-Offset"), 10, 64); err == nil {
+			*newOff = off
+			return false, nil
+		}
+		return false, err
+
+	case 409:
+		return false, ErrOffsetMismatch
+	case 412:
+		return false, ErrVersionMismatch
+	case 413:
+		return false, ErrLargeUpload
+	}
+
+	return f.shouldRetry(ctx, resp, err)
+}
+
+func (u *Uploader) uploadChunk(ctx context.Context, body io.Reader, size int64, offset int64, options ...fs.OpenOption) (int64, error) {
+	var method string
+
+	if !u.overridePatchMethod {
+		method = "PATCH"
+	} else {
+		method = "POST"
+	}
+
+	extraHeaders := map[string]string{} // FIXME: Use extraHeaders(ctx, src) from Object maybe?
+	extraHeaders["Upload-Offset"] = strconv.FormatInt(offset, 10)
+	extraHeaders["Tus-Resumable"] = "1.0.0"
+	extraHeaders["filetype"] = u.upload.Metadata["filetype"]
+	if u.overridePatchMethod {
+		extraHeaders["X-HTTP-Method-Override"] = "PATCH"
+	}
+
+	url, err := url.Parse(u.url)
+	if err != nil {
+		return 0, fmt.Errorf("upload Chunk failed, could not parse url")
+	}
+
+	// FIXME: Use GetBody func as in chunking.go
+	opts := rest.Opts{
+		Method:        method,
+		Path:          url.Path,
+		NoResponse:    true,
+		RootURL:       fmt.Sprintf("%s://%s", url.Scheme, url.Host),
+		ContentLength: &size,
+		Body:          body,
+		ContentType:   "application/offset+octet-stream",
+		ExtraHeaders:  extraHeaders,
+		Options:       options,
+	}
+
+	var newOffset int64
+
+	err = u.fs.pacer.CallNoRetry(func() (bool, error) {
+		res, err := u.fs.srv.Call(ctx, &opts)
+		return u.fs.shouldRetryChunk(ctx, res, err, &newOffset)
+	})
+	if err != nil {
+		return 0, fmt.Errorf("uploadChunk failed: %w", err)
+		// FIXME What do we do here? Remove the entire upload?
+		// See https://github.com/tus/tusd/issues/176
+	}
+
+	return newOffset, nil
+}
+
+// Upload uploads the entire body to the server.
+func (u *Uploader) Upload(ctx context.Context, options ...fs.OpenOption) error {
+	cnt := 1
+
+	fs.Debug(u.fs, "Uploaded starts")
+	for u.offset < u.upload.size && !u.aborted {
+		err := u.UploadChunk(ctx, cnt, options...)
+		cnt++
+		if err != nil {
+			return err
+		}
+	}
+	fs.Debug(u.fs, "-- Uploaded finished")
+
+	return nil
+}
+
+// UploadChunk uploads a single chunk.
+func (u *Uploader) UploadChunk(ctx context.Context, cnt int, options ...fs.OpenOption) error {
+	chunkSize := u.fs.opt.ChunkSize
+	data := make([]byte, chunkSize)
+
+	_, err := u.upload.stream.Seek(u.offset, 0)
+
+	if err != nil {
+		fs.Errorf(u.fs, "Chunk %d: Error seek in stream failed: %v", cnt, err)
+		return err
+	}
+
+	size, err := u.upload.stream.Read(data)
+
+	if err != nil {
+		fs.Errorf(u.fs, "Chunk %d: Error: Can not read from data strem: %v", cnt, err)
+		return err
+	}
+
+	body := bytes.NewBuffer(data[:size])
+
+	newOffset, err := u.uploadChunk(ctx, body, int64(size), u.offset, options...)
+
+	if err == nil {
+		fs.Debugf(u.fs, "Uploaded chunk no %d ok, range %d -> %d", cnt, u.offset, newOffset)
+	} else {
+		fs.Errorf(u.fs, "Uploaded chunk no %d failed: %v", cnt, err)
+
+		return err
+	}
+
+	u.offset = newOffset
+
+	u.upload.updateProgress(u.offset)
+
+	u.notifyChan <- true
+
+	return nil
+}
+
+// Waits for a signal to broadcast to all subscribers
+func (u *Uploader) broadcastProgress() {
+	for range u.notifyChan {
+		for _, c := range u.uploadSubs {
+			c <- *u.upload
+		}
+	}
+}
+
+// NewUploader creates a new Uploader.
+func NewUploader(f *Fs, url string, upload *Upload, offset int64) *Uploader {
+	notifyChan := make(chan bool)
+
+	uploader := &Uploader{
+		f,
+		url,
+		upload,
+		offset,
+		false,
+		nil,
+		notifyChan,
+		false,
+	}
+
+	go uploader.broadcastProgress()
+
+	return uploader
+}

backend/webdav/tus.go

@@ -0,0 +1,108 @@
+package webdav
+
+/*
+   Chunked upload based on the tus protocol for ownCloud Infinite Scale
+   See https://tus.io/protocols/resumable-upload
+*/
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"path/filepath"
+	"strconv"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+func (o *Object) updateViaTus(ctx context.Context, in io.Reader, contentType string, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+
+	fn := filepath.Base(src.Remote())
+	metadata := map[string]string{
+		"filename": fn,
+		"mtime":    strconv.FormatInt(src.ModTime(ctx).Unix(), 10),
+		"filetype": contentType,
+	}
+
+	// Fingerprint is used to identify the upload when resuming. That is not yet implemented
+	fingerprint := ""
+
+	// create an upload from a file.
+	upload := NewUpload(in, src.Size(), metadata, fingerprint)
+
+	// create the uploader.
+	uploader, err := o.CreateUploader(ctx, upload, options...)
+	if err == nil {
+		// start the uploading process.
+		err = uploader.Upload(ctx, options...)
+	}
+
+	return err
+}
+
+func (f *Fs) getTusLocationOrRetry(ctx context.Context, resp *http.Response, err error) (bool, string, error) {
+
+	switch resp.StatusCode {
+	case 201:
+		location := resp.Header.Get("Location")
+		return false, location, nil
+	case 412:
+		return false, "", ErrVersionMismatch
+	case 413:
+		return false, "", ErrLargeUpload
+	}
+
+	retry, err := f.shouldRetry(ctx, resp, err)
+	return retry, "", err
+}
+
+// CreateUploader creates a new upload to the server.
+func (o *Object) CreateUploader(ctx context.Context, u *Upload, options ...fs.OpenOption) (*Uploader, error) {
+	if u == nil {
+		return nil, ErrNilUpload
+	}
+
+	// if c.Config.Resume && len(u.Fingerprint) == 0 {
+	//		return nil, ErrFingerprintNotSet
+	//	}
+
+	l := int64(0)
+	p := o.filePath()
+	// cut the filename off
+	dir, _ := filepath.Split(p)
+	if dir == "" {
+		dir = "/"
+	}
+
+	opts := rest.Opts{
+		Method:        "POST",
+		Path:          dir,
+		NoResponse:    true,
+		RootURL:       o.fs.endpointURL,
+		ContentLength: &l,
+		ExtraHeaders:  o.extraHeaders(ctx, o),
+		Options:       options,
+	}
+	opts.ExtraHeaders["Upload-Length"] = strconv.FormatInt(u.size, 10)
+	opts.ExtraHeaders["Upload-Metadata"] = u.EncodedMetadata()
+	opts.ExtraHeaders["Tus-Resumable"] = "1.0.0"
+	// opts.ExtraHeaders["mtime"] = strconv.FormatInt(src.ModTime(ctx).Unix(), 10)
+
+	var tusLocation string
+	// rclone http call
+	err := o.fs.pacer.CallNoRetry(func() (bool, error) {
+		var retry bool
+		res, err := o.fs.srv.Call(ctx, &opts)
+		retry, tusLocation, err = o.fs.getTusLocationOrRetry(ctx, res, err)
+		return retry, err
+	})
+	if err != nil {
+		return nil, fmt.Errorf("making upload directory failed: %w", err)
+	}
+
+	uploader := NewUploader(o.fs, tusLocation, u, 0)
+
+	return uploader, nil
+}

backend/webdav/webdav.go

@@ -84,7 +84,10 @@ func init() {
 				Help:  "Nextcloud",
 			}, {
 				Value: "owncloud",
-				Help:  "Owncloud",
+				Help:  "Owncloud 10 PHP based WebDAV server",
+			}, {
+				Value: "infinitescale",
+				Help:  "ownCloud Infinite Scale",
 			}, {
 				Value: "sharepoint",
 				Help:  "Sharepoint Online, authenticated by Microsoft account",
@@ -212,6 +215,7 @@ type Fs struct {
 	pacer              *fs.Pacer     // pacer for API calls
 	precision          time.Duration // mod time precision
 	canStream          bool          // set if can stream
+	canTus             bool          // supports the TUS upload protocol
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	propsetMtime       bool          // set if can use propset
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
@@ -632,6 +636,15 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		f.propsetMtime = true
 		f.hasOCMD5 = true
 		f.hasOCSHA1 = true
+	case "infinitescale":
+		f.precision = time.Second
+		f.useOCMtime = true
+		f.propsetMtime = true
+		f.hasOCMD5 = false
+		f.hasOCSHA1 = true
+		f.canChunk = false
+		f.canTus = true
+		f.opt.ChunkSize = 10 * fs.Mebi
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
@@ -1329,7 +1342,7 @@ func (o *Object) Size() int64 {
 	ctx := context.TODO()
 	err := o.readMetaData(ctx)
 	if err != nil {
-		fs.Logf(o, "Failed to read metadata: %v", err)
+		fs.Infof(o, "Failed to read metadata: %v", err)
 		return 0
 	}
 	return o.size
@@ -1373,7 +1386,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 func (o *Object) ModTime(ctx context.Context) time.Time {
 	err := o.readMetaData(ctx)
 	if err != nil {
-		fs.Logf(o, "Failed to read metadata: %v", err)
+		fs.Infof(o, "Failed to read metadata: %v", err)
 		return time.Now()
 	}
 	return o.modTime
@@ -1499,9 +1512,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return fmt.Errorf("Update mkParentDir failed: %w", err)
 	}
 
-	if o.shouldUseChunkedUpload(src) {
-		fs.Debugf(src, "Update will use the chunked upload strategy")
-		err = o.updateChunked(ctx, in, src, options...)
+	if o.fs.canTus { // supports the tus upload protocol, ie. InfiniteScale
+		fs.Debugf(src, "Update will use the tus protocol to upload")
+		contentType := fs.MimeType(ctx, src)
+		err = o.updateViaTus(ctx, in, contentType, src, options...)
+		if err != nil {
+			fs.Debug(src, "tus update failed.")
+			return fmt.Errorf("tus update failed: %w", err)
+		}
+	} else if o.shouldUseChunkedUpload(src) {
+		if o.fs.opt.Vendor == "nextcloud" {
+			fs.Debugf(src, "Update will use the chunked upload strategy")
+			err = o.updateChunked(ctx, in, src, options...)
+		} else {
+			fs.Debug(src, "Chunking - unknown vendor")
+		}
 		if err != nil {
 			return err
 		}
@@ -1513,10 +1538,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// TODO: define getBody() to enable low-level HTTP/2 retries
 		err = o.updateSimple(ctx, in, nil, filePath, src.Size(), contentType, extraHeaders, o.fs.endpointURL, options...)
 		if err != nil {
-			return err
+			return fmt.Errorf("unchunked simple update failed: %w", err)
 		}
 	}
-
 	// read metadata from remote
 	o.hasMetaData = false
 	return o.readMetaData(ctx)

bin/.ignore-emails

@@ -12,4 +12,5 @@
 <seb•ɑƬ•chezwam•ɖɵʈ•org>
 <allllaboutyou@gmail.com>
 <psycho@feltzv.fr>
-<afw5059@gmail.com>
+<afw5059@gmail.com>
+<piyushgarg80>

bin/check_autogenerated_edits.py

@@ -77,6 +77,10 @@ def check_file(file):
     viol = False
     new_lines = get_file_content("HEAD", file)
     old_lines = get_file_content("HEAD~1", file)
+
+    # If old file did not exist or was empty then don't check
+    if not old_lines:
+        return
     
     # Entire autogenerated file check.
     if any("autogenerated - DO NOT EDIT" in l for l in new_lines[:10]):

bin/make_manual.py

@@ -41,8 +41,10 @@ docs = [
     "crypt.md",
     "compress.md",
     "combine.md",
+    "doi.md",
     "dropbox.md",
     "filefabric.md",
+    "filelu.md",
     "filescom.md",
     "ftp.md",
     "gofile.md",

cmd/all/all.go

@@ -15,6 +15,7 @@ import (
 	_ "github.com/rclone/rclone/cmd/cleanup"
 	_ "github.com/rclone/rclone/cmd/cmount"
 	_ "github.com/rclone/rclone/cmd/config"
+	_ "github.com/rclone/rclone/cmd/convmv"
 	_ "github.com/rclone/rclone/cmd/copy"
 	_ "github.com/rclone/rclone/cmd/copyto"
 	_ "github.com/rclone/rclone/cmd/copyurl"

cmd/bisync/bilib/names.go

@@ -5,8 +5,6 @@ import (
 	"os"
 	"sort"
 	"strconv"
-	"strings"
-	"time"
 )
 
 // Names comprises a set of file names
@@ -85,81 +83,3 @@ func (am AliasMap) Alias(name1 string) string {
 	}
 	return name1
 }
-
-// ParseGlobs determines whether a string contains {brackets}
-// and returns the substring (including both brackets) for replacing
-// substring is first opening bracket to last closing bracket --
-// good for {{this}} but not {this}{this}
-func ParseGlobs(s string) (hasGlobs bool, substring string) {
-	open := strings.Index(s, "{")
-	close := strings.LastIndex(s, "}")
-	if open >= 0 && close > open {
-		return true, s[open : close+1]
-	}
-	return false, ""
-}
-
-// TrimBrackets converts {{this}} to this
-func TrimBrackets(s string) string {
-	return strings.Trim(s, "{}")
-}
-
-// TimeFormat converts a user-supplied string to a Go time constant, if possible
-func TimeFormat(timeFormat string) string {
-	switch timeFormat {
-	case "Layout":
-		timeFormat = time.Layout
-	case "ANSIC":
-		timeFormat = time.ANSIC
-	case "UnixDate":
-		timeFormat = time.UnixDate
-	case "RubyDate":
-		timeFormat = time.RubyDate
-	case "RFC822":
-		timeFormat = time.RFC822
-	case "RFC822Z":
-		timeFormat = time.RFC822Z
-	case "RFC850":
-		timeFormat = time.RFC850
-	case "RFC1123":
-		timeFormat = time.RFC1123
-	case "RFC1123Z":
-		timeFormat = time.RFC1123Z
-	case "RFC3339":
-		timeFormat = time.RFC3339
-	case "RFC3339Nano":
-		timeFormat = time.RFC3339Nano
-	case "Kitchen":
-		timeFormat = time.Kitchen
-	case "Stamp":
-		timeFormat = time.Stamp
-	case "StampMilli":
-		timeFormat = time.StampMilli
-	case "StampMicro":
-		timeFormat = time.StampMicro
-	case "StampNano":
-		timeFormat = time.StampNano
-	case "DateTime":
-		// timeFormat = time.DateTime // missing in go1.19
-		timeFormat = "2006-01-02 15:04:05"
-	case "DateOnly":
-		// timeFormat = time.DateOnly // missing in go1.19
-		timeFormat = "2006-01-02"
-	case "TimeOnly":
-		// timeFormat = time.TimeOnly // missing in go1.19
-		timeFormat = "15:04:05"
-	case "MacFriendlyTime", "macfriendlytime", "mac":
-		timeFormat = "2006-01-02 0304PM" // not actually a Go constant -- but useful as macOS filenames can't have colons
-	}
-	return timeFormat
-}
-
-// AppyTimeGlobs converts "myfile-{DateOnly}.txt" to "myfile-2006-01-02.txt"
-func AppyTimeGlobs(s string, t time.Time) string {
-	hasGlobs, substring := ParseGlobs(s)
-	if !hasGlobs {
-		return s
-	}
-	timeString := t.Local().Format(TimeFormat(TrimBrackets(substring)))
-	return strings.ReplaceAll(s, substring, timeString)
-}

cmd/bisync/bilib/output.go

@@ -3,20 +3,22 @@ package bilib
 
 import (
 	"bytes"
-	"log"
+	"log/slog"
 
-	"github.com/sirupsen/logrus"
+	"github.com/rclone/rclone/fs/log"
 )
 
-// CaptureOutput runs a function capturing its output.
+// CaptureOutput runs a function capturing its output at log level INFO.
 func CaptureOutput(fun func()) []byte {
-	logSave := log.Writer()
-	logrusSave := logrus.StandardLogger().Out
 	buf := &bytes.Buffer{}
-	log.SetOutput(buf)
-	logrus.SetOutput(buf)
+	oldLevel := log.Handler.SetLevel(slog.LevelInfo)
+	log.Handler.SetOutput(func(level slog.Level, text string) {
+		buf.WriteString(text)
+	})
+	defer func() {
+		log.Handler.ResetOutput()
+		log.Handler.SetLevel(oldLevel)
+	}()
 	fun()
-	log.SetOutput(logSave)
-	logrus.SetOutput(logrusSave)
 	return buf.Bytes()
 }

cmd/bisync/bisync_test.go

@@ -278,7 +278,7 @@ func testBisync(t *testing.T, path1, path2 string) {
 	}
 	bisync.Colors = true
 	time.Local = bisync.TZ
-	ci.FsCacheExpireDuration = 5 * time.Hour
+	ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour)
 
 	baseDir, err := os.Getwd()
 	require.NoError(t, err, "get current directory")

cmd/bisync/cmd.go

@@ -55,7 +55,7 @@ type Options struct {
 	Compare               CompareOpt
 	CompareFlag           string
 	DebugName             string
-	MaxLock               time.Duration
+	MaxLock               fs.Duration
 	ConflictResolve       Prefer
 	ConflictLoser         ConflictLoserAction
 	ConflictSuffixFlag    string
@@ -146,7 +146,7 @@ func init() {
 	flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
 	flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")
 	flags.BoolVarP(cmdFlags, &Opt.Compare.DownloadHash, "download-hash", "", Opt.Compare.DownloadHash, "Compute hash by downloading when otherwise unavailable. (warning: may be slow and use lots of data!)", "")
-	flags.DurationVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", Opt.MaxLock, "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
+	flags.FVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
 	flags.FVarP(cmdFlags, &Opt.ConflictResolve, "conflict-resolve", "", "Automatically resolve conflicts by preferring the version that is: "+ConflictResolveList+" (default: none)", "")
 	flags.FVarP(cmdFlags, &Opt.ConflictLoser, "conflict-loser", "", "Action to take on the loser of a sync conflict (when there is a winner) or on both files (when there is no winner): "+ConflictLoserList+" (default: num)", "")
 	flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")

cmd/bisync/lockfile.go

@@ -14,7 +14,7 @@ import (
 	"github.com/rclone/rclone/lib/terminal"
 )
 
-const basicallyforever = 200 * 365 * 24 * time.Hour
+const basicallyforever = fs.Duration(200 * 365 * 24 * time.Hour)
 
 var stopRenewal func()
 
@@ -66,9 +66,9 @@ func (b *bisyncRun) removeLockFile() {
 }
 
 func (b *bisyncRun) setLockFileExpiration() {
-	if b.opt.MaxLock > 0 && b.opt.MaxLock < 2*time.Minute {
+	if b.opt.MaxLock > 0 && b.opt.MaxLock < fs.Duration(2*time.Minute) {
 		fs.Logf(nil, Color(terminal.YellowFg, "--max-lock cannot be shorter than 2 minutes (unless 0.) Changing --max-lock from %v to %v"), b.opt.MaxLock, 2*time.Minute)
-		b.opt.MaxLock = 2 * time.Minute
+		b.opt.MaxLock = fs.Duration(2 * time.Minute)
 	} else if b.opt.MaxLock <= 0 {
 		b.opt.MaxLock = basicallyforever
 	}
@@ -80,7 +80,7 @@ func (b *bisyncRun) renewLockFile() {
 		data.Session = b.basePath
 		data.PID = strconv.Itoa(os.Getpid())
 		data.TimeRenewed = time.Now()
-		data.TimeExpires = time.Now().Add(b.opt.MaxLock)
+		data.TimeExpires = time.Now().Add(time.Duration(b.opt.MaxLock))
 
 		// save data file
 		df, err := os.Create(b.lockFile)
@@ -131,7 +131,7 @@ func (b *bisyncRun) startLockRenewal() func() {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		ticker := time.NewTicker(b.opt.MaxLock - time.Minute)
+		ticker := time.NewTicker(time.Duration(b.opt.MaxLock) - time.Minute)
 		for {
 			select {
 			case <-ticker.C:

cmd/bisync/operations.go

@@ -138,7 +138,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 					if b.SyncCI != nil {
 						fs.Infoc(nil, Color(terminal.YellowFg, "Telling Sync to wrap up early."))
 						b.SyncCI.MaxTransfer = 1
-						b.SyncCI.MaxDuration = 1 * time.Second
+						b.SyncCI.MaxDuration = fs.Duration(1 * time.Second)
 						b.SyncCI.CutoffMode = fs.CutoffModeSoft
 						gracePeriod := 30 * time.Second // TODO: flag to customize this?
 						if !waitFor("Canceling Sync if not done in", gracePeriod, func() bool { return b.CleanupCompleted }) {

cmd/bisync/queue.go

@@ -376,8 +376,8 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
 	return files.Save(queueFile)
 }
 
-func naptime(totalWait time.Duration) {
-	expireTime := time.Now().Add(totalWait)
+func naptime(totalWait fs.Duration) {
+	expireTime := time.Now().Add(time.Duration(totalWait))
 	fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))
 	for i := 0; time.Until(expireTime) > 0; i++ {
 		if i > 0 && i%10 == 0 {

cmd/bisync/resolve.go

@@ -4,8 +4,6 @@ import (
 	"context"
 	"fmt"
 	"math"
-	"mime"
-	"path"
 	"strings"
 	"time"
 
@@ -13,6 +11,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/lib/terminal"
+	"github.com/rclone/rclone/lib/transform"
 )
 
 // Prefer describes strategies for resolving sync conflicts
@@ -97,8 +96,8 @@ func (b *bisyncRun) setResolveDefaults(ctx context.Context) error {
 	}
 	// replace glob variables, if any
 	t := time.Now() // capture static time here so it is the same for all files throughout this run
-	b.opt.ConflictSuffix1 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix1, t)
-	b.opt.ConflictSuffix2 = bilib.AppyTimeGlobs(b.opt.ConflictSuffix2, t)
+	b.opt.ConflictSuffix1 = transform.AppyTimeGlobs(b.opt.ConflictSuffix1, t)
+	b.opt.ConflictSuffix2 = transform.AppyTimeGlobs(b.opt.ConflictSuffix2, t)
 
 	// append dot (intentionally allow more than one)
 	b.opt.ConflictSuffix1 = "." + b.opt.ConflictSuffix1
@@ -130,6 +129,7 @@ type (
 		path2 namePair
 	}
 )
+
 type namePair struct {
 	oldName string
 	newName string
@@ -240,24 +240,7 @@ func SuffixName(ctx context.Context, remote, suffix string) string {
 	}
 	ci := fs.GetConfig(ctx)
 	if ci.SuffixKeepExtension {
-		var (
-			base  = remote
-			exts  = ""
-			first = true
-			ext   = path.Ext(remote)
-		)
-		for ext != "" {
-			// Look second and subsequent extensions in mime types.
-			// If they aren't found then don't keep it as an extension.
-			if !first && mime.TypeByExtension(ext) == "" {
-				break
-			}
-			base = base[:len(base)-len(ext)]
-			exts = ext + exts
-			first = false
-			ext = path.Ext(base)
-		}
-		return base + suffix + exts
+		return transform.SuffixKeepExtension(remote, suffix)
 	}
 	return remote + suffix
 }

cmd/check/check.go

@@ -73,7 +73,7 @@ you what happened to it. These are reminiscent of diff files.
 - |* path| means path was present in source and destination but different.
 - |! path| means there was an error reading or hashing the source or dest.
 
-The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
+The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
 option for more information.
 `, "|", "`")
 

cmd/cmd.go

@@ -288,7 +288,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 			accounting.GlobalStats().ResetErrors()
 		}
 		if ci.RetriesInterval > 0 {
-			time.Sleep(ci.RetriesInterval)
+			time.Sleep(time.Duration(ci.RetriesInterval))
 		}
 	}
 	stopStats()

cmd/convmv/convmv.go

@@ -0,0 +1,106 @@
+// Package convmv provides the convmv command.
+package convmv
+
+import (
+	"context"
+	"errors"
+	"strings"
+
+	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs/config/flags"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
+	"github.com/rclone/rclone/lib/transform"
+	"github.com/spf13/cobra"
+)
+
+// Globals
+var (
+	deleteEmptySrcDirs = false
+	createEmptySrcDirs = false
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "")
+	flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "")
+}
+
+var commandDefinition = &cobra.Command{
+	Use:   "convmv dest:path --name-transform XXX",
+	Short: `Convert file and directory names in place.`,
+	// Warning¡ "¡" will be replaced by backticks below
+	Long: strings.ReplaceAll(`
+convmv supports advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations.
+
+`+transform.Help()+`Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
+
+The ¡--name-transform¡ flag is also available in ¡sync¡, ¡copy¡, and ¡move¡.
+
+## Files vs Directories
+
+By default ¡--name-transform¡ will only apply to file names. The means only the leaf file name will be transformed.
+However some of the transforms would be better applied to the whole path or just directories.
+To choose which which part of the file path is affected some tags can be added to the ¡--name-transform¡.
+
+| Tag | Effect |
+|------|------|
+| ¡file¡ | Only transform the leaf name of files (DEFAULT) |
+| ¡dir¡ | Only transform name of directories - these may appear anywhere in the path |
+| ¡all¡ | Transform the entire path for files and directories |
+
+This is used by adding the tag into the transform name like this: ¡--name-transform file,prefix=ABC¡ or ¡--name-transform dir,prefix=DEF¡.
+
+For some conversions using all is more likely to be useful, for example ¡--name-transform all,nfc¡.
+
+Note that ¡--name-transform¡ may not add path separators ¡/¡ to the name. This will cause an error.
+
+## Ordering and Conflicts
+
+* Transformations will be applied in the order specified by the user.
+    * If the ¡file¡ tag is in use (the default) then only the leaf name of files will be transformed.
+    * If the ¡dir¡ tag is in use then directories anywhere in the path will be transformed
+    * If the ¡all¡ tag is in use then directories and files anywhere in the path will be transformed
+    * Each transformation will be run one path segment at a time.
+    * If a transformation adds a ¡/¡ or ends up with an empty path segment then that will be an error.
+* It is up to the user to put the transformations in a sensible order.
+    * Conflicting transformations, such as ¡prefix¡ followed by ¡trimprefix¡ or ¡nfc¡ followed by ¡nfd¡, are possible.
+    * Instead of enforcing mutual exclusivity, transformations are applied in sequence as specified by the
+user, allowing for intentional use cases (e.g., trimming one prefix before adding another).
+    * Users should be aware that certain combinations may lead to unexpected results and should verify
+transformations using ¡--dry-run¡ before execution.
+
+## Race Conditions and Non-Deterministic Behavior
+
+Some transformations, such as ¡replace=old:new¡, may introduce conflicts where multiple source files map to the same destination name.
+This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these.
+* If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic.
+* Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.
+
+To minimize risks, users should:
+* Carefully review transformations that may introduce conflicts.
+* Use ¡--dry-run¡ to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
+* Avoid transformations that cause multiple distinct source files to map to the same destination name.
+* Consider disabling concurrency with ¡--transfers=1¡ if necessary.
+* Certain transformations (e.g. ¡prefix¡) will have a multiplying effect every time they are used. Avoid these when using ¡bisync¡.
+
+	`, "¡", "`"),
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.70",
+		"groups":            "Filter,Listing,Important,Copy",
+	},
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		fdst, srcFileName := cmd.NewFsFile(args[0])
+		cmd.Run(false, true, command, func() error {
+			if !transform.Transforming(context.Background()) {
+				return errors.New("--name-transform must be set")
+			}
+			if srcFileName == "" {
+				return sync.Transform(context.Background(), fdst, deleteEmptySrcDirs, createEmptySrcDirs)
+			}
+			return operations.TransformFile(context.Background(), fdst, srcFileName)
+		})
+	},
+}

cmd/convmv/convmv_test.go

@@ -0,0 +1,275 @@
+// Package convmv provides the convmv command.
+package convmv
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"path"
+	"slices"
+	"strings"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/all" // import all backends
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/filter"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/sync"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/lib/transform"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/text/unicode/norm"
+)
+
+// Some times used in the tests
+var (
+	t1    = fstest.Time("2001-02-03T04:05:06.499999999Z")
+	debug = ``
+)
+
+// TestMain drives the tests
+func TestMain(m *testing.M) {
+	fstest.TestMain(m)
+}
+
+func TestTransform(t *testing.T) {
+	type args struct {
+		TransformOpt     []string
+		TransformBackOpt []string
+		Lossless         bool // whether the TransformBackAlgo is always losslessly invertible
+	}
+	tests := []struct {
+		name string
+		args args
+	}{
+		{name: "NFC", args: args{
+			TransformOpt:     []string{"all,nfc"},
+			TransformBackOpt: []string{"all,nfd"},
+			Lossless:         false,
+		}},
+		{name: "NFD", args: args{
+			TransformOpt:     []string{"all,nfd"},
+			TransformBackOpt: []string{"all,nfc"},
+			Lossless:         false,
+		}},
+		{name: "base64", args: args{
+			TransformOpt:     []string{"all,base64encode"},
+			TransformBackOpt: []string{"all,base64encode"},
+			Lossless:         false,
+		}},
+		{name: "prefix", args: args{
+			TransformOpt:     []string{"all,prefix=PREFIX"},
+			TransformBackOpt: []string{"all,trimprefix=PREFIX"},
+			Lossless:         true,
+		}},
+		{name: "suffix", args: args{
+			TransformOpt:     []string{"all,suffix=SUFFIX"},
+			TransformBackOpt: []string{"all,trimsuffix=SUFFIX"},
+			Lossless:         true,
+		}},
+		{name: "truncate", args: args{
+			TransformOpt:     []string{"all,truncate=10"},
+			TransformBackOpt: []string{"all,truncate=10"},
+			Lossless:         false,
+		}},
+		{name: "encoder", args: args{
+			TransformOpt:     []string{"all,encoder=Colon,SquareBracket"},
+			TransformBackOpt: []string{"all,decoder=Colon,SquareBracket"},
+			Lossless:         true,
+		}},
+		{name: "ISO-8859-1", args: args{
+			TransformOpt:     []string{"all,ISO-8859-1"},
+			TransformBackOpt: []string{"all,ISO-8859-1"},
+			Lossless:         false,
+		}},
+		{name: "charmap", args: args{
+			TransformOpt:     []string{"all,charmap=ISO-8859-7"},
+			TransformBackOpt: []string{"all,charmap=ISO-8859-7"},
+			Lossless:         false,
+		}},
+		{name: "lowercase", args: args{
+			TransformOpt:     []string{"all,lowercase"},
+			TransformBackOpt: []string{"all,lowercase"},
+			Lossless:         false,
+		}},
+		{name: "ascii", args: args{
+			TransformOpt:     []string{"all,ascii"},
+			TransformBackOpt: []string{"all,ascii"},
+			Lossless:         false,
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := fstest.NewRun(t)
+			defer r.Finalise()
+
+			ctx := context.Background()
+			r.Mkdir(ctx, r.Flocal)
+			r.Mkdir(ctx, r.Fremote)
+			items := makeTestFiles(t, r, "dir1")
+			err := r.Fremote.Mkdir(ctx, "empty/empty")
+			require.NoError(t, err)
+			err = r.Flocal.Mkdir(ctx, "empty/empty")
+			require.NoError(t, err)
+			deleteDSStore(t, r)
+			r.CheckRemoteListing(t, items, []string{"dir1", "empty", "empty/empty"})
+			r.CheckLocalListing(t, items, []string{"dir1", "empty", "empty/empty"})
+
+			err = transform.SetOptions(ctx, tt.args.TransformOpt...)
+			require.NoError(t, err)
+
+			err = sync.Transform(ctx, r.Fremote, true, true)
+			assert.NoError(t, err)
+			compareNames(ctx, t, r, items)
+
+			transformedItems := transformItems(ctx, t, items)
+			r.CheckRemoteListing(t, transformedItems, []string{transform.Path(ctx, "dir1", true), transform.Path(ctx, "empty", true), transform.Path(ctx, "empty/empty", true)})
+			err = transform.SetOptions(ctx, tt.args.TransformBackOpt...)
+			require.NoError(t, err)
+			err = sync.Transform(ctx, r.Fremote, true, true)
+			assert.NoError(t, err)
+			compareNames(ctx, t, r, transformedItems)
+
+			if tt.args.Lossless {
+				deleteDSStore(t, r)
+				r.CheckRemoteListing(t, items, []string{"dir1", "empty", "empty/empty"})
+			}
+		})
+	}
+}
+
+// const alphabet = "ƀɀɠʀҠԀڀڠݠހ߀ကႠᄀᄠᅀᆀᇠሀሠበዠጠᎠᏀᐠᑀᑠᒀᒠᓀᓠᔀᔠᕀᕠᖀᖠᗀᗠᘀᘠᙀᚠᛀកᠠᡀᣀᦀ᧠ᨠᯀᰀᴀ⇠⋀⍀⍠⎀⎠⏀␀─┠╀╠▀■◀◠☀☠♀♠⚀⚠⛀⛠✀✠❀➀➠⠀⠠⡀⡠⢀⢠⣀⣠⤀⤠⥀⥠⦠⨠⩀⪀⪠⫠⬀⬠⭀ⰀⲀⲠⳀⴀⵀ⺠⻀㇀㐀㐠㑀㑠㒀㒠㓀㓠㔀㔠㕀㕠㖀㖠㗀㗠㘀㘠㙀㙠㚀㚠㛀㛠㜀㜠㝀㝠㞀㞠㟀㟠㠀㠠㡀㡠㢀㢠㣀㣠㤀㤠㥀㥠㦀㦠㧀㧠㨀㨠㩀㩠㪀㪠㫀㫠㬀㬠㭀㭠㮀㮠㯀㯠㰀㰠㱀㱠㲀㲠㳀㳠㴀㴠㵀㵠㶀㶠㷀㷠㸀㸠㹀㹠㺀㺠㻀㻠㼀㼠㽀㽠㾀㾠㿀㿠䀀䀠䁀䁠䂀䂠䃀䃠䄀䄠䅀䅠䆀䆠䇀䇠䈀䈠䉀䉠䊀䊠䋀䋠䌀䌠䍀䍠䎀䎠䏀䏠䐀䐠䑀䑠䒀䒠䓀䓠䔀䔠䕀䕠䖀䖠䗀䗠䘀䘠䙀䙠䚀䚠䛀䛠䜀䜠䝀䝠䞀䞠䟀䟠䠀䠠䡀䡠䢀䢠䣀䣠䤀䤠䥀䥠䦀䦠䧀䧠䨀䨠䩀䩠䪀䪠䫀䫠䬀䬠䭀䭠䮀䮠䯀䯠䰀䰠䱀䱠䲀䲠䳀䳠䴀䴠䵀䵠䶀䷀䷠一丠乀习亀亠什仠伀传佀你侀侠俀俠倀倠偀偠傀傠僀僠儀儠兀兠冀冠净几刀删剀剠劀加勀勠匀匠區占厀厠叀叠吀吠呀呠咀咠哀哠唀唠啀啠喀喠嗀嗠嘀嘠噀噠嚀嚠囀因圀圠址坠垀垠埀埠堀堠塀塠墀墠壀壠夀夠奀奠妀妠姀姠娀娠婀婠媀媠嫀嫠嬀嬠孀孠宀宠寀寠尀尠局屠岀岠峀峠崀崠嵀嵠嶀嶠巀巠帀帠幀幠庀庠廀廠开张彀彠往徠忀忠怀怠恀恠悀悠惀惠愀愠慀慠憀憠懀懠戀戠所扠技抠拀拠挀挠捀捠掀掠揀揠搀搠摀摠撀撠擀擠攀攠敀敠斀斠旀无昀映晀晠暀暠曀曠最朠杀杠枀枠柀柠栀栠桀桠梀梠检棠椀椠楀楠榀榠槀槠樀樠橀橠檀檠櫀櫠欀欠歀歠殀殠毀毠氀氠汀池沀沠泀泠洀洠浀浠涀涠淀淠渀渠湀湠満溠滀滠漀漠潀潠澀澠激濠瀀瀠灀灠炀炠烀烠焀焠煀煠熀熠燀燠爀爠牀牠犀犠狀狠猀猠獀獠玀玠珀珠琀琠瑀瑠璀璠瓀瓠甀甠畀畠疀疠痀痠瘀瘠癀癠皀皠盀盠眀眠着睠瞀瞠矀矠砀砠础硠碀碠磀磠礀礠祀祠禀禠秀秠稀稠穀穠窀窠竀章笀笠筀筠简箠節篠簀簠籀籠粀粠糀糠紀素絀絠綀綠緀締縀縠繀繠纀纠绀绠缀缠罀罠羀羠翀翠耀耠聀聠肀肠胀胠脀脠腀腠膀膠臀臠舀舠艀艠芀芠苀苠茀茠荀荠莀莠菀菠萀萠葀葠蒀蒠蓀蓠蔀蔠蕀蕠薀薠藀藠蘀蘠虀虠蚀蚠蛀蛠蜀蜠蝀蝠螀螠蟀蟠蠀蠠血衠袀袠裀裠褀褠襀襠覀覠觀觠言訠詀詠誀誠諀諠謀謠譀譠讀讠诀诠谀谠豀豠貀負賀賠贀贠赀赠趀趠跀跠踀踠蹀蹠躀躠軀軠輀輠轀轠辀辠迀迠退造遀遠邀邠郀郠鄀鄠酀酠醀醠釀釠鈀鈠鉀鉠銀銠鋀鋠錀錠鍀鍠鎀鎠鏀鏠鐀鐠鑀鑠钀钠铀铠销锠镀镠門閠闀闠阀阠陀陠隀隠雀雠需霠靀靠鞀鞠韀韠頀頠顀顠颀颠飀飠餀餠饀饠馀馠駀駠騀騠驀驠骀骠髀髠鬀鬠魀魠鮀鮠鯀鯠鰀鰠鱀鱠鲀鲠鳀鳠鴀鴠鵀鵠鶀鶠鷀鷠鸀鸠鹀鹠麀麠黀黠鼀鼠齀齠龀龠ꀀꀠꁀꁠꂀꂠꃀꃠꄀꄠꅀꅠꆀꆠꇀꇠꈀꈠꉀꉠꊀꊠꋀꋠꌀꌠꍀꍠꎀꎠꏀꏠꐀꐠꑀꑠ꒠ꔀꔠꕀꕠꖀꖠꗀꗠꙀꚠꛀ꜀꜠ꝀꞀꡀ測試_Русский___ě_áñ"
+const alphabet = "abcdefg123456789Ü"
+
+var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}
+
+func makeTestFiles(t *testing.T, r *fstest.Run, dir string) []fstest.Item {
+	t.Helper()
+	n := 0
+	// Create test files
+	items := []fstest.Item{}
+	for _, c := range alphabet {
+		var out strings.Builder
+		for i := rune(0); i < 7; i++ {
+			out.WriteRune(c + i)
+		}
+		fileName := path.Join(dir, fmt.Sprintf("%04d-%s.txt", n, out.String()))
+		fileName = strings.ToValidUTF8(fileName, "")
+		fileName = strings.NewReplacer(":", "", "<", "", ">", "", "?", "").Replace(fileName) // remove characters illegal on windows
+
+		if debug != "" {
+			fileName = debug
+		}
+
+		item := r.WriteObject(context.Background(), fileName, fileName, t1)
+		r.WriteFile(fileName, fileName, t1)
+		items = append(items, item)
+		n++
+
+		if debug != "" {
+			break
+		}
+	}
+
+	for _, extra := range extras {
+		item := r.WriteObject(context.Background(), extra, extra, t1)
+		r.WriteFile(extra, extra, t1)
+		items = append(items, item)
+	}
+
+	return items
+}
+
+func deleteDSStore(t *testing.T, r *fstest.Run) {
+	ctxDSStore, fi := filter.AddConfig(context.Background())
+	err := fi.AddRule(`+ *.DS_Store`)
+	assert.NoError(t, err)
+	err = fi.AddRule(`- **`)
+	assert.NoError(t, err)
+	err = operations.Delete(ctxDSStore, r.Fremote)
+	assert.NoError(t, err)
+}
+
+func compareNames(ctx context.Context, t *testing.T, r *fstest.Run, items []fstest.Item) {
+	var entries fs.DirEntries
+
+	deleteDSStore(t, r)
+	err := walk.ListR(context.Background(), r.Fremote, "", true, -1, walk.ListObjects, func(e fs.DirEntries) error {
+		entries = append(entries, e...)
+		return nil
+	})
+	assert.NoError(t, err)
+	entries = slices.DeleteFunc(entries, func(E fs.DirEntry) bool { // remove those pesky .DS_Store files
+		if strings.Contains(E.Remote(), ".DS_Store") {
+			err := operations.DeleteFile(context.Background(), E.(fs.Object))
+			assert.NoError(t, err)
+			return true
+		}
+		return false
+	})
+	require.Equal(t, len(items), entries.Len())
+
+	// sort by CONVERTED name
+	slices.SortStableFunc(items, func(a, b fstest.Item) int {
+		aConv := transform.Path(ctx, a.Path, false)
+		bConv := transform.Path(ctx, b.Path, false)
+		return cmp.Compare(aConv, bConv)
+	})
+	slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
+		return cmp.Compare(a.Remote(), b.Remote())
+	})
+
+	for i, e := range entries {
+		expect := transform.Path(ctx, items[i].Path, false)
+		msg := fmt.Sprintf("expected %v, got %v", detectEncoding(expect), detectEncoding(e.Remote()))
+		assert.Equal(t, expect, e.Remote(), msg)
+	}
+}
+
+func transformItems(ctx context.Context, t *testing.T, items []fstest.Item) []fstest.Item {
+	transformedItems := []fstest.Item{}
+	for _, item := range items {
+		newPath := transform.Path(ctx, item.Path, false)
+		newItem := item
+		newItem.Path = newPath
+		transformedItems = append(transformedItems, newItem)
+	}
+	return transformedItems
+}
+
+func detectEncoding(s string) string {
+	if norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
+		return "BOTH"
+	}
+	if !norm.NFC.IsNormalString(s) && norm.NFD.IsNormalString(s) {
+		return "NFD"
+	}
+	if norm.NFC.IsNormalString(s) && !norm.NFD.IsNormalString(s) {
+		return "NFC"
+	}
+	return "OTHER"
+}
+
+func TestUnicodeEquivalence(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+
+	ctx := context.Background()
+	r.Mkdir(ctx, r.Fremote)
+	const remote = "Über"
+	item := r.WriteObject(ctx, remote, "", t1)
+
+	obj, err := r.Fremote.NewObject(ctx, remote) // can't use r.CheckRemoteListing here as it forces NFC
+	require.NoError(t, err)
+	require.NotEmpty(t, obj)
+
+	err = transform.SetOptions(ctx, "all,nfc")
+	require.NoError(t, err)
+
+	err = sync.Transform(ctx, r.Fremote, true, true)
+	assert.NoError(t, err)
+	item.Path = norm.NFC.String(item.Path)
+	r.CheckRemoteListing(t, []fstest.Item{item}, nil)
+}

cmd/copy/copy.go

@@ -8,18 +8,23 @@ import (
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
 
 var (
 	createEmptySrcDirs = false
+	loggerOpt          = operations.LoggerOpt{}
+	loggerFlagsOpt     = operationsflags.AddLoggerFlagsOptions{}
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy", "")
+	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
+	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -90,19 +95,30 @@ for more info.
 **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
 
 **Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
-`, "|", "`"),
+
+`, "|", "`") + operationsflags.Help(),
 	Annotations: map[string]string{
 		"groups": "Copy,Filter,Listing,Important",
 	},
 	Run: func(command *cobra.Command, args []string) {
-
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
-			if srcFileName == "" {
-				return sync.CopyDir(context.Background(), fdst, fsrc, createEmptySrcDirs)
+			ctx := context.Background()
+			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
+			if err != nil {
+				return err
 			}
-			return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
+			defer close()
+
+			if loggerFlagsOpt.AnySet() {
+				ctx = operations.WithSyncLogger(ctx, loggerOpt)
+			}
+
+			if srcFileName == "" {
+				return sync.CopyDir(ctx, fdst, fsrc, createEmptySrcDirs)
+			}
+			return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
 		})
 	},
 }

cmd/copyto/copyto.go

@@ -6,12 +6,21 @@ import (
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
 
+var (
+	loggerOpt      = operations.LoggerOpt{}
+	loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
+	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -46,7 +55,8 @@ the destination.
 *If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'*
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
-`,
+
+` + operationsflags.Help(),
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.35",
 		"groups":            "Copy,Filter,Listing,Important",
@@ -55,10 +65,21 @@ the destination.
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 		cmd.Run(true, true, command, func() error {
-			if srcFileName == "" {
-				return sync.CopyDir(context.Background(), fdst, fsrc, false)
+			ctx := context.Background()
+			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
+			if err != nil {
+				return err
 			}
-			return operations.CopyFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
+			defer close()
+
+			if loggerFlagsOpt.AnySet() {
+				ctx = operations.WithSyncLogger(ctx, loggerOpt)
+			}
+
+			if srcFileName == "" {
+				return sync.CopyDir(ctx, fdst, fsrc, false)
+			}
+			return operations.CopyFile(ctx, fdst, fsrc, dstFileName, srcFileName)
 		})
 	},
 }

cmd/help.go

@@ -191,7 +191,6 @@ func setupRootCommand(rootCmd *cobra.Command) {
 	})
 
 	cobra.OnInitialize(initConfig)
-
 }
 
 // Traverse the tree of commands running fn on each

cmd/mountlib/mount.md

@@ -240,7 +240,7 @@ to run as the SYSTEM account. A third alternative is to use the
 Read more in the [install documentation](https://rclone.org/install/).
 Note that when running rclone as another user, it will not use
 the configuration file from your profile unless you tell it to
-with the [`--config`](https://rclone.org/docs/#config-config-file) option.
+with the [`--config`](https://rclone.org/docs/#config-string) option.
 Note also that it is now the SYSTEM account that will have the owner
 permissions, and other accounts will have permissions according to the
 group or others scopes. As mentioned above, these will then not get the

cmd/move/move.go

@@ -8,6 +8,7 @@ import (
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
@@ -16,6 +17,8 @@ import (
 var (
 	deleteEmptySrcDirs = false
 	createEmptySrcDirs = false
+	loggerOpt          = operations.LoggerOpt{}
+	loggerFlagsOpt     = operationsflags.AddLoggerFlagsOptions{}
 )
 
 func init() {
@@ -23,6 +26,8 @@ func init() {
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "")
 	flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "")
+	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
+	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -66,7 +71,8 @@ for more info.
 |--dry-run| or the |--interactive|/|-i| flag.
 
 **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
-`, "|", "`"),
+
+`, "|", "`") + operationsflags.Help(),
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.19",
 		"groups":            "Filter,Listing,Important,Copy",
@@ -75,10 +81,21 @@ for more info.
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
-			if srcFileName == "" {
-				return sync.MoveDir(context.Background(), fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
+			ctx := context.Background()
+			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
+			if err != nil {
+				return err
 			}
-			return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
+			defer close()
+
+			if loggerFlagsOpt.AnySet() {
+				ctx = operations.WithSyncLogger(ctx, loggerOpt)
+			}
+
+			if srcFileName == "" {
+				return sync.MoveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
+			}
+			return operations.MoveFile(ctx, fdst, fsrc, srcFileName, srcFileName)
 		})
 	},
 }

cmd/moveto/moveto.go

@@ -6,12 +6,21 @@ import (
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
 
+var (
+	loggerOpt      = operations.LoggerOpt{}
+	loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
+	cmdFlags := commandDefinition.Flags()
+	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
+	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -47,7 +56,8 @@ successful transfer.
 ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
-`,
+
+` + operationsflags.Help(),
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.35",
 		"groups":            "Filter,Listing,Important,Copy",
@@ -57,10 +67,21 @@ successful transfer.
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 
 		cmd.Run(true, true, command, func() error {
-			if srcFileName == "" {
-				return sync.MoveDir(context.Background(), fdst, fsrc, false, false)
+			ctx := context.Background()
+			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
+			if err != nil {
+				return err
 			}
-			return operations.MoveFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
+			defer close()
+
+			if loggerFlagsOpt.AnySet() {
+				ctx = operations.WithSyncLogger(ctx, loggerOpt)
+			}
+
+			if srcFileName == "" {
+				return sync.MoveDir(ctx, fdst, fsrc, false, false)
+			}
+			return operations.MoveFile(ctx, fdst, fsrc, dstFileName, srcFileName)
 		})
 	},
 }

cmd/ncdu/ncdu.go

@@ -6,6 +6,8 @@ package ncdu
 import (
 	"context"
 	"fmt"
+	"log/slog"
+	"os"
 	"path"
 	"reflect"
 	"sort"
@@ -925,23 +927,19 @@ func (u *UI) Run() error {
 		return fmt.Errorf("screen init: %w", err)
 	}
 
-	// Hijack fs.LogOutput so that it doesn't corrupt the screen.
-	if logOutput := fs.LogOutput; !log.Redirected() {
-		type log struct {
-			text  string
-			level fs.LogLevel
-		}
-		var logs []log
-		fs.LogOutput = func(level fs.LogLevel, text string) {
+	// Hijack log output so that it doesn't corrupt the screen.
+	if !log.Redirected() {
+		var logs []string
+		log.Handler.SetOutput(func(level slog.Level, text string) {
 			if len(logs) > 100 {
 				logs = logs[len(logs)-100:]
 			}
-			logs = append(logs, log{level: level, text: text})
-		}
+			logs = append(logs, text)
+		})
 		defer func() {
-			fs.LogOutput = logOutput
-			for i := range logs {
-				logOutput(logs[i].level, logs[i].text)
+			log.Handler.ResetOutput()
+			for _, text := range logs {
+				_, _ = os.Stderr.WriteString(text)
 			}
 		}()
 	}

cmd/progress.go

@@ -5,11 +5,11 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"log/slog"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/operations"
@@ -19,8 +19,6 @@ import (
 const (
 	// interval between progress prints
 	defaultProgressInterval = 500 * time.Millisecond
-	// time format for logging
-	logTimeFormat = "2006/01/02 15:04:05"
 )
 
 // startProgress starts the progress bar printing
@@ -28,15 +26,13 @@ const (
 // It returns a func which should be called to stop the stats.
 func startProgress() func() {
 	stopStats := make(chan struct{})
-	oldLogOutput := fs.LogOutput
 	oldSyncPrint := operations.SyncPrintf
 
 	if !log.Redirected() {
 		// Intercept the log calls if not logging to file or syslog
-		fs.LogOutput = func(level fs.LogLevel, text string) {
-			printProgress(fmt.Sprintf("%s %-6s: %s", time.Now().Format(logTimeFormat), level, text))
-
-		}
+		log.Handler.SetOutput(func(level slog.Level, text string) {
+			printProgress(text)
+		})
 	}
 
 	// Intercept output from functions such as HashLister to stdout
@@ -60,7 +56,10 @@ func startProgress() func() {
 			case <-stopStats:
 				ticker.Stop()
 				printProgress("")
-				fs.LogOutput = oldLogOutput
+				if !log.Redirected() {
+					// Reset intercept of the log calls
+					log.Handler.ResetOutput()
+				}
 				operations.SyncPrintf = oldSyncPrint
 				fmt.Println("")
 				return

cmd/serve/s3/list.go

@@ -28,7 +28,8 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPr
 
 		if entry.IsDir() {
 			if addPrefix {
-				response.AddPrefix(objectPath)
+				prefixWithTrailingSlash := objectPath + "/"
+				response.AddPrefix(prefixWithTrailingSlash)
 				continue
 			}
 			err := b.entryListR(_vfs, bucket, path.Join(fdPath, object), "", false, response)

cmd/serve/s3/serve_s3.md

@@ -14,7 +14,7 @@ docs](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)).
 access.
 
 Please note that some clients may require HTTPS endpoints. See [the
-SSL docs](#ssl-tls) for more information.
+SSL docs](#tls-ssl) for more information.
 
 This command uses the [VFS directory cache](#vfs-virtual-file-system).
 All the functionality will work with `--vfs-cache-mode off`. Using

cmd/serve/s3/server.go

@@ -81,7 +81,6 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
 		gofakes3.WithIntegrityCheck(true), // Check Content-MD5 if supplied
 	)
 
-	w.handler = http.NewServeMux()
 	w.handler = w.faker.Server()
 
 	if proxy.Opt.AuthProxy != "" {

cmd/serve/s3/utils.go

@@ -45,6 +45,10 @@ func getFileHashByte(node any, hashType hash.Type) []byte {
 }
 
 func getFileHash(node any, hashType hash.Type) string {
+	if hashType == hash.None {
+		return ""
+	}
+
 	var o fs.Object
 
 	switch b := node.(type) {

cmd/serve/sftp/connection.go

@@ -95,61 +95,44 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
 		if err != nil {
 			return fmt.Errorf("send output failed: %w", err)
 		}
-	case "md5sum", "sha1sum":
-		ht := hash.MD5
-		if binary == "sha1sum" {
-			ht = hash.SHA1
+	case "md5sum":
+		return c.handleHashsumCommand(ctx, out, hash.MD5, args)
+	case "sha1sum":
+		return c.handleHashsumCommand(ctx, out, hash.SHA1, args)
+	case "crc32":
+		return c.handleHashsumCommand(ctx, out, hash.CRC32, args)
+	case "sha256sum":
+		return c.handleHashsumCommand(ctx, out, hash.SHA256, args)
+	case "b3sum":
+		return c.handleHashsumCommand(ctx, out, hash.BLAKE3, args)
+	case "xxh128sum":
+		return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
+	case "xxhsum":
+		argv := strings.SplitN(args, " ", 2)
+		if len(argv) == 0 || argv[0] != "-H2" {
+			return fmt.Errorf("%q not implemented", command)
 		}
-		if !c.vfs.Fs().Hashes().Contains(ht) {
-			return fmt.Errorf("%v hash not supported", ht)
-		}
-		var hashSum string
-		if args == "" {
-			// empty hash for no input
-			if ht == hash.MD5 {
-				hashSum = "d41d8cd98f00b204e9800998ecf8427e"
-			} else {
-				hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
-			}
-			args = "-"
+		if len(argv) > 1 {
+			args = argv[1]
 		} else {
-			node, err := c.vfs.Stat(args)
-			if err != nil {
-				return fmt.Errorf("hash failed finding file %q: %w", args, err)
+			args = ""
+		}
+		return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
+	case "rclone":
+		argv := strings.SplitN(args, " ", 3)
+		if len(argv) > 1 && argv[0] == "hashsum" {
+			var ht hash.Type
+			if err := ht.Set(argv[1]); err != nil {
+				return err
 			}
-			if node.IsDir() {
-				return errors.New("can't hash directory")
-			}
-			o, ok := node.DirEntry().(fs.ObjectInfo)
-			if !ok {
-				fs.Debugf(args, "File uploading - reading hash from VFS cache")
-				in, err := node.Open(os.O_RDONLY)
-				if err != nil {
-					return fmt.Errorf("hash vfs open failed: %w", err)
-				}
-				defer func() {
-					_ = in.Close()
-				}()
-				h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
-				if err != nil {
-					return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
-				}
-				_, err = io.Copy(h, in)
-				if err != nil {
-					return fmt.Errorf("hash vfs copy failed: %w", err)
-				}
-				hashSum = h.Sums()[ht]
+			if len(argv) > 2 {
+				args = argv[2]
 			} else {
-				hashSum, err = o.Hash(ctx, ht)
-				if err != nil {
-					return fmt.Errorf("hash failed: %w", err)
-				}
+				args = ""
 			}
+			return c.handleHashsumCommand(ctx, out, ht, args)
 		}
-		_, err = fmt.Fprintf(out, "%s  %s\n", hashSum, args)
-		if err != nil {
-			return fmt.Errorf("send output failed: %w", err)
-		}
+		return fmt.Errorf("%q not implemented", command)
 	case "echo":
 		// Special cases for legacy rclone command detection.
 		// Before rclone v1.49.0 the sftp backend used "echo 'abc' | md5sum" when
@@ -189,6 +172,74 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
 	return nil
 }
 
+// handleHashsumCommand is a helper to execCommand for common functionality of hashsum related commands
+func (c *conn) handleHashsumCommand(ctx context.Context, out io.Writer, ht hash.Type, args string) (err error) {
+	if !c.vfs.Fs().Hashes().Contains(ht) {
+		return fmt.Errorf("%v hash not supported", ht)
+	}
+	var hashSum string
+	if args == "" {
+		// empty hash for no input
+		switch ht {
+		case hash.MD5:
+			hashSum = "d41d8cd98f00b204e9800998ecf8427e"
+		case hash.SHA1:
+			hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+		case hash.CRC32:
+			hashSum = "00000000"
+		case hash.SHA256:
+			hashSum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+		case hash.BLAKE3:
+			hashSum = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
+		case hash.XXH3:
+			hashSum = "2d06800538d394c2"
+		case hash.XXH128:
+			hashSum = "99aa06d3014798d86001c324468d497f"
+		default:
+			return fmt.Errorf("%v hash not implemented", ht)
+		}
+		args = "-"
+	} else {
+		node, err := c.vfs.Stat(args)
+		if err != nil {
+			return fmt.Errorf("hash failed finding file %q: %w", args, err)
+		}
+		if node.IsDir() {
+			return errors.New("can't hash directory")
+		}
+		o, ok := node.DirEntry().(fs.ObjectInfo)
+		if !ok {
+			fs.Debugf(args, "File uploading - reading hash from VFS cache")
+			in, err := node.Open(os.O_RDONLY)
+			if err != nil {
+				return fmt.Errorf("hash vfs open failed: %w", err)
+			}
+			defer func() {
+				_ = in.Close()
+			}()
+			h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+			if err != nil {
+				return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
+			}
+			_, err = io.Copy(h, in)
+			if err != nil {
+				return fmt.Errorf("hash vfs copy failed: %w", err)
+			}
+			hashSum = h.Sums()[ht]
+		} else {
+			hashSum, err = o.Hash(ctx, ht)
+			if err != nil {
+				return fmt.Errorf("hash failed: %w", err)
+			}
+		}
+	}
+	_, err = fmt.Fprintf(out, "%s  %s\n", hashSum, args)
+	if err != nil {
+		return fmt.Errorf("send output failed: %w", err)
+	}
+	return nil
+}
+
 // handle a new incoming channel request
 func (c *conn) handleChannel(newChannel ssh.NewChannel) {
 	fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType())