docs: Fix help string for --name-transform

adds local caching to avoid re-parsing when possible

.github/workflows/build.yml

@@ -23,9 +23,6 @@ jobs:
   build:
     if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
     timeout-minutes: 60
-    defaults:
-      run:
-        shell: bash
     strategy:
       fail-fast: false
       matrix:
@@ -106,6 +103,7 @@ jobs:
           check-latest: true
 
       - name: Set environment variables
+        shell: bash
         run: |
           echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
           echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
@@ -114,6 +112,7 @@ jobs:
           if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
 
       - name: Install Libraries on Linux
+        shell: bash
         run: |
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
@@ -123,6 +122,7 @@ jobs:
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
+        shell: bash
         run: |
           # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
           # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
@@ -151,6 +151,7 @@ jobs:
         if: matrix.os == 'windows-latest'
 
       - name: Print Go version and environment
+        shell: bash
         run: |
           printf "Using go at: $(which go)\n"
           printf "Go version: $(go version)\n"
@@ -162,24 +163,29 @@ jobs:
           env
 
       - name: Build rclone
+        shell: bash
         run: |
           make
 
       - name: Rclone version
+        shell: bash
         run: |
           rclone version
 
       - name: Run tests
+        shell: bash
         run: |
           make quicktest
         if: matrix.quicktest
 
       - name: Race test
+        shell: bash
         run: |
           make racequicktest
         if: matrix.racequicktest
 
       - name: Run librclone tests
+        shell: bash
         run: |
           make -C librclone/ctest test
           make -C librclone/ctest clean
@@ -187,12 +193,14 @@ jobs:
         if: matrix.librclonetest
 
       - name: Compile all architectures test
+        shell: bash
         run: |
           make
           make compile_all
         if: matrix.compile_all
 
       - name: Deploy built binaries
+        shell: bash
         run: |
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
           make ci_beta
@@ -211,6 +219,7 @@ jobs:
     steps:
       - name: Get runner parameters
         id: get-runner-parameters
+        shell: bash
         run: |
           echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
           echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
@@ -305,6 +314,7 @@ jobs:
           go-version: '>=1.24.0-rc.1'
 
       - name: Set global environment variables
+        shell: bash
         run: |
           echo "VERSION=$(make version)" >> $GITHUB_ENV
 
@@ -323,6 +333,7 @@ jobs:
         run: env PATH=$PATH:~/go/bin gomobile bind -androidapi ${RCLONE_NDK_VERSION} -v -target=android/arm -javapkg=org.rclone -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} github.com/rclone/rclone/librclone/gomobile
 
       - name: arm-v7a Set environment variables
+        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/armv7a-linux-androideabi${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
@@ -336,6 +347,7 @@ jobs:
         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv7a .
 
       - name: arm64-v8a Set environment variables
+        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
@@ -348,6 +360,7 @@ jobs:
         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-armv8a .
 
       - name: x86 Set environment variables
+        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/i686-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV
@@ -360,6 +373,7 @@ jobs:
         run: go build -v -tags android -trimpath -ldflags '-s -X github.com/rclone/rclone/fs.Version='${VERSION} -o build/rclone-android-${RCLONE_NDK_VERSION}-x86 .
 
       - name: x64 Set environment variables
+        shell: bash
         run: |
           echo "CC=$(echo $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/x86_64-linux-android${RCLONE_NDK_VERSION}-clang)" >> $GITHUB_ENV
           echo "CC_FOR_TARGET=$CC" >> $GITHUB_ENV

CONTRIBUTING.md

@@ -572,19 +572,3 @@ Then, run `go build -buildmode=plugin -o PLUGIN_NAME.so .` to build the plugin.
 [Go reference](https://godoc.org/github.com/rclone/rclone/lib/plugin)
 
 [Minimal example](https://gist.github.com/terorie/21b517ee347828e899e1913efc1d684f)
-
-## Keeping a backend or command out of tree
-
-Rclone was designed to be modular so it is very easy to keep a backend
-or a command out of the main rclone source tree.
-
-So for example if you had a backend which accessed your proprietary
-systems or a command which was specialised for your needs you could
-add them out of tree.
-
-This may be easier than using a plugin and is supported on all
-platforms not just macOS and Linux.
-
-This is explained further in https://github.com/rclone/rclone_out_of_tree_example
-which has an example of an out of tree backend `ram` (which is a
-renamed version of the `memory` backend).

README.md

@@ -39,9 +39,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
   * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
-  * FileLu [:page_facing_up:](https://rclone.org/filelu/)
   * Files.com [:page_facing_up:](https://rclone.org/filescom/)
-  * FlashBlade [:page_facing_up:](https://rclone.org/s3/#pure-storage-flashblade)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
   * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
@@ -66,8 +64,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Magalu Object Storage [:page_facing_up:](https://rclone.org/s3/#magalu)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
-  * MEGA [:page_facing_up:](https://rclone.org/mega/)
-  * MEGA S4 Object Storage [:page_facing_up:](https://rclone.org/s3/#mega)
+  * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
   * Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
@@ -97,7 +94,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
-  * Seagate Lyve Cloud [:page_facing_up:](https://rclone.org/s3/#lyve)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
   * Selectel Object Storage [:page_facing_up:](https://rclone.org/s3/#selectel)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
@@ -112,7 +108,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
   * Zoho WorkDrive [:page_facing_up:](https://rclone.org/zoho/)
-  * Zata.ai [:page_facing_up:](https://rclone.org/s3/#Zata)
   * The local filesystem [:page_facing_up:](https://rclone.org/local/)
 
 Please see [the full list of all storage providers and their features](https://rclone.org/overview/)

VERSION

@@ -1 +1 @@
-v1.71.0
+v1.70.0

backend/all/all.go

@@ -14,12 +14,10 @@ import (
 	_ "github.com/rclone/rclone/backend/combine"
 	_ "github.com/rclone/rclone/backend/compress"
 	_ "github.com/rclone/rclone/backend/crypt"
-	_ "github.com/rclone/rclone/backend/doi"
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/dropbox"
 	_ "github.com/rclone/rclone/backend/fichier"
 	_ "github.com/rclone/rclone/backend/filefabric"
-	_ "github.com/rclone/rclone/backend/filelu"
 	_ "github.com/rclone/rclone/backend/filescom"
 	_ "github.com/rclone/rclone/backend/ftp"
 	_ "github.com/rclone/rclone/backend/gofile"

backend/azureblob/azureblob.go

@@ -72,7 +72,6 @@ const (
 	emulatorAccount      = "devstoreaccount1"
 	emulatorAccountKey   = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 	emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
-	sasCopyValidity      = time.Hour // how long SAS should last when doing server side copy
 )
 
 var (
@@ -560,11 +559,6 @@ type Fs struct {
 	pacer         *fs.Pacer                    // To pace and retry the API calls
 	uploadToken   *pacer.TokenDispenser        // control concurrency
 	publicAccess  container.PublicAccessType   // Container Public Access Level
-
-	// user delegation cache
-	userDelegationMu     sync.Mutex
-	userDelegation       *service.UserDelegationCredential
-	userDelegationExpiry time.Time
 }
 
 // Object describes an azure object
@@ -618,9 +612,6 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
 	containerName, containerPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
-	if f.opt.DirectoryMarkers && strings.HasSuffix(containerPath, "//") {
-		containerPath = containerPath[:len(containerPath)-1]
-	}
 	return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
 }
 
@@ -937,7 +928,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		}
 	case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
 		// User with username and password
-		//nolint:staticcheck // this is deprecated due to Azure policy
 		options := azidentity.UsernamePasswordCredentialOptions{
 			ClientOptions: policyClientOptions,
 		}
@@ -990,38 +980,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
-	case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
-		// Workload Identity based authentication
-		var options azidentity.ManagedIdentityCredentialOptions
-		options.ID = azidentity.ClientID(opt.MSIClientID)
-
-		msiCred, err := azidentity.NewManagedIdentityCredential(&options)
-		if err != nil {
-			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
-		}
-
-		getClientAssertions := func(context.Context) (string, error) {
-			token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
-				Scopes: []string{"api://AzureADTokenExchange"},
-			})
-
-			if err != nil {
-				return "", fmt.Errorf("failed to acquire MSI token: %w", err)
-			}
-
-			return token.Token, nil
-		}
-
-		assertOpts := &azidentity.ClientAssertionCredentialOptions{}
-		f.cred, err = azidentity.NewClientAssertionCredential(
-			opt.Tenant,
-			opt.ClientID,
-			getClientAssertions,
-			assertOpts)
-
-		if err != nil {
-			return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
-		}
 	case opt.UseAZ:
 		var options = azidentity.AzureCLICredentialOptions{}
 		f.cred, err = azidentity.NewAzureCLICredential(&options)
@@ -1255,7 +1213,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 					continue
 				}
 				// process directory markers as directories
-				remote, _ = strings.CutSuffix(remote, "/")
+				remote = strings.TrimRight(remote, "/")
 			}
 			remote = remote[len(prefix):]
 			if addContainer {
@@ -1576,7 +1534,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote, _ = strings.CutSuffix(remote, "/")
+	remote = strings.TrimRight(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
 		dir = ""
@@ -1726,38 +1684,6 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	return f.deleteContainer(ctx, container)
 }
 
-// Get a user delegation which is valid for at least sasCopyValidity
-//
-// This value is cached in f
-func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCredential, error) {
-	f.userDelegationMu.Lock()
-	defer f.userDelegationMu.Unlock()
-
-	if f.userDelegation != nil && time.Until(f.userDelegationExpiry) > sasCopyValidity {
-		return f.userDelegation, nil
-	}
-
-	// Validity window
-	start := time.Now().UTC()
-	expiry := start.Add(2 * sasCopyValidity)
-	startStr := start.Format(time.RFC3339)
-	expiryStr := expiry.Format(time.RFC3339)
-
-	// Acquire user delegation key from the service client
-	info := service.KeyInfo{
-		Start:  &startStr,
-		Expiry: &expiryStr,
-	}
-	userDelegationKey, err := f.svc.GetUserDelegationCredential(ctx, info, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get user delegation key: %w", err)
-	}
-
-	f.userDelegation = userDelegationKey
-	f.userDelegationExpiry = expiry
-	return f.userDelegation, nil
-}
-
 // getAuth gets auth to copy o.
 //
 // tokenOK is used to signal that token based auth (Microsoft Entra
@@ -1769,7 +1695,7 @@ func (f *Fs) getUserDelegation(ctx context.Context) (*service.UserDelegationCred
 // URL (not a SAS) and token will be empty.
 //
 // If tokenOK is true it may also return a token for the auth.
-func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err error) {
+func (o *Object) getAuth(ctx context.Context, tokenOK bool, noAuth bool) (srcURL string, token *string, err error) {
 	f := o.fs
 	srcBlobSVC := o.getBlobSVC()
 	srcURL = srcBlobSVC.URL()
@@ -1778,47 +1704,29 @@ func (o *Object) getAuth(ctx context.Context, noAuth bool) (srcURL string, err e
 	case noAuth:
 		// If same storage account then no auth needed
 	case f.cred != nil:
-		// Generate a User Delegation SAS URL using Azure AD credentials
-		userDelegationKey, err := f.getUserDelegation(ctx)
+		if !tokenOK {
+			return srcURL, token, errors.New("not supported: Microsoft Entra ID")
+		}
+		options := policy.TokenRequestOptions{}
+		accessToken, err := f.cred.GetToken(ctx, options)
 		if err != nil {
-			return "", fmt.Errorf("sas creation: %w", err)
+			return srcURL, token, fmt.Errorf("failed to create access token: %w", err)
 		}
-
-		// Build the SAS values
-		perms := sas.BlobPermissions{Read: true}
-		container, containerPath := o.split()
-		start := time.Now().UTC()
-		expiry := start.Add(sasCopyValidity)
-		vals := sas.BlobSignatureValues{
-			StartTime:     start,
-			ExpiryTime:    expiry,
-			Permissions:   perms.String(),
-			ContainerName: container,
-			BlobName:      containerPath,
-		}
-
-		// Sign with the delegation key
-		queryParameters, err := vals.SignWithUserDelegation(userDelegationKey)
-		if err != nil {
-			return "", fmt.Errorf("signing SAS with user delegation failed: %w", err)
-		}
-
-		// Append the SAS to the URL
-		srcURL = srcBlobSVC.URL() + "?" + queryParameters.Encode()
+		token = &accessToken.Token
 	case f.sharedKeyCred != nil:
 		// Generate a short lived SAS URL if using shared key credentials
-		expiry := time.Now().Add(sasCopyValidity)
+		expiry := time.Now().Add(time.Hour)
 		sasOptions := blob.GetSASURLOptions{}
 		srcURL, err = srcBlobSVC.GetSASURL(sas.BlobPermissions{Read: true}, expiry, &sasOptions)
 		if err != nil {
-			return srcURL, fmt.Errorf("failed to create SAS URL: %w", err)
+			return srcURL, token, fmt.Errorf("failed to create SAS URL: %w", err)
 		}
 	case f.anonymous || f.opt.SASURL != "":
 		// If using a SASURL or anonymous, no need for any extra auth
 	default:
-		return srcURL, errors.New("unknown authentication type")
+		return srcURL, token, errors.New("unknown authentication type")
 	}
-	return srcURL, nil
+	return srcURL, token, nil
 }
 
 // Do multipart parallel copy.
@@ -1839,7 +1747,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
 	o.fs = f
 	o.remote = remote
 
-	srcURL, err := src.getAuth(ctx, false)
+	srcURL, token, err := src.getAuth(ctx, true, false)
 	if err != nil {
 		return nil, fmt.Errorf("multipart copy: %w", err)
 	}
@@ -1860,7 +1768,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
 	var (
 		srcSize  = src.size
 		partSize = int64(chunksize.Calculator(o, src.size, blockblob.MaxBlocks, f.opt.ChunkSize))
-		numParts = (srcSize + partSize - 1) / partSize
+		numParts = (srcSize-1)/partSize + 1
 		blockIDs = make([]string, numParts) // list of blocks for finalize
 		g, gCtx  = errgroup.WithContext(ctx)
 		checker  = newCheckForInvalidBlockOrBlob("copy", o)
@@ -1883,8 +1791,7 @@ func (f *Fs) copyMultipart(ctx context.Context, remote, dstContainer, dstPath st
 					Count:  partSize,
 				},
 				// Specifies the authorization scheme and signature for the copy source.
-				// We use SAS URLs as this doesn't seem to work always
-				// CopySourceAuthorization: token,
+				CopySourceAuthorization: token,
 				// CPKInfo *blob.CPKInfo
 				// CPKScopeInfo *blob.CPKScopeInfo
 			}
@@ -1954,7 +1861,7 @@ func (f *Fs) copySinglepart(ctx context.Context, remote, dstContainer, dstPath s
 	dstBlobSVC := f.getBlobSVC(dstContainer, dstPath)
 
 	// Get the source auth - none needed for same storage account
-	srcURL, err := src.getAuth(ctx, f == src.fs)
+	srcURL, _, err := src.getAuth(ctx, false, f == src.fs)
 	if err != nil {
 		return nil, fmt.Errorf("single part copy: source auth: %w", err)
 	}
@@ -2269,6 +2176,11 @@ func (o *Object) getTags() (tags map[string]string) {
 // getBlobSVC creates a blob client
 func (o *Object) getBlobSVC() *blob.Client {
 	container, directory := o.split()
+	// If we are trying to remove an all / directory marker then
+	// this will have one / too many now.
+	if bucket.IsAllSlashes(o.remote) {
+		directory = strings.TrimSuffix(directory, "/")
+	}
 	return o.fs.getBlobSVC(container, directory)
 }
 
@@ -2951,9 +2863,6 @@ func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options [
 			return ui, err
 		}
 	}
-	// if ui.isDirMarker && strings.HasSuffix(containerPath, "//") {
-	// 	containerPath = containerPath[:len(containerPath)-1]
-	// }
 
 	// Update Mod time
 	o.updateMetadataWithModTime(src.ModTime(ctx))

backend/azurefiles/azurefiles.go

@@ -516,7 +516,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		}
 	case opt.ClientID != "" && opt.Tenant != "" && opt.Username != "" && opt.Password != "":
 		// User with username and password
-		//nolint:staticcheck // this is deprecated due to Azure policy
 		options := azidentity.UsernamePasswordCredentialOptions{
 			ClientOptions: policyClientOptions,
 		}
@@ -569,38 +568,6 @@ func newFsFromOptions(ctx context.Context, name, root string, opt *Options) (fs.
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
-	case opt.ClientID != "" && opt.Tenant != "" && opt.MSIClientID != "":
-		// Workload Identity based authentication
-		var options azidentity.ManagedIdentityCredentialOptions
-		options.ID = azidentity.ClientID(opt.MSIClientID)
-
-		msiCred, err := azidentity.NewManagedIdentityCredential(&options)
-		if err != nil {
-			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
-		}
-
-		getClientAssertions := func(context.Context) (string, error) {
-			token, err := msiCred.GetToken(context.Background(), policy.TokenRequestOptions{
-				Scopes: []string{"api://AzureADTokenExchange"},
-			})
-
-			if err != nil {
-				return "", fmt.Errorf("failed to acquire MSI token: %w", err)
-			}
-
-			return token.Token, nil
-		}
-
-		assertOpts := &azidentity.ClientAssertionCredentialOptions{}
-		cred, err = azidentity.NewClientAssertionCredential(
-			opt.Tenant,
-			opt.ClientID,
-			getClientAssertions,
-			assertOpts)
-
-		if err != nil {
-			return nil, fmt.Errorf("failed to acquire client assertion token: %w", err)
-		}
 	default:
 		return nil, errors.New("no authentication method configured")
 	}
@@ -954,7 +921,7 @@ func (o *Object) setMetadata(resp *file.GetPropertiesResponse) {
 	}
 }
 
-// getMetadata gets the metadata if it hasn't already been fetched
+// readMetaData gets the metadata if it hasn't already been fetched
 func (o *Object) getMetadata(ctx context.Context) error {
 	resp, err := o.fileClient().GetProperties(ctx, nil)
 	if err != nil {

backend/b2/b2.go

@@ -1673,21 +1673,6 @@ func (o *Object) getMetaData(ctx context.Context) (info *api.File, err error) {
 			return o.getMetaDataListing(ctx)
 		}
 	}
-
-	// If using versionAt we need to list the find the correct version.
-	if o.fs.opt.VersionAt.IsSet() {
-		info, err := o.getMetaDataListing(ctx)
-		if err != nil {
-			return nil, err
-		}
-
-		if info.Action == "hide" {
-			// Rerturn object not found error if the current version is deleted.
-			return nil, fs.ErrorObjectNotFound
-		}
-		return info, nil
-	}
-
 	_, info, err = o.getOrHead(ctx, "HEAD", nil)
 	return info, err
 }
@@ -1898,14 +1883,9 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
 	// length read from the listing.
-	// Additionally, the official examples return S3 headers
-	// instead of native, i.e. no file ID, use ones from listing.
 	if info.Size < 0 {
 		info.Size = o.size
 	}
-	if info.ID == "" {
-		info.ID = o.id
-	}
 	return resp, info, nil
 }
 

backend/b2/b2_internal_test.go

@@ -446,14 +446,14 @@ func (f *Fs) InternalTestVersions(t *testing.T) {
 				t.Run("List", func(t *testing.T) {
 					fstest.CheckListing(t, f, test.want)
 				})
-
-				t.Run("NewObject", func(t *testing.T) {
-					gotObj, gotErr := f.NewObject(ctx, fileName)
-					assert.Equal(t, test.wantErr, gotErr)
-					if gotErr == nil {
-						assert.Equal(t, test.wantSize, gotObj.Size())
-					}
-				})
+				// b2 NewObject doesn't work with VersionAt
+				//t.Run("NewObject", func(t *testing.T) {
+				//	gotObj, gotErr := f.NewObject(ctx, fileName)
+				//	assert.Equal(t, test.wantErr, gotErr)
+				//	if gotErr == nil {
+				//		assert.Equal(t, test.wantSize, gotObj.Size())
+				//	}
+				//})
 			})
 		}
 	})

backend/chunker/chunker.go

@@ -1861,8 +1861,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // baseMove chains to the wrapped Move or simulates it by Copy+Delete
 func (f *Fs) baseMove(ctx context.Context, src fs.Object, remote string, delMode int) (fs.Object, error) {
-	ctx, ci := fs.AddConfig(ctx)
-	ci.NameTransform = nil // ensure operations.Move does not double-transform here
 	var (
 		dest fs.Object
 		err  error

backend/combine/combine.go

@@ -858,7 +858,7 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) e
 		}
 		return wrappedCallback(entries)
 	}
-	return listP(ctx, uRemote, wrappedCallback)
+	return listP(ctx, dir, wrappedCallback)
 }
 
 // ListR lists the objects and directories of the Fs starting

backend/doi/api/dataversetypes.go

@@ -1,38 +0,0 @@
-// Type definitions specific to Dataverse
-
-package api
-
-// DataverseDatasetResponse is returned by the Dataverse dataset API
-type DataverseDatasetResponse struct {
-	Status string           `json:"status"`
-	Data   DataverseDataset `json:"data"`
-}
-
-// DataverseDataset is the representation of a dataset
-type DataverseDataset struct {
-	LatestVersion DataverseDatasetVersion `json:"latestVersion"`
-}
-
-// DataverseDatasetVersion is the representation of a dataset version
-type DataverseDatasetVersion struct {
-	LastUpdateTime string          `json:"lastUpdateTime"`
-	Files          []DataverseFile `json:"files"`
-}
-
-// DataverseFile is the representation of a file found in a dataset
-type DataverseFile struct {
-	DirectoryLabel string            `json:"directoryLabel"`
-	DataFile       DataverseDataFile `json:"dataFile"`
-}
-
-// DataverseDataFile represents file metadata details
-type DataverseDataFile struct {
-	ID                 int64  `json:"id"`
-	Filename           string `json:"filename"`
-	ContentType        string `json:"contentType"`
-	FileSize           int64  `json:"filesize"`
-	OriginalFileFormat string `json:"originalFileFormat"`
-	OriginalFileSize   int64  `json:"originalFileSize"`
-	OriginalFileName   string `json:"originalFileName"`
-	MD5                string `json:"md5"`
-}

backend/doi/api/inveniotypes.go

@@ -1,33 +0,0 @@
-// Type definitions specific to InvenioRDM
-
-package api
-
-// InvenioRecordResponse is the representation of a record stored in InvenioRDM
-type InvenioRecordResponse struct {
-	Links InvenioRecordResponseLinks `json:"links"`
-}
-
-// InvenioRecordResponseLinks represents a record's links
-type InvenioRecordResponseLinks struct {
-	Self string `json:"self"`
-}
-
-// InvenioFilesResponse is the representation of a record's files
-type InvenioFilesResponse struct {
-	Entries []InvenioFilesResponseEntry `json:"entries"`
-}
-
-// InvenioFilesResponseEntry is the representation of a file entry
-type InvenioFilesResponseEntry struct {
-	Key      string                         `json:"key"`
-	Checksum string                         `json:"checksum"`
-	Size     int64                          `json:"size"`
-	Updated  string                         `json:"updated"`
-	MimeType string                         `json:"mimetype"`
-	Links    InvenioFilesResponseEntryLinks `json:"links"`
-}
-
-// InvenioFilesResponseEntryLinks represents file links details
-type InvenioFilesResponseEntryLinks struct {
-	Content string `json:"content"`
-}

backend/doi/api/types.go

@@ -1,26 +0,0 @@
-// Package api has general type definitions for doi
-package api
-
-// DoiResolverResponse is returned by the DOI resolver API
-//
-// Reference: https://www.doi.org/the-identifier/resources/factsheets/doi-resolution-documentation
-type DoiResolverResponse struct {
-	ResponseCode int                        `json:"responseCode"`
-	Handle       string                     `json:"handle"`
-	Values       []DoiResolverResponseValue `json:"values"`
-}
-
-// DoiResolverResponseValue is a single handle record value
-type DoiResolverResponseValue struct {
-	Index     int                          `json:"index"`
-	Type      string                       `json:"type"`
-	Data      DoiResolverResponseValueData `json:"data"`
-	TTL       int                          `json:"ttl"`
-	Timestamp string                       `json:"timestamp"`
-}
-
-// DoiResolverResponseValueData is the data held in a handle value
-type DoiResolverResponseValueData struct {
-	Format string `json:"format"`
-	Value  any    `json:"value"`
-}

backend/doi/dataverse.go

@@ -1,112 +0,0 @@
-// Implementation for Dataverse
-
-package doi
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/backend/doi/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// Returns true if resolvedURL is likely a DOI hosted on a Dataverse intallation
-func activateDataverse(resolvedURL *url.URL) (isActive bool) {
-	queryValues := resolvedURL.Query()
-	persistentID := queryValues.Get("persistentId")
-	return persistentID != ""
-}
-
-// Resolve the main API endpoint for a DOI hosted on a Dataverse installation
-func resolveDataverseEndpoint(resolvedURL *url.URL) (provider Provider, endpoint *url.URL, err error) {
-	queryValues := resolvedURL.Query()
-	persistentID := queryValues.Get("persistentId")
-
-	query := url.Values{}
-	query.Add("persistentId", persistentID)
-	endpointURL := resolvedURL.ResolveReference(&url.URL{Path: "/api/datasets/:persistentId/", RawQuery: query.Encode()})
-
-	return Dataverse, endpointURL, nil
-}
-
-// dataverseProvider implements the doiProvider interface for Dataverse installations
-type dataverseProvider struct {
-	f *Fs
-}
-
-// ListEntries returns the full list of entries found at the remote, regardless of root
-func (dp *dataverseProvider) ListEntries(ctx context.Context) (entries []*Object, err error) {
-	// Use the cache if populated
-	cachedEntries, found := dp.f.cache.GetMaybe("files")
-	if found {
-		parsedEntries, ok := cachedEntries.([]Object)
-		if ok {
-			for _, entry := range parsedEntries {
-				newEntry := entry
-				entries = append(entries, &newEntry)
-			}
-			return entries, nil
-		}
-	}
-
-	filesURL := dp.f.endpoint
-	var res *http.Response
-	var result api.DataverseDatasetResponse
-	opts := rest.Opts{
-		Method:     "GET",
-		Path:       strings.TrimLeft(filesURL.EscapedPath(), "/"),
-		Parameters: filesURL.Query(),
-	}
-	err = dp.f.pacer.Call(func() (bool, error) {
-		res, err = dp.f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("readDir failed: %w", err)
-	}
-	modTime, modTimeErr := time.Parse(time.RFC3339, result.Data.LatestVersion.LastUpdateTime)
-	if modTimeErr != nil {
-		fs.Logf(dp.f, "error: could not parse last update time %v", modTimeErr)
-		modTime = timeUnset
-	}
-	for _, file := range result.Data.LatestVersion.Files {
-		contentURLPath := fmt.Sprintf("/api/access/datafile/%d", file.DataFile.ID)
-		query := url.Values{}
-		query.Add("format", "original")
-		contentURL := dp.f.endpoint.ResolveReference(&url.URL{Path: contentURLPath, RawQuery: query.Encode()})
-		entry := &Object{
-			fs:          dp.f,
-			remote:      path.Join(file.DirectoryLabel, file.DataFile.Filename),
-			contentURL:  contentURL.String(),
-			size:        file.DataFile.FileSize,
-			modTime:     modTime,
-			md5:         file.DataFile.MD5,
-			contentType: file.DataFile.ContentType,
-		}
-		if file.DataFile.OriginalFileName != "" {
-			entry.remote = path.Join(file.DirectoryLabel, file.DataFile.OriginalFileName)
-			entry.size = file.DataFile.OriginalFileSize
-			entry.contentType = file.DataFile.OriginalFileFormat
-		}
-		entries = append(entries, entry)
-	}
-	// Populate the cache
-	cacheEntries := []Object{}
-	for _, entry := range entries {
-		cacheEntries = append(cacheEntries, *entry)
-	}
-	dp.f.cache.Put("files", cacheEntries)
-	return entries, nil
-}
-
-func newDataverseProvider(f *Fs) doiProvider {
-	return &dataverseProvider{
-		f: f,
-	}
-}

backend/doi/doi.go

@@ -1,649 +0,0 @@
-// Package doi provides a filesystem interface for digital objects identified by DOIs.
-//
-// See: https://www.doi.org/the-identifier/what-is-a-doi/
-package doi
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/backend/doi/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/lib/cache"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-const (
-	// the URL of the DOI resolver
-	//
-	// Reference: https://www.doi.org/the-identifier/resources/factsheets/doi-resolution-documentation
-	doiResolverAPIURL = "https://doi.org/api"
-	minSleep          = 10 * time.Millisecond
-	maxSleep          = 2 * time.Second
-	decayConstant     = 2 // bigger for slower decay, exponential
-)
-
-var (
-	errorReadOnly = errors.New("doi remotes are read only")
-	timeUnset     = time.Unix(0, 0)
-)
-
-func init() {
-	fsi := &fs.RegInfo{
-		Name:        "doi",
-		Description: "DOI datasets",
-		NewFs:       NewFs,
-		CommandHelp: commandHelp,
-		Options: []fs.Option{{
-			Name:     "doi",
-			Help:     "The DOI or the doi.org URL.",
-			Required: true,
-		}, {
-			Name: fs.ConfigProvider,
-			Help: `DOI provider.
-
-The DOI provider can be set when rclone does not automatically recognize a supported DOI provider.`,
-			Examples: []fs.OptionExample{
-				{
-					Value: "auto",
-					Help:  "Auto-detect provider",
-				},
-				{
-					Value: string(Zenodo),
-					Help:  "Zenodo",
-				}, {
-					Value: string(Dataverse),
-					Help:  "Dataverse",
-				}, {
-					Value: string(Invenio),
-					Help:  "Invenio",
-				}},
-			Required: false,
-			Advanced: true,
-		}, {
-			Name: "doi_resolver_api_url",
-			Help: `The URL of the DOI resolver API to use.
-
-The DOI resolver can be set for testing or for cases when the the canonical DOI resolver API cannot be used.
-
-Defaults to "https://doi.org/api".`,
-			Required: false,
-			Advanced: true,
-		}},
-	}
-	fs.Register(fsi)
-}
-
-// Provider defines the type of provider hosting the DOI
-type Provider string
-
-const (
-	// Zenodo provider, see https://zenodo.org
-	Zenodo Provider = "zenodo"
-	// Dataverse provider, see https://dataverse.harvard.edu
-	Dataverse Provider = "dataverse"
-	// Invenio provider, see https://inveniordm.docs.cern.ch
-	Invenio Provider = "invenio"
-)
-
-// Options defines the configuration for this backend
-type Options struct {
-	Doi               string `config:"doi"`                  // The DOI, a digital identifier of an object, usually a dataset
-	Provider          string `config:"provider"`             // The DOI provider
-	DoiResolverAPIURL string `config:"doi_resolver_api_url"` // The URL of the DOI resolver API to use.
-}
-
-// Fs stores the interface to the remote HTTP files
-type Fs struct {
-	name        string         // name of this remote
-	root        string         // the path we are working on
-	provider    Provider       // the DOI provider
-	doiProvider doiProvider    // the interface used to interact with the DOI provider
-	features    *fs.Features   // optional features
-	opt         Options        // options for this backend
-	ci          *fs.ConfigInfo // global config
-	endpoint    *url.URL       // the main API endpoint for this remote
-	endpointURL string         // endpoint as a string
-	srv         *rest.Client   // the connection to the server
-	pacer       *fs.Pacer      // pacer for API calls
-	cache       *cache.Cache   // a cache for the remote metadata
-}
-
-// Object is a remote object that has been stat'd (so it exists, but is not necessarily open for reading)
-type Object struct {
-	fs          *Fs       // what this object is part of
-	remote      string    // the remote path
-	contentURL  string    // the URL where the contents of the file can be downloaded
-	size        int64     // size of the object
-	modTime     time.Time // modification time of the object
-	contentType string    // content type of the object
-	md5         string    // MD5 hash of the object content
-}
-
-// doiProvider is the interface used to list objects in a DOI
-type doiProvider interface {
-	// ListEntries returns the full list of entries found at the remote, regardless of root
-	ListEntries(ctx context.Context) (entries []*Object, err error)
-}
-
-// Parse the input string as a DOI
-// Examples:
-// 10.1000/182 -> 10.1000/182
-// https://doi.org/10.1000/182 -> 10.1000/182
-// doi:10.1000/182 -> 10.1000/182
-func parseDoi(doi string) string {
-	doiURL, err := url.Parse(doi)
-	if err != nil {
-		return doi
-	}
-	if doiURL.Scheme == "doi" {
-		return strings.TrimLeft(strings.TrimPrefix(doi, "doi:"), "/")
-	}
-	if strings.HasSuffix(doiURL.Hostname(), "doi.org") {
-		return strings.TrimLeft(doiURL.Path, "/")
-	}
-	return doi
-}
-
-// Resolve a DOI to a URL
-// Reference: https://www.doi.org/the-identifier/resources/factsheets/doi-resolution-documentation
-func resolveDoiURL(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, opt *Options) (doiURL *url.URL, err error) {
-	resolverURL := opt.DoiResolverAPIURL
-	if resolverURL == "" {
-		resolverURL = doiResolverAPIURL
-	}
-
-	var result api.DoiResolverResponse
-	params := url.Values{}
-	params.Add("index", "1")
-	opts := rest.Opts{
-		Method:     "GET",
-		RootURL:    resolverURL,
-		Path:       "/handles/" + opt.Doi,
-		Parameters: params,
-	}
-	err = pacer.Call(func() (bool, error) {
-		res, err := srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	if result.ResponseCode != 1 {
-		return nil, fmt.Errorf("could not resolve DOI (error code %d)", result.ResponseCode)
-	}
-	resolvedURLStr := ""
-	for _, value := range result.Values {
-		if value.Type == "URL" && value.Data.Format == "string" {
-			valueStr, ok := value.Data.Value.(string)
-			if !ok {
-				return nil, fmt.Errorf("could not resolve DOI (incorrect response format)")
-			}
-			resolvedURLStr = valueStr
-		}
-	}
-	resolvedURL, err := url.Parse(resolvedURLStr)
-	if err != nil {
-		return nil, err
-	}
-	return resolvedURL, nil
-}
-
-// Resolve the passed configuration into a provider and enpoint
-func resolveEndpoint(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, opt *Options) (provider Provider, endpoint *url.URL, err error) {
-	resolvedURL, err := resolveDoiURL(ctx, srv, pacer, opt)
-	if err != nil {
-		return "", nil, err
-	}
-
-	switch opt.Provider {
-	case string(Dataverse):
-		return resolveDataverseEndpoint(resolvedURL)
-	case string(Invenio):
-		return resolveInvenioEndpoint(ctx, srv, pacer, resolvedURL)
-	case string(Zenodo):
-		return resolveZenodoEndpoint(ctx, srv, pacer, resolvedURL, opt.Doi)
-	}
-
-	hostname := strings.ToLower(resolvedURL.Hostname())
-	if hostname == "dataverse.harvard.edu" || activateDataverse(resolvedURL) {
-		return resolveDataverseEndpoint(resolvedURL)
-	}
-	if hostname == "zenodo.org" || strings.HasSuffix(hostname, ".zenodo.org") {
-		return resolveZenodoEndpoint(ctx, srv, pacer, resolvedURL, opt.Doi)
-	}
-	if activateInvenio(ctx, srv, pacer, resolvedURL) {
-		return resolveInvenioEndpoint(ctx, srv, pacer, resolvedURL)
-	}
-
-	return "", nil, fmt.Errorf("provider '%s' is not supported", resolvedURL.Hostname())
-}
-
-// Make the http connection from the passed options
-func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err error) {
-	provider, endpoint, err := resolveEndpoint(ctx, f.srv, f.pacer, opt)
-	if err != nil {
-		return false, err
-	}
-
-	// Update f with the new parameters
-	f.srv.SetRoot(endpoint.ResolveReference(&url.URL{Path: "/"}).String())
-	f.endpoint = endpoint
-	f.endpointURL = endpoint.String()
-	f.provider = provider
-	f.opt.Provider = string(provider)
-
-	switch f.provider {
-	case Dataverse:
-		f.doiProvider = newDataverseProvider(f)
-	case Invenio, Zenodo:
-		f.doiProvider = newInvenioProvider(f)
-	default:
-		return false, fmt.Errorf("provider type '%s' not supported", f.provider)
-	}
-
-	// Determine if the root is a file
-	entries, err := f.doiProvider.ListEntries(ctx)
-	if err != nil {
-		return false, err
-	}
-	for _, entry := range entries {
-		if entry.remote == f.root {
-			isFile = true
-			break
-		}
-	}
-	return isFile, nil
-}
-
-// retryErrorCodes is a slice of error codes that we will retry
-var retryErrorCodes = []int{
-	429, // Too Many Requests.
-	500, // Internal Server Error
-	502, // Bad Gateway
-	503, // Service Unavailable
-	504, // Gateway Timeout
-	509, // Bandwidth Limit Exceeded
-}
-
-// shouldRetry returns a boolean as to whether this res and err
-// deserve to be retried. It returns the err as a convenience.
-func shouldRetry(ctx context.Context, res *http.Response, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(res, retryErrorCodes), err
-}
-
-// NewFs creates a new Fs object from the name and root. It connects to
-// the host specified in the config file.
-func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
-	root = strings.Trim(root, "/")
-
-	// Parse config into Options struct
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, err
-	}
-	opt.Doi = parseDoi(opt.Doi)
-
-	client := fshttp.NewClient(ctx)
-	ci := fs.GetConfig(ctx)
-	f := &Fs{
-		name:  name,
-		root:  root,
-		opt:   *opt,
-		ci:    ci,
-		srv:   rest.NewClient(client),
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		cache: cache.New(),
-	}
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-	}).Fill(ctx, f)
-
-	isFile, err := f.httpConnection(ctx, opt)
-	if err != nil {
-		return nil, err
-	}
-
-	if isFile {
-		// return an error with an fs which points to the parent
-		newRoot := path.Dir(f.root)
-		if newRoot == "." {
-			newRoot = ""
-		}
-		f.root = newRoot
-		return f, fs.ErrorIsFile
-	}
-
-	return f, nil
-}
-
-// Name returns the configured name of the file system
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root returns the root for the filesystem
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// String returns the URL for the filesystem
-func (f *Fs) String() string {
-	return fmt.Sprintf("DOI %s", f.opt.Doi)
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-// Precision is the remote http file system's modtime precision, which we have no way of knowing. We estimate at 1s
-func (f *Fs) Precision() time.Duration {
-	return time.Second
-}
-
-// Hashes returns hash.HashNone to indicate remote hashing is unavailable
-func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
-	// return hash.Set(hash.None)
-}
-
-// Mkdir makes the root directory of the Fs object
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	return errorReadOnly
-}
-
-// Remove a remote http file object
-func (o *Object) Remove(ctx context.Context) error {
-	return errorReadOnly
-}
-
-// Rmdir removes the root directory of the Fs object
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return errorReadOnly
-}
-
-// NewObject creates a new remote http file object
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	entries, err := f.doiProvider.ListEntries(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	remoteFullPath := remote
-	if f.root != "" {
-		remoteFullPath = path.Join(f.root, remote)
-	}
-
-	for _, entry := range entries {
-		if entry.Remote() == remoteFullPath {
-			return entry, nil
-		}
-	}
-
-	return nil, fs.ErrorObjectNotFound
-}
-
-// List the objects and directories in dir into entries.  The
-// entries can be returned in any order but should be for a
-// complete directory.
-//
-// dir should be "" to list the root, and should not have
-// trailing slashes.
-//
-// This should return ErrDirNotFound if the directory isn't
-// found.
-func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	fileEntries, err := f.doiProvider.ListEntries(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("error listing %q: %w", dir, err)
-	}
-
-	fullDir := path.Join(f.root, dir)
-	if fullDir != "" {
-		fullDir += "/"
-	}
-
-	dirPaths := map[string]bool{}
-	for _, entry := range fileEntries {
-		// First, filter out files not in `fullDir`
-		if !strings.HasPrefix(entry.remote, fullDir) {
-			continue
-		}
-		// Then, find entries in subfolers
-		remotePath := entry.remote
-		if fullDir != "" {
-			remotePath = strings.TrimLeft(strings.TrimPrefix(remotePath, fullDir), "/")
-		}
-		parts := strings.SplitN(remotePath, "/", 2)
-		if len(parts) == 1 {
-			newEntry := *entry
-			newEntry.remote = path.Join(dir, remotePath)
-			entries = append(entries, &newEntry)
-		} else {
-			dirPaths[path.Join(dir, parts[0])] = true
-		}
-	}
-
-	for dirPath := range dirPaths {
-		entry := fs.NewDir(dirPath, time.Time{})
-		entries = append(entries, entry)
-	}
-
-	return entries, nil
-}
-
-// Put in to the remote path with the modTime given of the given size
-//
-// May create the object even if it returns an error - if so
-// will return the object and the error, otherwise will return
-// nil and the error
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return nil, errorReadOnly
-}
-
-// PutStream uploads to the remote path with the modTime given of indeterminate size
-func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	return nil, errorReadOnly
-}
-
-// Fs is the filesystem this remote http file object is located within
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// String returns the URL to the remote HTTP file
-func (o *Object) String() string {
-	if o == nil {
-		return "<nil>"
-	}
-	return o.remote
-}
-
-// Remote the name of the remote HTTP file, relative to the fs root
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// Hash returns "" since HTTP (in Go or OpenSSH) doesn't support remote calculation of hashes
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.MD5 {
-		return "", hash.ErrUnsupported
-	}
-	return o.md5, nil
-}
-
-// Size returns the size in bytes of the remote http file
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// ModTime returns the modification time of the remote http file
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// SetModTime sets the modification and access time to the specified time
-//
-// it also updates the info field
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return errorReadOnly
-}
-
-// Storable returns whether the remote http file is a regular file (not a directory, symbolic link, block device, character device, named pipe, etc.)
-func (o *Object) Storable() bool {
-	return true
-}
-
-// Open a remote http file object for reading. Seek is supported
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	fs.FixRangeOption(options, o.size)
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: o.contentURL,
-		Options: options,
-	}
-	var res *http.Response
-	err = o.fs.pacer.Call(func() (bool, error) {
-		res, err = o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("Open failed: %w", err)
-	}
-
-	// Handle non-compliant redirects
-	if res.Header.Get("Location") != "" {
-		newURL, err := res.Location()
-		if err == nil {
-			opts.RootURL = newURL.String()
-			err = o.fs.pacer.Call(func() (bool, error) {
-				res, err = o.fs.srv.Call(ctx, &opts)
-				return shouldRetry(ctx, res, err)
-			})
-			if err != nil {
-				return nil, fmt.Errorf("Open failed: %w", err)
-			}
-		}
-	}
-
-	return res.Body, nil
-}
-
-// Update in to the object with the modTime given of the given size
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	return errorReadOnly
-}
-
-// MimeType of an Object if known, "" otherwise
-func (o *Object) MimeType(ctx context.Context) string {
-	return o.contentType
-}
-
-var commandHelp = []fs.CommandHelp{{
-	Name:  "metadata",
-	Short: "Show metadata about the DOI.",
-	Long: `This command returns a JSON object with some information about the DOI.
-
-    rclone backend medatadata doi: 
-
-It returns a JSON object representing metadata about the DOI.
-`,
-}, {
-	Name:  "set",
-	Short: "Set command for updating the config parameters.",
-	Long: `This set command can be used to update the config parameters
-for a running doi backend.
-
-Usage Examples:
-
-    rclone backend set doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
-    rclone rc backend/command command=set fs=doi: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
-    rclone rc backend/command command=set fs=doi: -o doi=NEW_DOI
-
-The option keys are named as they are in the config file.
-
-This rebuilds the connection to the doi backend when it is called with
-the new parameters. Only new parameters need be passed as the values
-will default to those currently in use.
-
-It doesn't return anything.
-`,
-}}
-
-// Command the backend to run a named command
-//
-// The command run is name
-// args may be used to read arguments from
-// opts may be used to read optional arguments from
-//
-// The result should be capable of being JSON encoded
-// If it is a string or a []string it will be shown to the user
-// otherwise it will be JSON encoded and shown to the user like that
-func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
-	switch name {
-	case "metadata":
-		return f.ShowMetadata(ctx)
-	case "set":
-		newOpt := f.opt
-		err := configstruct.Set(configmap.Simple(opt), &newOpt)
-		if err != nil {
-			return nil, fmt.Errorf("reading config: %w", err)
-		}
-		_, err = f.httpConnection(ctx, &newOpt)
-		if err != nil {
-			return nil, fmt.Errorf("updating session: %w", err)
-		}
-		f.opt = newOpt
-		keys := []string{}
-		for k := range opt {
-			keys = append(keys, k)
-		}
-		fs.Logf(f, "Updated config values: %s", strings.Join(keys, ", "))
-		return nil, nil
-	default:
-		return nil, fs.ErrorCommandNotFound
-	}
-}
-
-// ShowMetadata returns some metadata about the corresponding DOI
-func (f *Fs) ShowMetadata(ctx context.Context) (metadata interface{}, err error) {
-	doiURL, err := url.Parse("https://doi.org/" + f.opt.Doi)
-	if err != nil {
-		return nil, err
-	}
-
-	info := map[string]any{}
-	info["DOI"] = f.opt.Doi
-	info["URL"] = doiURL.String()
-	info["metadataURL"] = f.endpointURL
-	info["provider"] = f.provider
-	return info, nil
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs          = (*Fs)(nil)
-	_ fs.PutStreamer = (*Fs)(nil)
-	_ fs.Commander   = (*Fs)(nil)
-	_ fs.Object      = (*Object)(nil)
-	_ fs.MimeTyper   = (*Object)(nil)
-)

backend/doi/doi_internal_test.go

@@ -1,260 +0,0 @@
-package doi
-
-import (
-	"context"
-	"crypto/md5"
-	"encoding/hex"
-	"encoding/json"
-	"io"
-	"net/http"
-	"net/http/httptest"
-	"net/url"
-	"sort"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/rclone/rclone/backend/doi/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/hash"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-var remoteName = "TestDoi"
-
-func TestParseDoi(t *testing.T) {
-	// 10.1000/182 -> 10.1000/182
-	doi := "10.1000/182"
-	parsed := parseDoi(doi)
-	assert.Equal(t, "10.1000/182", parsed)
-
-	// https://doi.org/10.1000/182 -> 10.1000/182
-	doi = "https://doi.org/10.1000/182"
-	parsed = parseDoi(doi)
-	assert.Equal(t, "10.1000/182", parsed)
-
-	// https://dx.doi.org/10.1000/182 -> 10.1000/182
-	doi = "https://dxdoi.org/10.1000/182"
-	parsed = parseDoi(doi)
-	assert.Equal(t, "10.1000/182", parsed)
-
-	// doi:10.1000/182 -> 10.1000/182
-	doi = "doi:10.1000/182"
-	parsed = parseDoi(doi)
-	assert.Equal(t, "10.1000/182", parsed)
-
-	// doi://10.1000/182 -> 10.1000/182
-	doi = "doi://10.1000/182"
-	parsed = parseDoi(doi)
-	assert.Equal(t, "10.1000/182", parsed)
-}
-
-// prepareMockDoiResolverServer prepares a test server to resolve DOIs
-func prepareMockDoiResolverServer(t *testing.T, resolvedURL string) (doiResolverAPIURL string) {
-	mux := http.NewServeMux()
-
-	// Handle requests for resolving DOIs
-	mux.HandleFunc("GET /api/handles/{handle...}", func(w http.ResponseWriter, r *http.Request) {
-		// Check that we are resolving a DOI
-		handle := strings.TrimPrefix(r.URL.Path, "/api/handles/")
-		assert.NotEmpty(t, handle)
-		index := r.URL.Query().Get("index")
-		assert.Equal(t, "1", index)
-
-		// Return the most basic response
-		result := api.DoiResolverResponse{
-			ResponseCode: 1,
-			Handle:       handle,
-			Values: []api.DoiResolverResponseValue{
-				{
-					Index: 1,
-					Type:  "URL",
-					Data: api.DoiResolverResponseValueData{
-						Format: "string",
-						Value:  resolvedURL,
-					},
-				},
-			},
-		}
-		resultBytes, err := json.Marshal(result)
-		require.NoError(t, err)
-		w.Header().Add("Content-Type", "application/json")
-		_, err = w.Write(resultBytes)
-		require.NoError(t, err)
-	})
-
-	// Make the test server
-	ts := httptest.NewServer(mux)
-
-	// Close the server at the end of the test
-	t.Cleanup(ts.Close)
-
-	return ts.URL + "/api"
-}
-
-func md5Sum(text string) string {
-	hash := md5.Sum([]byte(text))
-	return hex.EncodeToString(hash[:])
-}
-
-// prepareMockZenodoServer prepares a test server that mocks Zenodo.org
-func prepareMockZenodoServer(t *testing.T, files map[string]string) *httptest.Server {
-	mux := http.NewServeMux()
-
-	// Handle requests for a single record
-	mux.HandleFunc("GET /api/records/{recordID...}", func(w http.ResponseWriter, r *http.Request) {
-		// Check that we are returning data about a single record
-		recordID := strings.TrimPrefix(r.URL.Path, "/api/records/")
-		assert.NotEmpty(t, recordID)
-
-		// Return the most basic response
-		selfURL, err := url.Parse("http://" + r.Host)
-		require.NoError(t, err)
-		selfURL = selfURL.JoinPath(r.URL.String())
-		result := api.InvenioRecordResponse{
-			Links: api.InvenioRecordResponseLinks{
-				Self: selfURL.String(),
-			},
-		}
-		resultBytes, err := json.Marshal(result)
-		require.NoError(t, err)
-		w.Header().Add("Content-Type", "application/json")
-		_, err = w.Write(resultBytes)
-		require.NoError(t, err)
-	})
-	// Handle requests for listing files in a record
-	mux.HandleFunc("GET /api/records/{record}/files", func(w http.ResponseWriter, r *http.Request) {
-		// Return the most basic response
-		filesBaseURL, err := url.Parse("http://" + r.Host)
-		require.NoError(t, err)
-		filesBaseURL = filesBaseURL.JoinPath("/api/files/")
-
-		entries := []api.InvenioFilesResponseEntry{}
-		for filename, contents := range files {
-			entries = append(entries,
-				api.InvenioFilesResponseEntry{
-					Key:      filename,
-					Checksum: md5Sum(contents),
-					Size:     int64(len(contents)),
-					Updated:  time.Now().UTC().Format(time.RFC3339),
-					MimeType: "text/plain; charset=utf-8",
-					Links: api.InvenioFilesResponseEntryLinks{
-						Content: filesBaseURL.JoinPath(filename).String(),
-					},
-				},
-			)
-		}
-
-		result := api.InvenioFilesResponse{
-			Entries: entries,
-		}
-		resultBytes, err := json.Marshal(result)
-		require.NoError(t, err)
-		w.Header().Add("Content-Type", "application/json")
-		_, err = w.Write(resultBytes)
-		require.NoError(t, err)
-	})
-	// Handle requests for file contents
-	mux.HandleFunc("/api/files/{file}", func(w http.ResponseWriter, r *http.Request) {
-		// Check that we are returning the contents of a file
-		filename := strings.TrimPrefix(r.URL.Path, "/api/files/")
-		assert.NotEmpty(t, filename)
-		contents, found := files[filename]
-		if !found {
-			w.WriteHeader(404)
-			return
-		}
-
-		// Return the most basic response
-		_, err := w.Write([]byte(contents))
-		require.NoError(t, err)
-	})
-
-	// Make the test server
-	ts := httptest.NewServer(mux)
-
-	// Close the server at the end of the test
-	t.Cleanup(ts.Close)
-
-	return ts
-}
-
-func TestZenodoRemote(t *testing.T) {
-	recordID := "2600782"
-	doi := "10.5281/zenodo.2600782"
-
-	// The files in the dataset
-	files := map[string]string{
-		"README.md": "This is a dataset.",
-		"data.txt":  "Some data",
-	}
-
-	ts := prepareMockZenodoServer(t, files)
-	resolvedURL := ts.URL + "/record/" + recordID
-
-	doiResolverAPIURL := prepareMockDoiResolverServer(t, resolvedURL)
-
-	testConfig := configmap.Simple{
-		"type":                 "doi",
-		"doi":                  doi,
-		"provider":             "zenodo",
-		"doi_resolver_api_url": doiResolverAPIURL,
-	}
-	f, err := NewFs(context.Background(), remoteName, "", testConfig)
-	require.NoError(t, err)
-
-	// Test listing the DOI files
-	entries, err := f.List(context.Background(), "")
-	require.NoError(t, err)
-
-	sort.Sort(entries)
-
-	require.Equal(t, len(files), len(entries))
-
-	e := entries[0]
-	assert.Equal(t, "README.md", e.Remote())
-	assert.Equal(t, int64(18), e.Size())
-	_, ok := e.(*Object)
-	assert.True(t, ok)
-
-	e = entries[1]
-	assert.Equal(t, "data.txt", e.Remote())
-	assert.Equal(t, int64(9), e.Size())
-	_, ok = e.(*Object)
-	assert.True(t, ok)
-
-	// Test reading the DOI files
-	o, err := f.NewObject(context.Background(), "README.md")
-	require.NoError(t, err)
-	assert.Equal(t, int64(18), o.Size())
-	md5Hash, err := o.Hash(context.Background(), hash.MD5)
-	require.NoError(t, err)
-	assert.Equal(t, "464352b1cab5240e44528a56fda33d9d", md5Hash)
-	fd, err := o.Open(context.Background())
-	require.NoError(t, err)
-	data, err := io.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, []byte(files["README.md"]), data)
-	do, ok := o.(fs.MimeTyper)
-	require.True(t, ok)
-	assert.Equal(t, "text/plain; charset=utf-8", do.MimeType(context.Background()))
-
-	o, err = f.NewObject(context.Background(), "data.txt")
-	require.NoError(t, err)
-	assert.Equal(t, int64(9), o.Size())
-	md5Hash, err = o.Hash(context.Background(), hash.MD5)
-	require.NoError(t, err)
-	assert.Equal(t, "5b82f8bf4df2bfb0e66ccaa7306fd024", md5Hash)
-	fd, err = o.Open(context.Background())
-	require.NoError(t, err)
-	data, err = io.ReadAll(fd)
-	require.NoError(t, err)
-	require.NoError(t, fd.Close())
-	assert.Equal(t, []byte(files["data.txt"]), data)
-	do, ok = o.(fs.MimeTyper)
-	require.True(t, ok)
-	assert.Equal(t, "text/plain; charset=utf-8", do.MimeType(context.Background()))
-}

backend/doi/doi_test.go

@@ -1,16 +0,0 @@
-// Test DOI filesystem interface
-package doi
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestDoi:",
-		NilObject:  (*Object)(nil),
-	})
-}

backend/doi/invenio.go

@@ -1,164 +0,0 @@
-// Implementation for InvenioRDM
-
-package doi
-
-import (
-	"context"
-	"fmt"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/backend/doi/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-var invenioRecordRegex = regexp.MustCompile(`\/records?\/(.+)`)
-
-// Returns true if resolvedURL is likely a DOI hosted on an InvenioRDM intallation
-func activateInvenio(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL) (isActive bool) {
-	_, _, err := resolveInvenioEndpoint(ctx, srv, pacer, resolvedURL)
-	return err == nil
-}
-
-// Resolve the main API endpoint for a DOI hosted on an InvenioRDM installation
-func resolveInvenioEndpoint(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL) (provider Provider, endpoint *url.URL, err error) {
-	var res *http.Response
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: resolvedURL.String(),
-	}
-	err = pacer.Call(func() (bool, error) {
-		res, err = srv.Call(ctx, &opts)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return "", nil, err
-	}
-
-	// First, attempt to grab the API URL from the headers
-	var linksetURL *url.URL
-	links := parseLinkHeader(res.Header.Get("Link"))
-	for _, link := range links {
-		if link.Rel == "linkset" && link.Type == "application/linkset+json" {
-			parsed, err := url.Parse(link.Href)
-			if err == nil {
-				linksetURL = parsed
-				break
-			}
-		}
-	}
-
-	if linksetURL != nil {
-		endpoint, err = checkInvenioAPIURL(ctx, srv, pacer, linksetURL)
-		if err == nil {
-			return Invenio, endpoint, nil
-		}
-		fs.Logf(nil, "using linkset URL failed: %s", err.Error())
-	}
-
-	// If there is no linkset header, try to grab the record ID from the URL
-	recordID := ""
-	resURL := res.Request.URL
-	match := invenioRecordRegex.FindStringSubmatch(resURL.EscapedPath())
-	if match != nil {
-		recordID = match[1]
-		guessedURL := res.Request.URL.ResolveReference(&url.URL{
-			Path: "/api/records/" + recordID,
-		})
-		endpoint, err = checkInvenioAPIURL(ctx, srv, pacer, guessedURL)
-		if err == nil {
-			return Invenio, endpoint, nil
-		}
-		fs.Logf(nil, "guessing the URL failed: %s", err.Error())
-	}
-
-	return "", nil, fmt.Errorf("could not resolve the Invenio API endpoint for '%s'", resolvedURL.String())
-}
-
-func checkInvenioAPIURL(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL) (endpoint *url.URL, err error) {
-	var result api.InvenioRecordResponse
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: resolvedURL.String(),
-	}
-	err = pacer.Call(func() (bool, error) {
-		res, err := srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	if result.Links.Self == "" {
-		return nil, fmt.Errorf("could not parse API response from '%s'", resolvedURL.String())
-	}
-	return url.Parse(result.Links.Self)
-}
-
-// invenioProvider implements the doiProvider interface for InvenioRDM installations
-type invenioProvider struct {
-	f *Fs
-}
-
-// ListEntries returns the full list of entries found at the remote, regardless of root
-func (ip *invenioProvider) ListEntries(ctx context.Context) (entries []*Object, err error) {
-	// Use the cache if populated
-	cachedEntries, found := ip.f.cache.GetMaybe("files")
-	if found {
-		parsedEntries, ok := cachedEntries.([]Object)
-		if ok {
-			for _, entry := range parsedEntries {
-				newEntry := entry
-				entries = append(entries, &newEntry)
-			}
-			return entries, nil
-		}
-	}
-
-	filesURL := ip.f.endpoint.JoinPath("files")
-	var result api.InvenioFilesResponse
-	opts := rest.Opts{
-		Method: "GET",
-		Path:   strings.TrimLeft(filesURL.EscapedPath(), "/"),
-	}
-	err = ip.f.pacer.Call(func() (bool, error) {
-		res, err := ip.f.srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("readDir failed: %w", err)
-	}
-	for _, file := range result.Entries {
-		modTime, modTimeErr := time.Parse(time.RFC3339, file.Updated)
-		if modTimeErr != nil {
-			fs.Logf(ip.f, "error: could not parse last update time %v", modTimeErr)
-			modTime = timeUnset
-		}
-		entry := &Object{
-			fs:          ip.f,
-			remote:      file.Key,
-			contentURL:  file.Links.Content,
-			size:        file.Size,
-			modTime:     modTime,
-			contentType: file.MimeType,
-			md5:         strings.TrimPrefix(file.Checksum, "md5:"),
-		}
-		entries = append(entries, entry)
-	}
-	// Populate the cache
-	cacheEntries := []Object{}
-	for _, entry := range entries {
-		cacheEntries = append(cacheEntries, *entry)
-	}
-	ip.f.cache.Put("files", cacheEntries)
-	return entries, nil
-}
-
-func newInvenioProvider(f *Fs) doiProvider {
-	return &invenioProvider{
-		f: f,
-	}
-}

backend/doi/link_header.go

@@ -1,75 +0,0 @@
-package doi
-
-import (
-	"regexp"
-	"strings"
-)
-
-var linkRegex = regexp.MustCompile(`^<(.+)>$`)
-var valueRegex = regexp.MustCompile(`^"(.+)"$`)
-
-// headerLink represents a link as presented in HTTP headers
-// MDN Reference: https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Link
-type headerLink struct {
-	Href   string
-	Rel    string
-	Type   string
-	Extras map[string]string
-}
-
-func parseLinkHeader(header string) (links []headerLink) {
-	for _, link := range strings.Split(header, ",") {
-		link = strings.TrimSpace(link)
-		parsed := parseLink(link)
-		if parsed != nil {
-			links = append(links, *parsed)
-		}
-	}
-	return links
-}
-
-func parseLink(link string) (parsedLink *headerLink) {
-	var parts []string
-	for _, part := range strings.Split(link, ";") {
-		parts = append(parts, strings.TrimSpace(part))
-	}
-
-	match := linkRegex.FindStringSubmatch(parts[0])
-	if match == nil {
-		return nil
-	}
-
-	result := &headerLink{
-		Href:   match[1],
-		Extras: map[string]string{},
-	}
-
-	for _, keyValue := range parts[1:] {
-		parsed := parseKeyValue(keyValue)
-		if parsed != nil {
-			key, value := parsed[0], parsed[1]
-			switch strings.ToLower(key) {
-			case "rel":
-				result.Rel = value
-			case "type":
-				result.Type = value
-			default:
-				result.Extras[key] = value
-			}
-		}
-	}
-	return result
-}
-
-func parseKeyValue(keyValue string) []string {
-	parts := strings.SplitN(keyValue, "=", 2)
-	if parts[0] == "" || len(parts) < 2 {
-		return nil
-	}
-	match := valueRegex.FindStringSubmatch(parts[1])
-	if match != nil {
-		parts[1] = match[1]
-		return parts
-	}
-	return parts
-}

backend/doi/link_header_internal_test.go

@@ -1,44 +0,0 @@
-package doi
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseLinkHeader(t *testing.T) {
-	header := "<https://zenodo.org/api/records/15063252> ; rel=\"linkset\" ; type=\"application/linkset+json\""
-	links := parseLinkHeader(header)
-	expected := headerLink{
-		Href:   "https://zenodo.org/api/records/15063252",
-		Rel:    "linkset",
-		Type:   "application/linkset+json",
-		Extras: map[string]string{},
-	}
-	assert.Contains(t, links, expected)
-
-	header = "<https://api.example.com/issues?page=2>; rel=\"prev\", <https://api.example.com/issues?page=4>; rel=\"next\", <https://api.example.com/issues?page=10>; rel=\"last\", <https://api.example.com/issues?page=1>; rel=\"first\""
-	links = parseLinkHeader(header)
-	expectedList := []headerLink{{
-		Href:   "https://api.example.com/issues?page=2",
-		Rel:    "prev",
-		Type:   "",
-		Extras: map[string]string{},
-	}, {
-		Href:   "https://api.example.com/issues?page=4",
-		Rel:    "next",
-		Type:   "",
-		Extras: map[string]string{},
-	}, {
-		Href:   "https://api.example.com/issues?page=10",
-		Rel:    "last",
-		Type:   "",
-		Extras: map[string]string{},
-	}, {
-		Href:   "https://api.example.com/issues?page=1",
-		Rel:    "first",
-		Type:   "",
-		Extras: map[string]string{},
-	}}
-	assert.Equal(t, links, expectedList)
-}

backend/doi/zenodo.go

@@ -1,47 +0,0 @@
-// Implementation for Zenodo
-
-package doi
-
-import (
-	"context"
-	"fmt"
-	"net/url"
-	"regexp"
-
-	"github.com/rclone/rclone/backend/doi/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-var zenodoRecordRegex = regexp.MustCompile(`zenodo[.](.+)`)
-
-// Resolve the main API endpoint for a DOI hosted on Zenodo
-func resolveZenodoEndpoint(ctx context.Context, srv *rest.Client, pacer *fs.Pacer, resolvedURL *url.URL, doi string) (provider Provider, endpoint *url.URL, err error) {
-	match := zenodoRecordRegex.FindStringSubmatch(doi)
-	if match == nil {
-		return "", nil, fmt.Errorf("could not derive API endpoint URL from '%s'", resolvedURL.String())
-	}
-
-	recordID := match[1]
-	endpointURL := resolvedURL.ResolveReference(&url.URL{Path: "/api/records/" + recordID})
-
-	var result api.InvenioRecordResponse
-	opts := rest.Opts{
-		Method:  "GET",
-		RootURL: endpointURL.String(),
-	}
-	err = pacer.Call(func() (bool, error) {
-		res, err := srv.CallJSON(ctx, &opts, nil, &result)
-		return shouldRetry(ctx, res, err)
-	})
-	if err != nil {
-		return "", nil, err
-	}
-
-	endpointURL, err = url.Parse(result.Links.Self)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return Zenodo, endpointURL, nil
-}

backend/filelu/api/types.go

@@ -1,81 +0,0 @@
-// Package api defines types for interacting with the FileLu API.
-package api
-
-import "encoding/json"
-
-// CreateFolderResponse represents the response for creating a folder.
-type CreateFolderResponse struct {
-	Status int    `json:"status"`
-	Msg    string `json:"msg"`
-	Result struct {
-		FldID interface{} `json:"fld_id"`
-	} `json:"result"`
-}
-
-// DeleteFolderResponse represents the response for deleting a folder.
-type DeleteFolderResponse struct {
-	Status int    `json:"status"`
-	Msg    string `json:"msg"`
-}
-
-// FolderListResponse represents the response for listing folders.
-type FolderListResponse struct {
-	Status int    `json:"status"`
-	Msg    string `json:"msg"`
-	Result struct {
-		Files []struct {
-			Name     string      `json:"name"`
-			FldID    json.Number `json:"fld_id"`
-			Path     string      `json:"path"`
-			FileCode string      `json:"file_code"`
-			Size     int64       `json:"size"`
-		} `json:"files"`
-		Folders []struct {
-			Name  string      `json:"name"`
-			FldID json.Number `json:"fld_id"`
-			Path  string      `json:"path"`
-		} `json:"folders"`
-	} `json:"result"`
-}
-
-// FileDirectLinkResponse represents the response for a direct link to a file.
-type FileDirectLinkResponse struct {
-	Status int    `json:"status"`
-	Msg    string `json:"msg"`
-	Result struct {
-		URL  string `json:"url"`
-		Size int64  `json:"size"`
-	} `json:"result"`
-}
-
-// FileInfoResponse represents the response for file information.
-type FileInfoResponse struct {
-	Status int    `json:"status"`
-	Msg    string `json:"msg"`
-	Result []struct {
-		Size     string `json:"size"`
-		Name     string `json:"name"`
-		FileCode string `json:"filecode"`
-		Hash     string `json:"hash"`
-		Status   int    `json:"status"`
-	} `json:"result"`
-}
-
-// DeleteFileResponse represents the response for deleting a file.
-type DeleteFileResponse struct {
-	Status int    `json:"status"`
-	Msg    string `json:"msg"`
-}
-
-// AccountInfoResponse represents the response for account information.
-type AccountInfoResponse struct {
-	Status int    `json:"status"` // HTTP status code of the response.
-	Msg    string `json:"msg"`    // Message describing the response.
-	Result struct {
-		PremiumExpire string `json:"premium_expire"` // Expiration date of premium access.
-		Email         string `json:"email"`          // User's email address.
-		UType         string `json:"utype"`          // User type (e.g., premium or free).
-		Storage       string `json:"storage"`        // Total storage available to the user.
-		StorageUsed   string `json:"storage_used"`   // Amount of storage used.
-	} `json:"result"` // Nested result structure containing account details.
-}

backend/filelu/filelu.go

@@ -1,366 +0,0 @@
-// Package filelu provides an interface to the FileLu storage system.
-package filelu
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/config"
-	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// Register the backend with Rclone
-func init() {
-	fs.Register(&fs.RegInfo{
-		Name:        "filelu",
-		Description: "FileLu Cloud Storage",
-		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Name:      "key",
-			Help:      "Your FileLu Rclone key from My Account",
-			Required:  true,
-			Sensitive: true,
-		},
-			{
-				Name:     config.ConfigEncoding,
-				Help:     config.ConfigEncodingHelp,
-				Advanced: true,
-				Default: (encoder.Base | //  Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot
-					encoder.EncodeSlash |
-					encoder.EncodeLtGt |
-					encoder.EncodeExclamation |
-					encoder.EncodeDoubleQuote |
-					encoder.EncodeSingleQuote |
-					encoder.EncodeBackQuote |
-					encoder.EncodeQuestion |
-					encoder.EncodeDollar |
-					encoder.EncodeColon |
-					encoder.EncodeAsterisk |
-					encoder.EncodePipe |
-					encoder.EncodeHash |
-					encoder.EncodePercent |
-					encoder.EncodeBackSlash |
-					encoder.EncodeCrLf |
-					encoder.EncodeDel |
-					encoder.EncodeCtl |
-					encoder.EncodeLeftSpace |
-					encoder.EncodeLeftPeriod |
-					encoder.EncodeLeftTilde |
-					encoder.EncodeLeftCrLfHtVt |
-					encoder.EncodeRightPeriod |
-					encoder.EncodeRightCrLfHtVt |
-					encoder.EncodeSquareBracket |
-					encoder.EncodeSemicolon |
-					encoder.EncodeRightSpace |
-					encoder.EncodeInvalidUtf8 |
-					encoder.EncodeDot),
-			},
-		}})
-}
-
-// Options defines the configuration for the FileLu backend
-type Options struct {
-	Key string               `config:"key"`
-	Enc encoder.MultiEncoder `config:"encoding"`
-}
-
-// Fs represents the FileLu file system
-type Fs struct {
-	name       string
-	root       string
-	opt        Options
-	features   *fs.Features
-	endpoint   string
-	pacer      *pacer.Pacer
-	srv        *rest.Client
-	client     *http.Client
-	targetFile string
-}
-
-// NewFs creates a new Fs object for FileLu
-func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
-	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse config: %w", err)
-	}
-
-	if opt.Key == "" {
-		return nil, fmt.Errorf("FileLu Rclone Key is required")
-	}
-
-	client := fshttp.NewClient(ctx)
-
-	if strings.TrimSpace(root) == "" {
-		root = ""
-	}
-	root = strings.Trim(root, "/")
-
-	filename := ""
-
-	f := &Fs{
-		name:       name,
-		opt:        *opt,
-		endpoint:   "https://filelu.com/rclone",
-		client:     client,
-		srv:        rest.NewClient(client).SetRoot("https://filelu.com/rclone"),
-		pacer:      pacer.New(),
-		targetFile: filename,
-		root:       root,
-	}
-
-	f.features = (&fs.Features{
-		CanHaveEmptyDirectories: true,
-		WriteMetadata:           false,
-		SlowHash:                true,
-	}).Fill(ctx, f)
-
-	rootContainer, rootDirectory := rootSplit(f.root)
-	if rootContainer != "" && rootDirectory != "" {
-		// Check to see if the (container,directory) is actually an existing file
-		oldRoot := f.root
-		newRoot, leaf := path.Split(oldRoot)
-		f.root = strings.Trim(newRoot, "/")
-		_, err := f.NewObject(ctx, leaf)
-		if err != nil {
-			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
-				// File doesn't exist or is a directory so return old f
-				f.root = strings.Trim(oldRoot, "/")
-				return f, nil
-			}
-			return nil, err
-		}
-		// return an error with an fs which points to the parent
-		return f, fs.ErrorIsFile
-	}
-
-	return f, nil
-}
-
-// Mkdir to create directory on remote server.
-func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	fullPath := path.Clean(f.root + "/" + dir)
-	_, err := f.createFolder(ctx, fullPath)
-	return err
-}
-
-// About provides usage statistics for the remote
-func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
-	accountInfo, err := f.getAccountInfo(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	totalStorage, err := parseStorageToBytes(accountInfo.Result.Storage)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse total storage: %w", err)
-	}
-
-	usedStorage, err := parseStorageToBytes(accountInfo.Result.StorageUsed)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse used storage: %w", err)
-	}
-
-	return &fs.Usage{
-		Total: fs.NewUsageValue(totalStorage), // Total bytes available
-		Used:  fs.NewUsageValue(usedStorage),  // Total bytes used
-		Free:  fs.NewUsageValue(totalStorage - usedStorage),
-	}, nil
-}
-
-// Purge deletes the directory and all its contents
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	fullPath := path.Join(f.root, dir)
-	if fullPath != "" {
-		fullPath = "/" + strings.Trim(fullPath, "/")
-	}
-	return f.deleteFolder(ctx, fullPath)
-}
-
-// List returns a list of files and folders
-// List returns a list of files and folders for the given directory
-func (f *Fs) List(ctx context.Context, dir string) (fs.DirEntries, error) {
-	// Compose full path for API call
-	fullPath := path.Join(f.root, dir)
-	fullPath = "/" + strings.Trim(fullPath, "/")
-	if fullPath == "/" {
-		fullPath = ""
-	}
-
-	var entries fs.DirEntries
-	result, err := f.getFolderList(ctx, fullPath)
-	if err != nil {
-		return nil, err
-	}
-
-	fldMap := map[string]bool{}
-	for _, folder := range result.Result.Folders {
-		fldMap[folder.FldID.String()] = true
-		if f.root == "" && dir == "" && strings.Contains(folder.Path, "/") {
-			continue
-		}
-
-		paths := strings.Split(folder.Path, fullPath+"/")
-		remote := paths[0]
-		if len(paths) > 1 {
-			remote = paths[1]
-		}
-
-		if strings.Contains(remote, "/") {
-			continue
-		}
-
-		pathsWithoutRoot := strings.Split(folder.Path, "/"+f.root+"/")
-		remotePathWithoutRoot := pathsWithoutRoot[0]
-		if len(pathsWithoutRoot) > 1 {
-			remotePathWithoutRoot = pathsWithoutRoot[1]
-		}
-		remotePathWithoutRoot = strings.TrimPrefix(remotePathWithoutRoot, "/")
-		entries = append(entries, fs.NewDir(remotePathWithoutRoot, time.Now()))
-	}
-	for _, file := range result.Result.Files {
-		if _, ok := fldMap[file.FldID.String()]; ok {
-			continue
-		}
-		remote := path.Join(dir, file.Name)
-		// trim leading slashes
-		remote = strings.TrimPrefix(remote, "/")
-		obj := &Object{
-			fs:      f,
-			remote:  remote,
-			size:    file.Size,
-			modTime: time.Now(),
-		}
-		entries = append(entries, obj)
-	}
-	return entries, nil
-}
-
-// Put uploads a file directly to the destination folder in the FileLu storage system.
-func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if src.Size() == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
-
-	err := f.uploadFile(ctx, in, src.Remote())
-	if err != nil {
-		return nil, err
-	}
-
-	newObject := &Object{
-		fs:      f,
-		remote:  src.Remote(),
-		size:    src.Size(),
-		modTime: src.ModTime(ctx),
-	}
-	fs.Infof(f, "Put: Successfully uploaded new file %q", src.Remote())
-	return newObject, nil
-}
-
-// Move moves the file to the specified location
-func (f *Fs) Move(ctx context.Context, src fs.Object, destinationPath string) (fs.Object, error) {
-
-	if strings.HasPrefix(destinationPath, "/") || strings.Contains(destinationPath, ":\\") {
-		dir := path.Dir(destinationPath)
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			return nil, fmt.Errorf("failed to create destination directory: %w", err)
-		}
-
-		reader, err := src.Open(ctx)
-		if err != nil {
-			return nil, fmt.Errorf("failed to open source file: %w", err)
-		}
-		defer func() {
-			if err := reader.Close(); err != nil {
-				fs.Logf(nil, "Failed to close file body: %v", err)
-			}
-		}()
-
-		dest, err := os.Create(destinationPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create destination file: %w", err)
-		}
-		defer func() {
-			if err := dest.Close(); err != nil {
-				fs.Logf(nil, "Failed to close file body: %v", err)
-			}
-		}()
-
-		if _, err := io.Copy(dest, reader); err != nil {
-			return nil, fmt.Errorf("failed to copy file content: %w", err)
-		}
-
-		if err := src.Remove(ctx); err != nil {
-			return nil, fmt.Errorf("failed to remove source file: %w", err)
-		}
-
-		return nil, nil
-	}
-
-	reader, err := src.Open(ctx)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open source object: %w", err)
-	}
-	defer func() {
-		if err := reader.Close(); err != nil {
-			fs.Logf(nil, "Failed to close file body: %v", err)
-		}
-	}()
-
-	err = f.uploadFile(ctx, reader, destinationPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to upload file to destination: %w", err)
-	}
-
-	if err := src.Remove(ctx); err != nil {
-		return nil, fmt.Errorf("failed to delete source file: %w", err)
-	}
-
-	return &Object{
-		fs:      f,
-		remote:  destinationPath,
-		size:    src.Size(),
-		modTime: src.ModTime(ctx),
-	}, nil
-}
-
-// Rmdir removes a directory
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	fullPath := path.Join(f.root, dir)
-	if fullPath != "" {
-		fullPath = "/" + strings.Trim(fullPath, "/")
-	}
-
-	// Step 1: Check if folder is empty
-	listResp, err := f.getFolderList(ctx, fullPath)
-	if err != nil {
-		return err
-	}
-	if len(listResp.Result.Files) > 0 || len(listResp.Result.Folders) > 0 {
-		return fmt.Errorf("Rmdir: directory %q is not empty", fullPath)
-	}
-
-	// Step 2: Delete the folder
-	return f.deleteFolder(ctx, fullPath)
-}
-
-// Check the interfaces are satisfied
-var (
-	_ fs.Fs      = (*Fs)(nil)
-	_ fs.Purger  = (*Fs)(nil)
-	_ fs.Abouter = (*Fs)(nil)
-	_ fs.Mover   = (*Fs)(nil)
-	_ fs.Object  = (*Object)(nil)
-)

backend/filelu/filelu_client.go

@@ -1,324 +0,0 @@
-package filelu
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/rclone/rclone/backend/filelu/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/rest"
-)
-
-// createFolder creates a folder at the specified path.
-func (f *Fs) createFolder(ctx context.Context, dirPath string) (*api.CreateFolderResponse, error) {
-	encodedDir := f.fromStandardPath(dirPath)
-	apiURL := fmt.Sprintf("%s/folder/create?folder_path=%s&key=%s",
-		f.endpoint,
-		url.QueryEscape(encodedDir),
-		url.QueryEscape(f.opt.Key), // assuming f.opt.Key is the correct field
-	)
-
-	req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create request: %w", err)
-	}
-
-	var resp *http.Response
-	result := api.CreateFolderResponse{}
-	err = f.pacer.Call(func() (bool, error) {
-		var innerErr error
-		resp, innerErr = f.client.Do(req)
-		return fserrors.ShouldRetry(innerErr), innerErr
-	})
-	if err != nil {
-		return nil, fmt.Errorf("request failed: %w", err)
-	}
-	defer func() {
-		if err := resp.Body.Close(); err != nil {
-			fs.Logf(nil, "Failed to close response body: %v", err)
-		}
-	}()
-
-	err = json.NewDecoder(resp.Body).Decode(&result)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding response: %w", err)
-	}
-	if result.Status != 200 {
-		return nil, fmt.Errorf("error: %s", result.Msg)
-	}
-
-	fs.Infof(f, "Successfully created folder %q with ID %v", dirPath, result.Result.FldID)
-	return &result, nil
-}
-
-// getFolderList List both files and folders in a directory.
-func (f *Fs) getFolderList(ctx context.Context, path string) (*api.FolderListResponse, error) {
-	encodedDir := f.fromStandardPath(path)
-	apiURL := fmt.Sprintf("%s/folder/list?folder_path=%s&key=%s",
-		f.endpoint,
-		url.QueryEscape(encodedDir),
-		url.QueryEscape(f.opt.Key),
-	)
-
-	var body []byte
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-		if err != nil {
-			return false, fmt.Errorf("failed to create request: %w", err)
-		}
-
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to list directory: %w", err)
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		body, err = io.ReadAll(resp.Body)
-		if err != nil {
-			return false, fmt.Errorf("error reading response body: %w", err)
-		}
-
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	var response api.FolderListResponse
-	if err := json.NewDecoder(bytes.NewReader(body)).Decode(&response); err != nil {
-		return nil, fmt.Errorf("error decoding response: %w", err)
-	}
-	if response.Status != 200 {
-		if strings.Contains(response.Msg, "Folder not found") {
-			return nil, fs.ErrorDirNotFound
-		}
-		return nil, fmt.Errorf("API error: %s", response.Msg)
-	}
-
-	for index := range response.Result.Folders {
-		response.Result.Folders[index].Path = f.toStandardPath(response.Result.Folders[index].Path)
-	}
-
-	for index := range response.Result.Files {
-		response.Result.Files[index].Name = f.toStandardPath(response.Result.Files[index].Name)
-	}
-
-	return &response, nil
-
-}
-
-// deleteFolder deletes a folder at the specified path.
-func (f *Fs) deleteFolder(ctx context.Context, fullPath string) error {
-	fullPath = f.fromStandardPath(fullPath)
-	deleteURL := fmt.Sprintf("%s/folder/delete?folder_path=%s&key=%s",
-		f.endpoint,
-		url.QueryEscape(fullPath),
-		url.QueryEscape(f.opt.Key),
-	)
-
-	delResp := api.DeleteFolderResponse{}
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", deleteURL, nil)
-		if err != nil {
-			return false, err
-		}
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return fserrors.ShouldRetry(err), err
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		body, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return false, err
-		}
-
-		if err := json.Unmarshal(body, &delResp); err != nil {
-			return false, fmt.Errorf("error decoding delete response: %w", err)
-		}
-		if delResp.Status != 200 {
-			return false, fmt.Errorf("delete error: %s", delResp.Msg)
-		}
-		return false, nil
-	})
-	if err != nil {
-		return err
-	}
-
-	fs.Infof(f, "Rmdir: successfully deleted %q", fullPath)
-	return nil
-}
-
-// getDirectLink of file from FileLu to download.
-func (f *Fs) getDirectLink(ctx context.Context, filePath string) (string, int64, error) {
-	filePath = f.fromStandardPath(filePath)
-	apiURL := fmt.Sprintf("%s/file/direct_link?file_path=%s&key=%s",
-		f.endpoint,
-		url.QueryEscape(filePath),
-		url.QueryEscape(f.opt.Key),
-	)
-
-	result := api.FileDirectLinkResponse{}
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-		if err != nil {
-			return false, fmt.Errorf("failed to create request: %w", err)
-		}
-
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to fetch direct link: %w", err)
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-			return false, fmt.Errorf("error decoding response: %w", err)
-		}
-
-		if result.Status != 200 {
-			return false, fmt.Errorf("API error: %s", result.Msg)
-		}
-
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-	if err != nil {
-		return "", 0, err
-	}
-
-	return result.Result.URL, result.Result.Size, nil
-}
-
-// deleteFile deletes a file based on filePath
-func (f *Fs) deleteFile(ctx context.Context, filePath string) error {
-	filePath = f.fromStandardPath(filePath)
-	apiURL := fmt.Sprintf("%s/file/remove?file_path=%s&key=%s",
-		f.endpoint,
-		url.QueryEscape(filePath),
-		url.QueryEscape(f.opt.Key),
-	)
-
-	result := api.DeleteFileResponse{}
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-		if err != nil {
-			return false, fmt.Errorf("failed to create request: %w", err)
-		}
-
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to fetch direct link: %w", err)
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-			return false, fmt.Errorf("error decoding response: %w", err)
-		}
-
-		if result.Status != 200 {
-			return false, fmt.Errorf("API error: %s", result.Msg)
-		}
-
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-	return err
-}
-
-// getAccountInfo retrieves account information
-func (f *Fs) getAccountInfo(ctx context.Context) (*api.AccountInfoResponse, error) {
-	opts := rest.Opts{
-		Method: "GET",
-		Path:   "/account/info",
-		Parameters: url.Values{
-			"key": {f.opt.Key},
-		},
-	}
-
-	var result api.AccountInfoResponse
-	err := f.pacer.Call(func() (bool, error) {
-		_, callErr := f.srv.CallJSON(ctx, &opts, nil, &result)
-		return fserrors.ShouldRetry(callErr), callErr
-	})
-
-	if err != nil {
-		return nil, err
-	}
-
-	if result.Status != 200 {
-		return nil, fmt.Errorf("error: %s", result.Msg)
-	}
-
-	return &result, nil
-}
-
-// getFileInfo retrieves file information based on file code
-func (f *Fs) getFileInfo(ctx context.Context, fileCode string) (*api.FileInfoResponse, error) {
-	u, _ := url.Parse(f.endpoint + "/file/info2")
-	q := u.Query()
-	q.Set("file_code", fileCode) // raw path — Go handles escaping properly here
-	q.Set("key", f.opt.Key)
-	u.RawQuery = q.Encode()
-
-	apiURL := f.endpoint + "/file/info2?" + u.RawQuery
-
-	var body []byte
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-		if err != nil {
-			return false, fmt.Errorf("failed to create request: %w", err)
-		}
-
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to fetch file info: %w", err)
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		body, err = io.ReadAll(resp.Body)
-		if err != nil {
-			return false, fmt.Errorf("error reading response body: %w", err)
-		}
-
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-	if err != nil {
-		return nil, err
-	}
-	result := api.FileInfoResponse{}
-
-	if err := json.NewDecoder(bytes.NewReader(body)).Decode(&result); err != nil {
-		return nil, fmt.Errorf("error decoding response: %w", err)
-	}
-
-	if result.Status != 200 || len(result.Result) == 0 {
-		return nil, fs.ErrorObjectNotFound
-	}
-
-	return &result, nil
-}

backend/filelu/filelu_file_uploader.go

@@ -1,193 +0,0 @@
-package filelu
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"mime/multipart"
-	"net/http"
-	"net/url"
-	"path"
-	"strings"
-
-	"github.com/rclone/rclone/fs"
-)
-
-// uploadFile uploads a file to FileLu
-func (f *Fs) uploadFile(ctx context.Context, fileContent io.Reader, fileFullPath string) error {
-	directory := path.Dir(fileFullPath)
-	fileName := path.Base(fileFullPath)
-	if directory == "." {
-		directory = ""
-	}
-	destinationFolderPath := path.Join(f.root, directory)
-	if destinationFolderPath != "" {
-		destinationFolderPath = "/" + strings.Trim(destinationFolderPath, "/")
-	}
-
-	existingEntries, err := f.List(ctx, path.Dir(fileFullPath))
-	if err != nil {
-		if errors.Is(err, fs.ErrorDirNotFound) {
-			err = f.Mkdir(ctx, path.Dir(fileFullPath))
-			if err != nil {
-				return fmt.Errorf("failed to create directory: %w", err)
-			}
-		} else {
-			return fmt.Errorf("failed to list existing files: %w", err)
-		}
-	}
-
-	for _, entry := range existingEntries {
-		if entry.Remote() == fileFullPath {
-			_, ok := entry.(fs.Object)
-			if !ok {
-				continue
-			}
-
-			// If the file exists but is different, remove it
-			filePath := "/" + strings.Trim(destinationFolderPath+"/"+fileName, "/")
-			err = f.deleteFile(ctx, filePath)
-			if err != nil {
-				return fmt.Errorf("failed to delete existing file: %w", err)
-			}
-		}
-	}
-
-	uploadURL, sessID, err := f.getUploadServer(ctx)
-	if err != nil {
-		return fmt.Errorf("failed to retrieve upload server: %w", err)
-	}
-
-	// Since the fileCode isn't used, just handle the error
-	if _, err := f.uploadFileWithDestination(ctx, uploadURL, sessID, fileName, fileContent, destinationFolderPath); err != nil {
-		return fmt.Errorf("failed to upload file: %w", err)
-	}
-
-	return nil
-}
-
-// getUploadServer gets the upload server URL with proper key authentication
-func (f *Fs) getUploadServer(ctx context.Context) (string, string, error) {
-	apiURL := fmt.Sprintf("%s/upload/server?key=%s", f.endpoint, url.QueryEscape(f.opt.Key))
-
-	var result struct {
-		Status int    `json:"status"`
-		SessID string `json:"sess_id"`
-		Result string `json:"result"`
-		Msg    string `json:"msg"`
-	}
-
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-		if err != nil {
-			return false, fmt.Errorf("failed to create request: %w", err)
-		}
-
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to get upload server: %w", err)
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-			return false, fmt.Errorf("error decoding response: %w", err)
-		}
-
-		if result.Status != 200 {
-			return false, fmt.Errorf("API error: %s", result.Msg)
-		}
-
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-
-	if err != nil {
-		return "", "", err
-	}
-
-	return result.Result, result.SessID, nil
-}
-
-// uploadFileWithDestination uploads a file directly to a specified folder using file content reader.
-func (f *Fs) uploadFileWithDestination(ctx context.Context, uploadURL, sessID, fileName string, fileContent io.Reader, dirPath string) (string, error) {
-	destinationPath := f.fromStandardPath(dirPath)
-	encodedFileName := f.fromStandardPath(fileName)
-	pr, pw := io.Pipe()
-	writer := multipart.NewWriter(pw)
-	isDeletionRequired := false
-	go func() {
-		defer func() {
-			if err := pw.Close(); err != nil {
-				fs.Logf(nil, "Failed to close: %v", err)
-			}
-		}()
-		_ = writer.WriteField("sess_id", sessID)
-		_ = writer.WriteField("utype", "prem")
-		_ = writer.WriteField("fld_path", destinationPath)
-
-		part, err := writer.CreateFormFile("file_0", encodedFileName)
-		if err != nil {
-			pw.CloseWithError(fmt.Errorf("failed to create form file: %w", err))
-			return
-		}
-
-		if _, err := io.Copy(part, fileContent); err != nil {
-			isDeletionRequired = true
-			pw.CloseWithError(fmt.Errorf("failed to copy file content: %w", err))
-			return
-		}
-
-		if err := writer.Close(); err != nil {
-			pw.CloseWithError(fmt.Errorf("failed to close writer: %w", err))
-		}
-	}()
-
-	var fileCode string
-	err := f.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "POST", uploadURL, pr)
-		if err != nil {
-			return false, fmt.Errorf("failed to create upload request: %w", err)
-		}
-		req.Header.Set("Content-Type", writer.FormDataContentType())
-
-		resp, err := f.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to send upload request: %w", err)
-		}
-		defer respBodyClose(resp.Body)
-
-		var result []struct {
-			FileCode   string `json:"file_code"`
-			FileStatus string `json:"file_status"`
-		}
-		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-			return false, fmt.Errorf("failed to parse upload response: %w", err)
-		}
-
-		if len(result) == 0 || result[0].FileStatus != "OK" {
-			return false, fmt.Errorf("upload failed with status: %s", result[0].FileStatus)
-		}
-
-		fileCode = result[0].FileCode
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-
-	if err != nil && isDeletionRequired {
-		// Attempt to delete the file if upload fails
-		_ = f.deleteFile(ctx, destinationPath+"/"+fileName)
-	}
-
-	return fileCode, err
-}
-
-// respBodyClose to check body response.
-func respBodyClose(responseBody io.Closer) {
-	if cerr := responseBody.Close(); cerr != nil {
-		fmt.Printf("Error closing response body: %v\n", cerr)
-	}
-}

backend/filelu/filelu_helper.go

@@ -1,112 +0,0 @@
-package filelu
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"path"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/fs/hash"
-)
-
-// errFileNotFound represent file not found error
-var errFileNotFound error = errors.New("file not found")
-
-// getFileCode retrieves the file code for a given file path
-func (f *Fs) getFileCode(ctx context.Context, filePath string) (string, error) {
-	// Prepare parent directory
-	parentDir := path.Dir(filePath)
-
-	// Call List to get all the files
-	result, err := f.getFolderList(ctx, parentDir)
-	if err != nil {
-		return "", err
-	}
-
-	for _, file := range result.Result.Files {
-		filePathFromServer := parentDir + "/" + file.Name
-		if parentDir == "/" {
-			filePathFromServer = "/" + file.Name
-		}
-		if filePath == filePathFromServer {
-			return file.FileCode, nil
-		}
-	}
-
-	return "", errFileNotFound
-}
-
-// Features returns the optional features of this Fs
-func (f *Fs) Features() *fs.Features {
-	return f.features
-}
-
-func (f *Fs) fromStandardPath(remote string) string {
-	return f.opt.Enc.FromStandardPath(remote)
-}
-
-func (f *Fs) toStandardPath(remote string) string {
-	return f.opt.Enc.ToStandardPath(remote)
-}
-
-// Hashes returns an empty hash set, indicating no hash support
-func (f *Fs) Hashes() hash.Set {
-	return hash.NewHashSet() // Properly creates an empty hash set
-}
-
-// Name returns the remote name
-func (f *Fs) Name() string {
-	return f.name
-}
-
-// Root returns the root path
-func (f *Fs) Root() string {
-	return f.root
-}
-
-// Precision returns the precision of the remote
-func (f *Fs) Precision() time.Duration {
-	return fs.ModTimeNotSupported
-}
-
-func (f *Fs) String() string {
-	return fmt.Sprintf("FileLu root '%s'", f.root)
-}
-
-// isFileCode checks if a string looks like a file code
-func isFileCode(s string) bool {
-	if len(s) != 12 {
-		return false
-	}
-	for _, c := range s {
-		if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')) {
-			return false
-		}
-	}
-	return true
-}
-
-func shouldRetry(err error) bool {
-	return fserrors.ShouldRetry(err)
-}
-
-func shouldRetryHTTP(code int) bool {
-	return code == 429 || code >= 500
-}
-
-func rootSplit(absPath string) (bucket, bucketPath string) {
-	// No bucket
-	if absPath == "" {
-		return "", ""
-	}
-	slash := strings.IndexRune(absPath, '/')
-	// Bucket but no path
-	if slash < 0 {
-		return absPath, ""
-	}
-	return absPath[:slash], absPath[slash+1:]
-}

backend/filelu/filelu_object.go

@@ -1,259 +0,0 @@
-package filelu
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"path"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/hash"
-)
-
-// Object describes a FileLu object
-type Object struct {
-	fs      *Fs
-	remote  string
-	size    int64
-	modTime time.Time
-}
-
-// NewObject creates a new Object for the given remote path
-func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	var filePath string
-	filePath = path.Join(f.root, remote)
-	filePath = "/" + strings.Trim(filePath, "/")
-
-	// Get File code
-	fileCode, err := f.getFileCode(ctx, filePath)
-	if err != nil {
-		return nil, fs.ErrorObjectNotFound
-	}
-
-	// Get File info
-	fileInfos, err := f.getFileInfo(ctx, fileCode)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get file info: %w", err)
-	}
-
-	fileInfo := fileInfos.Result[0]
-	size, _ := strconv.ParseInt(fileInfo.Size, 10, 64)
-
-	returnedRemote := remote
-	return &Object{
-		fs:      f,
-		remote:  returnedRemote,
-		size:    size,
-		modTime: time.Now(),
-	}, nil
-}
-
-// Open opens the object for reading
-func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	filePath := path.Join(o.fs.root, o.remote)
-	// Get direct link
-	directLink, size, err := o.fs.getDirectLink(ctx, filePath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get direct link: %w", err)
-	}
-
-	o.size = size
-
-	// Offset and Count for range download
-	var offset int64
-	var count int64
-	fs.FixRangeOption(options, o.size)
-	for _, option := range options {
-		switch x := option.(type) {
-		case *fs.RangeOption:
-			offset, count = x.Decode(o.size)
-			if count < 0 {
-				count = o.size - offset
-			}
-		case *fs.SeekOption:
-			offset = x.Offset
-			count = o.size
-		default:
-			if option.Mandatory() {
-				fs.Logf(o, "Unsupported mandatory option: %v", option)
-			}
-		}
-	}
-
-	var reader io.ReadCloser
-	err = o.fs.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", directLink, nil)
-		if err != nil {
-			return false, fmt.Errorf("failed to create download request: %w", err)
-		}
-
-		resp, err := o.fs.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), fmt.Errorf("failed to download file: %w", err)
-		}
-
-		if resp.StatusCode != http.StatusOK {
-			defer func() {
-				if err := resp.Body.Close(); err != nil {
-					fs.Logf(nil, "Failed to close response body: %v", err)
-				}
-			}()
-			return false, fmt.Errorf("failed to download file: HTTP %d", resp.StatusCode)
-		}
-
-		// Wrap the response body to handle offset and count
-		currentContents, err := io.ReadAll(resp.Body)
-		if err != nil {
-			return false, fmt.Errorf("failed to read response body: %w", err)
-		}
-
-		if offset > 0 {
-			if offset > int64(len(currentContents)) {
-				return false, fmt.Errorf("offset %d exceeds file size %d", offset, len(currentContents))
-			}
-			currentContents = currentContents[offset:]
-		}
-		if count > 0 && count < int64(len(currentContents)) {
-			currentContents = currentContents[:count]
-		}
-		reader = io.NopCloser(bytes.NewReader(currentContents))
-
-		return false, nil
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return reader, nil
-}
-
-// Update updates the object with new data
-func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if src.Size() <= 0 {
-		return fs.ErrorCantUploadEmptyFiles
-	}
-
-	err := o.fs.uploadFile(ctx, in, o.remote)
-	if err != nil {
-		return fmt.Errorf("failed to upload file: %w", err)
-	}
-	o.size = src.Size()
-	return nil
-}
-
-// Remove deletes the object from FileLu
-func (o *Object) Remove(ctx context.Context) error {
-	fullPath := "/" + strings.Trim(path.Join(o.fs.root, o.remote), "/")
-
-	err := o.fs.deleteFile(ctx, fullPath)
-	if err != nil {
-		return err
-	}
-	fs.Infof(o.fs, "Successfully deleted file: %s", fullPath)
-	return nil
-}
-
-// Hash returns the MD5 hash of an object
-func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.MD5 {
-		return "", hash.ErrUnsupported
-	}
-
-	var fileCode string
-	if isFileCode(o.fs.root) {
-		fileCode = o.fs.root
-	} else {
-		matches := regexp.MustCompile(`\((.*?)\)`).FindAllStringSubmatch(o.remote, -1)
-		for _, match := range matches {
-			if len(match) > 1 && len(match[1]) == 12 {
-				fileCode = match[1]
-				break
-			}
-		}
-	}
-	if fileCode == "" {
-		return "", fmt.Errorf("no valid file code found in the remote path")
-	}
-
-	apiURL := fmt.Sprintf("%s/file/info?file_code=%s&key=%s",
-		o.fs.endpoint, url.QueryEscape(fileCode), url.QueryEscape(o.fs.opt.Key))
-
-	var result struct {
-		Status int    `json:"status"`
-		Msg    string `json:"msg"`
-		Result []struct {
-			Hash string `json:"hash"`
-		} `json:"result"`
-	}
-	err := o.fs.pacer.Call(func() (bool, error) {
-		req, err := http.NewRequestWithContext(ctx, "GET", apiURL, nil)
-		if err != nil {
-			return false, err
-		}
-		resp, err := o.fs.client.Do(req)
-		if err != nil {
-			return shouldRetry(err), err
-		}
-		defer func() {
-			if err := resp.Body.Close(); err != nil {
-				fs.Logf(nil, "Failed to close response body: %v", err)
-			}
-		}()
-
-		if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
-			return false, err
-		}
-		return shouldRetryHTTP(resp.StatusCode), nil
-	})
-	if err != nil {
-		return "", err
-	}
-	if result.Status != 200 || len(result.Result) == 0 {
-		return "", fmt.Errorf("error: unable to fetch hash: %s", result.Msg)
-	}
-
-	return result.Result[0].Hash, nil
-}
-
-// String returns a string representation of the object
-func (o *Object) String() string {
-	return o.remote
-}
-
-// Fs returns the parent Fs
-func (o *Object) Fs() fs.Info {
-	return o.fs
-}
-
-// Remote returns the remote path
-func (o *Object) Remote() string {
-	return o.remote
-}
-
-// Size returns the size of the object
-func (o *Object) Size() int64 {
-	return o.size
-}
-
-// ModTime returns the modification time of the object
-func (o *Object) ModTime(ctx context.Context) time.Time {
-	return o.modTime
-}
-
-// SetModTime sets the modification time of the object
-func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	return fs.ErrorCantSetModTime
-}
-
-// Storable indicates whether the object is storable
-func (o *Object) Storable() bool {
-	return true
-}

backend/filelu/filelu_test.go

@@ -1,16 +0,0 @@
-package filelu_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests for the FileLu backend
-func TestIntegration(t *testing.T) {
-	fstests.Run(t, &fstests.Opt{
-		RemoteName:      "TestFileLu:",
-		NilObject:       nil,
-		SkipInvalidUTF8: true,
-	})
-}

backend/filelu/utils.go

@@ -1,15 +0,0 @@
-package filelu
-
-import (
-	"fmt"
-)
-
-// parseStorageToBytes converts a storage string (e.g., "10") to bytes
-func parseStorageToBytes(storage string) (int64, error) {
-	var gb float64
-	_, err := fmt.Sscanf(storage, "%f", &gb)
-	if err != nil {
-		return 0, fmt.Errorf("failed to parse storage: %w", err)
-	}
-	return int64(gb * 1024 * 1024 * 1024), nil
-}

backend/ftp/ftp.go

@@ -9,7 +9,6 @@ import (
 	"io"
 	"net"
 	"net/textproto"
-	"net/url"
 	"path"
 	"runtime"
 	"strings"
@@ -186,14 +185,6 @@ Supports the format user:pass@host:port, user@host:port, host:port.
 Example:
 		
     myUser:myPass@localhost:9005
-`,
-			Advanced: true,
-		}, {
-			Name:    "http_proxy",
-			Default: "",
-			Help: `URL for HTTP CONNECT proxy
-
-Set this to a URL for an HTTP proxy which supports the HTTP CONNECT verb.
 `,
 			Advanced: true,
 		}, {
@@ -257,7 +248,6 @@ type Options struct {
 	AskPassword       bool                 `config:"ask_password"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 	SocksProxy        string               `config:"socks_proxy"`
-	HTTPProxy         string               `config:"http_proxy"`
 	NoCheckUpload     bool                 `config:"no_check_upload"`
 }
 
@@ -276,7 +266,6 @@ type Fs struct {
 	pool     []*ftp.ServerConn
 	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
-	proxyURL *url.URL  // address of HTTP proxy read from environment
 	pacer    *fs.Pacer // pacer for FTP connections
 	fGetTime bool      // true if the ftp library accepts GetTime
 	fSetTime bool      // true if the ftp library accepts SetTime
@@ -424,26 +413,11 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	dial := func(network, address string) (conn net.Conn, err error) {
 		fs.Debugf(f, "dial(%q,%q)", network, address)
 		defer func() {
-			if err != nil {
-				fs.Debugf(f, "> dial: conn=%v, err=%v", conn, err)
-			} else {
-				fs.Debugf(f, "> dial: conn=%s->%s, err=%v", conn.LocalAddr(), conn.RemoteAddr(), err)
-			}
+			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
 		}()
 		baseDialer := fshttp.NewDialer(ctx)
 		if f.opt.SocksProxy != "" {
 			conn, err = proxy.SOCKS5Dial(network, address, f.opt.SocksProxy, baseDialer)
-		} else if f.proxyURL != nil {
-			// We need to make the onward connection to f.opt.Host. However the FTP
-			// library sets the host to the proxy IP after using EPSV or PASV so we need
-			// to correct that here.
-			var dialPort string
-			_, dialPort, err = net.SplitHostPort(address)
-			if err != nil {
-				return nil, err
-			}
-			dialAddress := net.JoinHostPort(f.opt.Host, dialPort)
-			conn, err = proxy.HTTPConnectDial(network, dialAddress, f.proxyURL, baseDialer)
 		} else {
 			conn, err = baseDialer.Dial(network, address)
 		}
@@ -657,14 +631,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		CanHaveEmptyDirectories: true,
 		PartialUploads:          true,
 	}).Fill(ctx, f)
-	// get proxy URL if set
-	if opt.HTTPProxy != "" {
-		proxyURL, err := url.Parse(opt.HTTPProxy)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse HTTP Proxy URL: %w", err)
-		}
-		f.proxyURL = proxyURL
-	}
 	// set the pool drainer timer going
 	if f.opt.IdleTimeout > 0 {
 		f.drain = time.AfterFunc(time.Duration(opt.IdleTimeout), func() { _ = f.drainPool(ctx) })

backend/ftp/ftp_internal_test.go

@@ -52,7 +52,7 @@ func (f *Fs) testUploadTimeout(t *testing.T) {
 		ci.Timeout = saveTimeout
 	}()
 	ci.LowLevelRetries = 1
-	ci.Timeout = fs.Duration(idleTimeout)
+	ci.Timeout = idleTimeout
 
 	upload := func(concurrency int, shutTimeout time.Duration) (obj fs.Object, err error) {
 		fixFs := deriveFs(ctx, t, f, settings{

backend/gofile/api/types.go

@@ -194,9 +194,33 @@ type DeleteResponse struct {
 	Data map[string]Error
 }
 
-// DirectUploadURL returns the direct upload URL for Gofile
-func DirectUploadURL() string {
-	return "https://upload.gofile.io/uploadfile"
+// Server is an upload server
+type Server struct {
+	Name string `json:"name"`
+	Zone string `json:"zone"`
+}
+
+// String returns a string representation of the Server
+func (s *Server) String() string {
+	return fmt.Sprintf("%s (%s)", s.Name, s.Zone)
+}
+
+// Root returns the root URL for the server
+func (s *Server) Root() string {
+	return fmt.Sprintf("https://%s.gofile.io/", s.Name)
+}
+
+// URL returns the upload URL for the server
+func (s *Server) URL() string {
+	return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)
+}
+
+// ServersResponse is the output from /servers
+type ServersResponse struct {
+	Error
+	Data struct {
+		Servers []Server `json:"servers"`
+	} `json:"data"`
 }
 
 // UploadResponse is returned by POST /contents/uploadfile

backend/gofile/gofile.go

@@ -8,11 +8,13 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/rclone/rclone/backend/gofile/api"
@@ -35,8 +37,10 @@ const (
 	maxSleep       = 20 * time.Second
 	decayConstant  = 1 // bigger for slower decay, exponential
 	rootURL        = "https://api.gofile.io"
-	rateLimitSleep = 5 * time.Second // penalise a goroutine by this long for making a rate limit error
-	maxDepth       = 4               // in ListR recursive list this deep (maximum is 16)
+	serversExpiry  = 60 * time.Second // check for new upload servers this often
+	serversActive  = 2                // choose this many closest upload servers to use
+	rateLimitSleep = 5 * time.Second  // penalise a goroutine by this long for making a rate limit error
+	maxDepth       = 4                // in ListR recursive list this deep (maximum is 16)
 )
 
 /*
@@ -124,13 +128,16 @@ type Options struct {
 
 // Fs represents a remote gofile
 type Fs struct {
-	name     string             // name of this remote
-	root     string             // the path we are working on
-	opt      Options            // parsed options
-	features *fs.Features       // optional features
-	srv      *rest.Client       // the connection to the server
-	dirCache *dircache.DirCache // Map of directory path to directory id
-	pacer    *fs.Pacer          // pacer for API calls
+	name           string             // name of this remote
+	root           string             // the path we are working on
+	opt            Options            // parsed options
+	features       *fs.Features       // optional features
+	srv            *rest.Client       // the connection to the server
+	dirCache       *dircache.DirCache // Map of directory path to directory id
+	pacer          *fs.Pacer          // pacer for API calls
+	serversMu      *sync.Mutex        // protect the servers info below
+	servers        []api.Server       // upload servers we can use
+	serversChecked time.Time          // time the servers were refreshed
 }
 
 // Object describes a gofile object
@@ -304,11 +311,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	client := fshttp.NewClient(ctx)
 
 	f := &Fs{
-		name:  name,
-		root:  root,
-		opt:   *opt,
-		srv:   rest.NewClient(client).SetRoot(rootURL),
-		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		name:      name,
+		root:      root,
+		opt:       *opt,
+		srv:       rest.NewClient(client).SetRoot(rootURL),
+		pacer:     fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		serversMu: new(sync.Mutex),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:          false,
@@ -427,6 +435,98 @@ func (f *Fs) readRootFolderID(ctx context.Context, m configmap.Mapper) (err erro
 	return nil
 }
 
+// Find the top n servers measured by response time
+func (f *Fs) bestServers(ctx context.Context, servers []api.Server, n int) (newServers []api.Server) {
+	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second))
+	defer cancel()
+
+	if n > len(servers) {
+		n = len(servers)
+	}
+	results := make(chan int, len(servers))
+
+	// Test how long the servers take to respond
+	for i := range servers {
+		i := i // for closure
+		go func() {
+			opts := rest.Opts{
+				Method:  "GET",
+				RootURL: servers[i].Root(),
+			}
+			var result api.UploadServerStatus
+			start := time.Now()
+			_, err := f.srv.CallJSON(ctx, &opts, nil, &result)
+			ping := time.Since(start)
+			err = result.Err(err)
+			if err != nil {
+				results <- -1 // send a -ve number on error
+				return
+			}
+			fs.Debugf(nil, "Upload server %v responded in %v", &servers[i], ping)
+			results <- i
+		}()
+	}
+
+	// Wait for n servers to respond
+	newServers = make([]api.Server, 0, n)
+	for range servers {
+		i := <-results
+		if i >= 0 {
+			newServers = append(newServers, servers[i])
+		}
+		if len(newServers) >= n {
+			break
+		}
+	}
+	return newServers
+}
+
+// Clear all the upload servers - call on an error
+func (f *Fs) clearServers() {
+	f.serversMu.Lock()
+	defer f.serversMu.Unlock()
+
+	fs.Debugf(f, "Clearing upload servers")
+	f.servers = nil
+}
+
+// Gets an upload server
+func (f *Fs) getServer(ctx context.Context) (server *api.Server, err error) {
+	f.serversMu.Lock()
+	defer f.serversMu.Unlock()
+
+	if len(f.servers) == 0 || time.Since(f.serversChecked) >= serversExpiry {
+		opts := rest.Opts{
+			Method: "GET",
+			Path:   "/servers",
+		}
+		var result api.ServersResponse
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err = result.Err(err); err != nil {
+			if len(f.servers) == 0 {
+				return nil, fmt.Errorf("failed to read upload servers: %w", err)
+			}
+			fs.Errorf(f, "failed to read new upload servers: %v", err)
+		} else {
+			// Find the top servers measured by response time
+			f.servers = f.bestServers(ctx, result.Data.Servers, serversActive)
+			f.serversChecked = time.Now()
+		}
+	}
+
+	if len(f.servers) == 0 {
+		return nil, errors.New("no upload servers found")
+	}
+
+	// Pick a server at random since we've already found the top ones
+	i := rand.Intn(len(f.servers))
+	return &f.servers[i], nil
+}
+
 // rootSlash returns root with a slash on if it is empty, otherwise empty string
 func (f *Fs) rootSlash() string {
 	if f.root == "" {
@@ -1426,6 +1526,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
+	// Find an upload server
+	server, err := o.fs.getServer(ctx)
+	if err != nil {
+		return err
+	}
+	fs.Debugf(o, "Using upload server %v", server)
+
 	// If the file exists, delete it after a successful upload
 	if o.id != "" {
 		id := o.id
@@ -1454,7 +1561,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		},
 		MultipartContentName: "file",
 		MultipartFileName:    o.fs.opt.Enc.FromStandardName(leaf),
-		RootURL:              api.DirectUploadURL(),
+		RootURL:              server.URL(),
 		Options:              options,
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1462,6 +1569,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return shouldRetry(ctx, resp, err)
 	})
 	if err = result.Err(err); err != nil {
+		if isAPIErr(err, "error-freespace") {
+			fs.Errorf(o, "Upload server out of space - need to retry upload")
+		}
+		o.fs.clearServers()
 		return fmt.Errorf("failed to upload file: %w", err)
 	}
 	return o.setMetaData(&result.Data)

backend/googlecloudstorage/googlecloudstorage.go

@@ -483,9 +483,6 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
 	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
-	if f.opt.DirectoryMarkers && strings.HasSuffix(bucketPath, "//") {
-		bucketPath = bucketPath[:len(bucketPath)-1]
-	}
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -715,7 +712,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 					continue
 				}
 				// process directory markers as directories
-				remote, _ = strings.CutSuffix(remote, "/")
+				remote = strings.TrimRight(remote, "/")
 			}
 			remote = remote[len(prefix):]
 			if addBucket {
@@ -962,7 +959,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote, _ = strings.CutSuffix(remote, "/")
+	remote = strings.TrimRight(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
 		dir = ""

backend/googlephotos/googlephotos.go

@@ -43,7 +43,6 @@ var (
 	errAlbumDelete = errors.New("google photos API does not implement deleting albums")
 	errRemove      = errors.New("google photos API only implements removing files from albums")
 	errOwnAlbums   = errors.New("google photos API only allows uploading to albums rclone created")
-	errReadOnly    = errors.New("can't upload files in read only mode")
 )
 
 const (
@@ -53,31 +52,19 @@ const (
 	listChunks                  = 100 // chunk size to read directory listings
 	albumChunks                 = 50  // chunk size to read album listings
 	minSleep                    = 10 * time.Millisecond
-	scopeAppendOnly             = "https://www.googleapis.com/auth/photoslibrary.appendonly"
-	scopeReadOnly               = "https://www.googleapis.com/auth/photoslibrary.readonly.appcreateddata"
-	scopeReadWrite              = "https://www.googleapis.com/auth/photoslibrary.edit.appcreateddata"
+	scopeReadOnly               = "https://www.googleapis.com/auth/photoslibrary.readonly"
+	scopeReadWrite              = "https://www.googleapis.com/auth/photoslibrary"
+	scopeAccess                 = 2 // position of access scope in list
 )
 
 var (
-	// scopes needed for read write access
-	scopesReadWrite = []string{
-		"openid",
-		"profile",
-		scopeAppendOnly,
-		scopeReadOnly,
-		scopeReadWrite,
-	}
-
-	// scopes needed for read only access
-	scopesReadOnly = []string{
-		"openid",
-		"profile",
-		scopeReadOnly,
-	}
-
 	// Description of how to auth for this app
 	oauthConfig = &oauthutil.Config{
-		Scopes:       scopesReadWrite,
+		Scopes: []string{
+			"openid",
+			"profile",
+			scopeReadWrite, // this must be at position scopeAccess
+		},
 		AuthURL:      google.Endpoint.AuthURL,
 		TokenURL:     google.Endpoint.TokenURL,
 		ClientID:     rcloneClientID,
@@ -113,9 +100,9 @@ func init() {
 			case "":
 				// Fill in the scopes
 				if opt.ReadOnly {
-					oauthConfig.Scopes = scopesReadOnly
+					oauthConfig.Scopes[scopeAccess] = scopeReadOnly
 				} else {
-					oauthConfig.Scopes = scopesReadWrite
+					oauthConfig.Scopes[scopeAccess] = scopeReadWrite
 				}
 				return oauthutil.ConfigOut("warning", &oauthutil.Options{
 					OAuth2Config: oauthConfig,
@@ -346,7 +333,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	baseClient := fshttp.NewClient(ctx)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient)
 	if err != nil {
-		return nil, fmt.Errorf("failed to configure google photos: %w", err)
+		return nil, fmt.Errorf("failed to configure Box: %w", err)
 	}
 
 	root = strings.Trim(path.Clean(root), "/")
@@ -1133,9 +1120,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 
 		if !album.IsWriteable {
-			if o.fs.opt.ReadOnly {
-				return errReadOnly
-			}
 			return errOwnAlbums
 		}
 

backend/iclouddrive/api/drive.go

@@ -252,14 +252,18 @@ func (d *DriveService) DownloadFile(ctx context.Context, url string, opt []fs.Op
 	}
 
 	resp, err := d.icloud.srv.Call(ctx, opts)
-	// icloud has some weird http codes
-	if err != nil && resp != nil && resp.StatusCode == 330 {
-		loc, err := resp.Location()
-		if err == nil {
-			return d.DownloadFile(ctx, loc.String(), opt)
+	if err != nil {
+		// icloud has some weird http codes
+		if resp.StatusCode == 330 {
+			loc, err := resp.Location()
+			if err == nil {
+				return d.DownloadFile(ctx, loc.String(), opt)
+			}
 		}
+
+		return resp, err
 	}
-	return resp, err
+	return d.icloud.srv.Call(ctx, opts)
 }
 
 // MoveItemToTrashByItemID moves an item to the trash based on the item ID.

backend/imagekit/imagekit.go

@@ -421,9 +421,6 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // will return the object and the error, otherwise will return
 // nil and the error
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if src.Size() == 0 {
-		return nil, fs.ErrorCantUploadEmptyFiles
-	}
 	return uploadFile(ctx, f, in, src.Remote(), options...)
 }
 
@@ -662,9 +659,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-	if src.Size() == 0 {
-		return fs.ErrorCantUploadEmptyFiles
-	}
 
 	srcRemote := o.Remote()
 
@@ -676,7 +670,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	var resp *client.UploadResult
 
-	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+	err = o.fs.pacer.Call(func() (bool, error) {
 		var res *http.Response
 		res, resp, err = o.fs.ik.Upload(ctx, in, client.UploadParam{
 			FileName:      fileName,
@@ -731,7 +725,7 @@ func uploadFile(ctx context.Context, f *Fs, in io.Reader, srcRemote string, opti
 	UseUniqueFileName := new(bool)
 	*UseUniqueFileName = false
 
-	err := f.pacer.CallNoRetry(func() (bool, error) {
+	err := f.pacer.Call(func() (bool, error) {
 		var res *http.Response
 		var err error
 		res, _, err = f.ik.Upload(ctx, in, client.UploadParam{
@@ -800,10 +794,35 @@ func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error)
 	return metadata, nil
 }
 
+// Copy src to this remote using server-side move operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		return nil, fs.ErrorCantMove
+	}
+
+	file, err := srcObj.Open(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return uploadFile(ctx, f, file, remote)
+}
+
 // Check the interfaces are satisfied.
 var (
 	_ fs.Fs           = &Fs{}
 	_ fs.Purger       = &Fs{}
 	_ fs.PublicLinker = &Fs{}
 	_ fs.Object       = &Object{}
+	_ fs.Copier       = &Fs{}
 )

backend/koofr/koofr.go

@@ -461,7 +461,7 @@ func translateErrorsDir(err error) error {
 	return err
 }
 
-// translateErrorsObject translates Koofr errors to rclone errors (for an object operation)
+// translatesErrorsObject translates Koofr errors to rclone errors (for an object operation)
 func translateErrorsObject(err error) error {
 	switch err := err.(type) {
 	case httpclient.InvalidStatusError:

backend/linkbox/linkbox.go

@@ -617,36 +617,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	case 1:
 		// upload file using link from first step
 		var res *http.Response
-		var location string
-
-		// Check to see if we are being redirected
-		opts := &rest.Opts{
-			Method:     "HEAD",
-			RootURL:    getFirstStepResult.Data.SignURL,
-			Options:    options,
-			NoRedirect: true,
-		}
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			res, err = o.fs.srv.Call(ctx, opts)
-			return o.fs.shouldRetry(ctx, res, err)
-		})
-		if res != nil {
-			location = res.Header.Get("Location")
-			if location != "" {
-				// set the URL to the new Location
-				opts.RootURL = location
-				err = nil
-			}
-		}
-		if err != nil {
-			return fmt.Errorf("head upload URL: %w", err)
-		}
 
 		file := io.MultiReader(bytes.NewReader(first10mBytes), in)
 
-		opts.Method = "PUT"
-		opts.Body = file
-		opts.ContentLength = &size
+		opts := &rest.Opts{
+			Method:        "PUT",
+			RootURL:       getFirstStepResult.Data.SignURL,
+			Options:       options,
+			Body:          file,
+			ContentLength: &size,
+		}
 
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			res, err = o.fs.srv.Call(ctx, opts)

backend/local/local.go

@@ -305,12 +305,6 @@ only useful for reading.
 					Help:  "The last status change time.",
 				}},
 			},
-			{
-				Name:     "hashes",
-				Help:     `Comma separated list of supported checksum types.`,
-				Default:  fs.CommaSepList{},
-				Advanced: true,
-			},
 			{
 				Name:     config.ConfigEncoding,
 				Help:     config.ConfigEncodingHelp,
@@ -337,7 +331,6 @@ type Options struct {
 	NoSparse          bool                 `config:"no_sparse"`
 	NoSetModTime      bool                 `config:"no_set_modtime"`
 	TimeType          timeType             `config:"time_type"`
-	Hashes            fs.CommaSepList      `config:"hashes"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 	NoClone           bool                 `config:"no_clone"`
 }
@@ -1028,19 +1021,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	if len(f.opt.Hashes) > 0 {
-		// Return only configured hashes.
-		// Note: Could have used hash.SupportOnly to limit supported hashes for all hash related features.
-		var supported hash.Set
-		for _, hashName := range f.opt.Hashes {
-			var ht hash.Type
-			if err := ht.Set(hashName); err != nil {
-				fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
-			}
-			supported.Add(ht)
-		}
-		return supported
-	}
 	return hash.Supported()
 }
 
@@ -1110,10 +1090,6 @@ func (o *Object) Remote() string {
 
 // Hash returns the requested hash of a file as a lowercase hex string
 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
-	if r == hash.None {
-		return "", nil
-	}
-
 	// Check that the underlying file hasn't changed
 	o.fs.objectMetaMu.RLock()
 	oldtime := o.modTime
@@ -1221,15 +1197,7 @@ func (o *Object) Storable() bool {
 	o.fs.objectMetaMu.RLock()
 	mode := o.mode
 	o.fs.objectMetaMu.RUnlock()
-
-	// On Windows items with os.ModeIrregular are likely Junction
-	// points so we treat them as symlinks for the purpose of ignoring them.
-	// https://github.com/golang/go/issues/73827
-	symlinkFlag := os.ModeSymlink
-	if runtime.GOOS == "windows" {
-		symlinkFlag |= os.ModeIrregular
-	}
-	if mode&symlinkFlag != 0 && !o.fs.opt.TranslateSymlinks {
+	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}

backend/local/local_internal_test.go

@@ -204,23 +204,6 @@ func TestSymlinkError(t *testing.T) {
 	assert.Equal(t, errLinksAndCopyLinks, err)
 }
 
-func TestHashWithTypeNone(t *testing.T) {
-	ctx := context.Background()
-	r := fstest.NewRun(t)
-	const filePath = "file.txt"
-	r.WriteFile(filePath, "content", time.Now())
-	f := r.Flocal.(*Fs)
-
-	// Get the object
-	o, err := f.NewObject(ctx, filePath)
-	require.NoError(t, err)
-
-	// Test the hash is as we expect
-	h, err := o.Hash(ctx, hash.None)
-	require.Empty(t, h)
-	require.NoError(t, err)
-}
-
 // Test hashes on updating an object
 func TestHashOnUpdate(t *testing.T) {
 	ctx := context.Background()

backend/mailru/mailru.go

@@ -634,7 +634,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 	return
 }
 
-// itemToDirEntry converts API item to rclone directory entry
+// itemToEntry converts API item to rclone directory entry
 // The dirSize return value is:
 //
 //	<0 - for a file or in case of error

backend/mega/mega.go

@@ -17,11 +17,9 @@ Improvements:
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"path"
 	"slices"
 	"strings"
@@ -218,25 +216,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	defer megaCacheMu.Unlock()
 	srv := megaCache[opt.User]
 	if srv == nil {
-		// srv = mega.New().SetClient(fshttp.NewClient(ctx))
-
-		// Workaround for Mega's use of insecure cipher suites which are no longer supported by default since Go 1.22.
-		// Relevant issues:
-		// https://github.com/rclone/rclone/issues/8565
-		// https://github.com/meganz/webclient/issues/103
-		clt := fshttp.NewClient(ctx)
-		clt.Transport = fshttp.NewTransportCustom(ctx, func(t *http.Transport) {
-			var ids []uint16
-			// Read default ciphers
-			for _, cs := range tls.CipherSuites() {
-				ids = append(ids, cs.ID)
-			}
-			// Insecure but Mega uses TLS_RSA_WITH_AES_128_GCM_SHA256 for storage endpoints
-			// (e.g. https://gfs302n114.userstorage.mega.co.nz) as of June 18, 2025.
-			t.TLSClientConfig.CipherSuites = append(ids, tls.TLS_RSA_WITH_AES_128_GCM_SHA256)
-		})
-		srv = mega.New().SetClient(clt)
-
+		srv = mega.New().SetClient(fshttp.NewClient(ctx))
 		srv.SetRetries(ci.LowLevelRetries) // let mega do the low level retries
 		srv.SetHTTPS(opt.UseHTTPS)
 		srv.SetLogger(func(format string, v ...any) {

backend/onedrive/metadata.go

@@ -749,8 +749,6 @@ func (o *Object) fetchMetadataForCreate(ctx context.Context, src fs.ObjectInfo,
 
 // Fetch metadata and update updateInfo if --metadata is in use
 // modtime will still be set when there is no metadata to set
-//
-// May return info=nil and err=nil if there was no metadata to update.
 func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *Object) (info *api.Item, err error) {
 	meta, err := fs.GetMetadataOptions(ctx, f, src, options)
 	if err != nil {
@@ -770,8 +768,6 @@ func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, opti
 }
 
 // updateMetadata calls Get, Set, and Write
-//
-// May return info=nil and err=nil if there was no metadata to update.
 func (o *Object) updateMetadata(ctx context.Context, meta fs.Metadata) (info *api.Item, err error) {
 	_, err = o.meta.Get(ctx) // refresh permissions
 	if err != nil {

backend/onedrive/onedrive.go

@@ -56,7 +56,6 @@ const (
 	driveTypeSharepoint         = "documentLibrary"
 	defaultChunkSize            = 10 * fs.Mebi
 	chunkSizeMultiple           = 320 * fs.Kibi
-	maxSinglePartSize           = 4 * fs.Mebi
 
 	regionGlobal = "global"
 	regionUS     = "us"
@@ -139,21 +138,6 @@ func init() {
 					Help:  "Azure and Office 365 operated by Vnet Group in China",
 				},
 			},
-		}, {
-			Name: "upload_cutoff",
-			Help: `Cutoff for switching to chunked upload.
-
-Any files larger than this will be uploaded in chunks of chunk_size.
-
-This is disabled by default as uploading using single part uploads
-causes rclone to use twice the storage on Onedrive business as when
-rclone sets the modification time after the upload Onedrive creates a
-new version.
-
-See: https://github.com/rclone/rclone/issues/1716
-`,
-			Default:  fs.SizeSuffix(-1),
-			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
@@ -762,7 +746,6 @@ Examples:
 // Options defines the configuration for this backend
 type Options struct {
 	Region                  string               `config:"region"`
-	UploadCutoff            fs.SizeSuffix        `config:"upload_cutoff"`
 	ChunkSize               fs.SizeSuffix        `config:"chunk_size"`
 	DriveID                 string               `config:"drive_id"`
 	DriveType               string               `config:"drive_type"`
@@ -1039,13 +1022,6 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-func checkUploadCutoff(cs fs.SizeSuffix) error {
-	if cs > maxSinglePartSize {
-		return fmt.Errorf("%v is greater than %v", cs, maxSinglePartSize)
-	}
-	return nil
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -1059,10 +1035,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, fmt.Errorf("onedrive: chunk size: %w", err)
 	}
-	err = checkUploadCutoff(opt.UploadCutoff)
-	if err != nil {
-		return nil, fmt.Errorf("onedrive: upload cutoff: %w", err)
-	}
 
 	if opt.DriveID == "" || opt.DriveType == "" {
 		return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
@@ -1782,9 +1754,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 	if err != nil {
 		return nil, err
 	}
-	if info != nil {
-		err = dstObj.setMetaData(info)
-	}
+	err = dstObj.setMetaData(info)
 	return dstObj, err
 }
 
@@ -1864,9 +1834,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	if info != nil {
-		err = dstObj.setMetaData(info)
-	}
+	err = dstObj.setMetaData(info)
 	return dstObj, err
 }
 
@@ -2501,10 +2469,6 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 				return false, nil
 			}
 			return true, fmt.Errorf("retry this chunk skipping %d bytes: %w", skip, err)
-		} else if err != nil && resp != nil && resp.StatusCode == http.StatusNotFound {
-			fs.Debugf(o, "Received 404 error: assuming eventual consistency problem with session - retrying chunk: %v", err)
-			time.Sleep(5 * time.Second) // a little delay to help things along
-			return true, err
 		}
 		if err != nil {
 			return shouldRetry(ctx, resp, err)
@@ -2599,8 +2563,8 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objec
 // This function will set modtime and metadata after uploading, which will create a new version for the remote file
 func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (info *api.Item, err error) {
 	size := src.Size()
-	if size < 0 || size > int64(maxSinglePartSize) {
-		return nil, fmt.Errorf("size passed into uploadSinglepart must be >= 0 and <= %v", maxSinglePartSize)
+	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
+		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4 MiB")
 	}
 
 	fs.Debugf(o, "Starting singlepart upload")
@@ -2633,10 +2597,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, src fs.Obje
 	if err != nil {
 		return nil, fmt.Errorf("failed to fetch and update metadata: %w", err)
 	}
-	if info != nil {
-		err = o.setMetaData(info)
-	}
-	return info, err
+	return info, o.setMetaData(info)
 }
 
 // Update the object with the contents of the io.Reader, modTime and size
@@ -2656,9 +2617,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	size := src.Size()
 
 	var info *api.Item
-	if size > 0 && size >= int64(o.fs.opt.UploadCutoff) {
+	if size > 0 {
 		info, err = o.uploadMultipart(ctx, in, src, options...)
-	} else if size >= 0 {
+	} else if size == 0 {
 		info, err = o.uploadSinglepart(ctx, in, src, options...)
 	} else {
 		return errors.New("unknown-sized upload not supported")

backend/pcloud/pcloud.go

@@ -378,20 +378,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
-// XOpenWriterAt opens with a handle for random access writes
+// OpenWriterAt opens with a handle for random access writes
 //
 // Pass in the remote desired and the size if known.
 //
-// It truncates any existing object.
-//
-// OpenWriterAt disabled because it seems to have been disabled at pcloud
-// PUT /file_open?flags=XXX&folderid=XXX&name=XXX HTTP/1.1
-//
-//	{
-//	        "result": 2003,
-//	        "error": "Access denied. You do not have permissions to perform this operation."
-//	}
-func (f *Fs) XOpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
 	client, err := f.newSingleConnClient(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("create client: %w", err)

backend/pikpak/helper.go

@@ -155,7 +155,6 @@ func (f *Fs) getFile(ctx context.Context, ID string) (info *api.File, err error)
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.rst.CallJSON(ctx, &opts, nil, &info)
 		if err == nil && !info.Links.ApplicationOctetStream.Valid() {
-			time.Sleep(5 * time.Second)
 			return true, errors.New("no link")
 		}
 		return f.shouldRetry(ctx, resp, err)

backend/pikpak/multipart.go

@@ -1,333 +0,0 @@
-package pikpak
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/aws/aws-sdk-go-v2/aws"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
-	"github.com/rclone/rclone/backend/pikpak/api"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/chunksize"
-	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/atexit"
-	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/pool"
-	"golang.org/x/sync/errgroup"
-)
-
-const (
-	bufferSize           = 1024 * 1024     // default size of the pages used in the reader
-	bufferCacheSize      = 64              // max number of buffers to keep in cache
-	bufferCacheFlushTime = 5 * time.Second // flush the cached buffers after this long
-)
-
-// bufferPool is a global pool of buffers
-var (
-	bufferPool     *pool.Pool
-	bufferPoolOnce sync.Once
-)
-
-// get a buffer pool
-func getPool() *pool.Pool {
-	bufferPoolOnce.Do(func() {
-		ci := fs.GetConfig(context.Background())
-		// Initialise the buffer pool when used
-		bufferPool = pool.New(bufferCacheFlushTime, bufferSize, bufferCacheSize, ci.UseMmap)
-	})
-	return bufferPool
-}
-
-// NewRW gets a pool.RW using the multipart pool
-func NewRW() *pool.RW {
-	return pool.NewRW(getPool())
-}
-
-// Upload does a multipart upload in parallel
-func (w *pikpakChunkWriter) Upload(ctx context.Context) (err error) {
-	// make concurrency machinery
-	tokens := pacer.NewTokenDispenser(w.con)
-
-	uploadCtx, cancel := context.WithCancel(ctx)
-	defer cancel()
-	defer atexit.OnError(&err, func() {
-		cancel()
-		fs.Debugf(w.o, "multipart upload: Cancelling...")
-		errCancel := w.Abort(ctx)
-		if errCancel != nil {
-			fs.Debugf(w.o, "multipart upload: failed to cancel: %v", errCancel)
-		}
-	})()
-
-	var (
-		g, gCtx   = errgroup.WithContext(uploadCtx)
-		finished  = false
-		off       int64
-		size      = w.size
-		chunkSize = w.chunkSize
-	)
-
-	// Do the accounting manually
-	in, acc := accounting.UnWrapAccounting(w.in)
-
-	for partNum := int64(0); !finished; partNum++ {
-		// Get a block of memory from the pool and token which limits concurrency.
-		tokens.Get()
-		rw := NewRW()
-		if acc != nil {
-			rw.SetAccounting(acc.AccountRead)
-		}
-
-		free := func() {
-			// return the memory and token
-			_ = rw.Close() // Can't return an error
-			tokens.Put()
-		}
-
-		// Fail fast, in case an errgroup managed function returns an error
-		// gCtx is cancelled. There is no point in uploading all the other parts.
-		if gCtx.Err() != nil {
-			free()
-			break
-		}
-
-		// Read the chunk
-		var n int64
-		n, err = io.CopyN(rw, in, chunkSize)
-		if err == io.EOF {
-			if n == 0 && partNum != 0 { // end if no data and if not first chunk
-				free()
-				break
-			}
-			finished = true
-		} else if err != nil {
-			free()
-			return fmt.Errorf("multipart upload: failed to read source: %w", err)
-		}
-
-		partNum := partNum
-		partOff := off
-		off += n
-		g.Go(func() (err error) {
-			defer free()
-			fs.Debugf(w.o, "multipart upload: starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(partOff), fs.SizeSuffix(size))
-			_, err = w.WriteChunk(gCtx, int32(partNum), rw)
-			return err
-		})
-	}
-
-	err = g.Wait()
-	if err != nil {
-		return err
-	}
-
-	err = w.Close(ctx)
-	if err != nil {
-		return fmt.Errorf("multipart upload: failed to finalise: %w", err)
-	}
-
-	return nil
-}
-
-var warnStreamUpload sync.Once
-
-// state of ChunkWriter
-type pikpakChunkWriter struct {
-	chunkSize      int64
-	size           int64
-	con            int
-	f              *Fs
-	o              *Object
-	in             io.Reader
-	mu             sync.Mutex
-	completedParts []types.CompletedPart
-	client         *s3.Client
-	mOut           *s3.CreateMultipartUploadOutput
-}
-
-func (f *Fs) newChunkWriter(ctx context.Context, remote string, size int64, p *api.ResumableParams, in io.Reader, options ...fs.OpenOption) (w *pikpakChunkWriter, err error) {
-	// Temporary Object under construction
-	o := &Object{
-		fs:     f,
-		remote: remote,
-	}
-
-	// calculate size of parts
-	chunkSize := f.opt.ChunkSize
-
-	// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
-	// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
-	// 48 GiB which seems like a not too unreasonable limit.
-	if size == -1 {
-		warnStreamUpload.Do(func() {
-			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
-				f.opt.ChunkSize, fs.SizeSuffix(int64(chunkSize)*int64(maxUploadParts)))
-		})
-	} else {
-		chunkSize = chunksize.Calculator(o, size, maxUploadParts, chunkSize)
-	}
-
-	client, err := f.newS3Client(ctx, p)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create upload client: %w", err)
-	}
-	w = &pikpakChunkWriter{
-		chunkSize:      int64(chunkSize),
-		size:           size,
-		con:            max(1, f.opt.UploadConcurrency),
-		f:              f,
-		o:              o,
-		in:             in,
-		completedParts: make([]types.CompletedPart, 0),
-		client:         client,
-	}
-
-	req := &s3.CreateMultipartUploadInput{
-		Bucket: &p.Bucket,
-		Key:    &p.Key,
-	}
-	// Apply upload options
-	for _, option := range options {
-		key, value := option.Header()
-		lowerKey := strings.ToLower(key)
-		switch lowerKey {
-		case "":
-			// ignore
-		case "cache-control":
-			req.CacheControl = aws.String(value)
-		case "content-disposition":
-			req.ContentDisposition = aws.String(value)
-		case "content-encoding":
-			req.ContentEncoding = aws.String(value)
-		case "content-type":
-			req.ContentType = aws.String(value)
-		}
-	}
-	err = w.f.pacer.Call(func() (bool, error) {
-		w.mOut, err = w.client.CreateMultipartUpload(ctx, req)
-		return w.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return nil, fmt.Errorf("create multipart upload failed: %w", err)
-	}
-	fs.Debugf(w.o, "multipart upload: %q initiated", *w.mOut.UploadId)
-	return
-}
-
-// shouldRetry returns a boolean as to whether this err
-// deserve to be retried. It returns the err as a convenience
-func (w *pikpakChunkWriter) shouldRetry(ctx context.Context, err error) (bool, error) {
-	if fserrors.ContextError(ctx, &err) {
-		return false, err
-	}
-	if fserrors.ShouldRetry(err) {
-		return true, err
-	}
-	return false, err
-}
-
-// add a part number and etag to the completed parts
-func (w *pikpakChunkWriter) addCompletedPart(part types.CompletedPart) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	w.completedParts = append(w.completedParts, part)
-}
-
-// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
-func (w *pikpakChunkWriter) WriteChunk(ctx context.Context, chunkNumber int32, reader io.ReadSeeker) (currentChunkSize int64, err error) {
-	if chunkNumber < 0 {
-		err := fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
-		return -1, err
-	}
-
-	partNumber := chunkNumber + 1
-	var res *s3.UploadPartOutput
-	err = w.f.pacer.Call(func() (bool, error) {
-		// Discover the size by seeking to the end
-		currentChunkSize, err = reader.Seek(0, io.SeekEnd)
-		if err != nil {
-			return false, err
-		}
-		// rewind the reader on retry and after reading md5
-		_, err := reader.Seek(0, io.SeekStart)
-		if err != nil {
-			return false, err
-		}
-		res, err = w.client.UploadPart(ctx, &s3.UploadPartInput{
-			Bucket:     w.mOut.Bucket,
-			Key:        w.mOut.Key,
-			UploadId:   w.mOut.UploadId,
-			PartNumber: &partNumber,
-			Body:       reader,
-		})
-		if err != nil {
-			if chunkNumber <= 8 {
-				return w.shouldRetry(ctx, err)
-			}
-			// retry all chunks once have done the first few
-			return true, err
-		}
-		return false, nil
-	})
-	if err != nil {
-		return -1, fmt.Errorf("failed to upload chunk %d with %v bytes: %w", partNumber, currentChunkSize, err)
-	}
-
-	w.addCompletedPart(types.CompletedPart{
-		PartNumber: &partNumber,
-		ETag:       res.ETag,
-	})
-
-	fs.Debugf(w.o, "multipart upload: wrote chunk %d with %v bytes", partNumber, currentChunkSize)
-	return currentChunkSize, err
-}
-
-// Abort the multipart upload
-func (w *pikpakChunkWriter) Abort(ctx context.Context) (err error) {
-	// Abort the upload session
-	err = w.f.pacer.Call(func() (bool, error) {
-		_, err = w.client.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{
-			Bucket:   w.mOut.Bucket,
-			Key:      w.mOut.Key,
-			UploadId: w.mOut.UploadId,
-		})
-		return w.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return fmt.Errorf("failed to abort multipart upload %q: %w", *w.mOut.UploadId, err)
-	}
-	fs.Debugf(w.o, "multipart upload: %q aborted", *w.mOut.UploadId)
-	return
-}
-
-// Close and finalise the multipart upload
-func (w *pikpakChunkWriter) Close(ctx context.Context) (err error) {
-	// sort the completed parts by part number
-	sort.Slice(w.completedParts, func(i, j int) bool {
-		return *w.completedParts[i].PartNumber < *w.completedParts[j].PartNumber
-	})
-	// Finalise the upload session
-	err = w.f.pacer.Call(func() (bool, error) {
-		_, err = w.client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
-			Bucket:   w.mOut.Bucket,
-			Key:      w.mOut.Key,
-			UploadId: w.mOut.UploadId,
-			MultipartUpload: &types.CompletedMultipartUpload{
-				Parts: w.completedParts,
-			},
-		})
-		return w.shouldRetry(ctx, err)
-	})
-	if err != nil {
-		return fmt.Errorf("failed to complete multipart upload: %w", err)
-	}
-	fs.Debugf(w.o, "multipart upload: %q finished", *w.mOut.UploadId)
-	return
-}

backend/pikpak/pikpak.go

@@ -41,10 +41,12 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	awsconfig "github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/rclone/rclone/backend/pikpak/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
@@ -64,22 +66,17 @@ import (
 
 // Constants
 const (
-	clientID         = "YUMx5nI8ZU8Ap8pm"
-	clientVersion    = "2.0.0"
-	packageName      = "mypikpak.com"
-	defaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
-	minSleep         = 100 * time.Millisecond
-	maxSleep         = 2 * time.Second
-	taskWaitTime     = 500 * time.Millisecond
-	decayConstant    = 2 // bigger for slower decay, exponential
-	rootURL          = "https://api-drive.mypikpak.com"
-
-	maxUploadParts      = 10000                          // Part number must be an integer between 1 and 10000, inclusive.
-	defaultChunkSize    = fs.SizeSuffix(1024 * 1024 * 5) // Part size should be in [100KB, 5GB]
-	minChunkSize        = 100 * fs.Kibi
-	maxChunkSize        = 5 * fs.Gibi
-	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
-	maxUploadCutoff     = 5 * fs.Gibi // maximum allowed size for singlepart uploads
+	clientID                 = "YUMx5nI8ZU8Ap8pm"
+	clientVersion            = "2.0.0"
+	packageName              = "mypikpak.com"
+	defaultUserAgent         = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0"
+	minSleep                 = 100 * time.Millisecond
+	maxSleep                 = 2 * time.Second
+	taskWaitTime             = 500 * time.Millisecond
+	decayConstant            = 2 // bigger for slower decay, exponential
+	rootURL                  = "https://api-drive.mypikpak.com"
+	minChunkSize             = fs.SizeSuffix(manager.MinUploadPartSize)
+	defaultUploadConcurrency = manager.DefaultUploadConcurrency
 )
 
 // Globals
@@ -226,14 +223,6 @@ Fill in for rclone to use a non root folder as its starting point.
 			Help:     "Files bigger than this will be cached on disk to calculate hash if required.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
-		}, {
-			Name: "upload_cutoff",
-			Help: `Cutoff for switching to chunked upload.
-
-Any files larger than this will be uploaded in chunks of chunk_size.
-The minimum is 0 and the maximum is 5 GiB.`,
-			Default:  defaultUploadCutoff,
-			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Chunk size for multipart uploads.
@@ -252,7 +241,7 @@ large file of known size to stay below the 10,000 chunks limit.
 
 Increasing the chunk size decreases the accuracy of the progress
 statistics displayed with "-P" flag.`,
-			Default:  defaultChunkSize,
+			Default:  minChunkSize,
 			Advanced: true,
 		}, {
 			Name: "upload_concurrency",
@@ -268,7 +257,7 @@ in memory.
 If you are uploading small numbers of large files over high-speed links
 and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
-			Default:  4,
+			Default:  defaultUploadConcurrency,
 			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
@@ -305,7 +294,6 @@ type Options struct {
 	NoMediaLink         bool                 `config:"no_media_link"`
 	HashMemoryThreshold fs.SizeSuffix        `config:"hash_memory_limit"`
 	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
-	UploadCutoff        fs.SizeSuffix        `config:"upload_cutoff"`
 	UploadConcurrency   int                  `config:"upload_concurrency"`
 	Enc                 encoder.MultiEncoder `config:"encoding"`
 }
@@ -479,11 +467,6 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 			// when a zero-byte file was uploaded with an invalid captcha token
 			f.rst.captcha.Invalidate()
 			return true, err
-		} else if strings.Contains(apiErr.Reason, "idx.shub.mypikpak.com") && apiErr.Code == 500 {
-			// internal server error: Post "http://idx.shub.mypikpak.com": context deadline exceeded (Client.Timeout exceeded while awaiting headers)
-			// This typically happens when trying to retrieve a gcid for which no record exists.
-			// No retry is needed in this case.
-			return false, err
 		}
 	}
 
@@ -541,39 +524,6 @@ func (f *Fs) newClientWithPacer(ctx context.Context) (err error) {
 	return nil
 }
 
-func checkUploadChunkSize(cs fs.SizeSuffix) error {
-	if cs < minChunkSize {
-		return fmt.Errorf("%s is less than %s", cs, minChunkSize)
-	}
-	if cs > maxChunkSize {
-		return fmt.Errorf("%s is greater than %s", cs, maxChunkSize)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadChunkSize(cs)
-	if err == nil {
-		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-	}
-	return
-}
-
-func checkUploadCutoff(cs fs.SizeSuffix) error {
-	if cs > maxUploadCutoff {
-		return fmt.Errorf("%s is greater than %s", cs, maxUploadCutoff)
-	}
-	return nil
-}
-
-func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
-	err = checkUploadCutoff(cs)
-	if err == nil {
-		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
-	}
-	return
-}
-
 // newFs partially constructs Fs from the path
 //
 // It constructs a valid Fs but doesn't attempt to figure out whether
@@ -581,17 +531,11 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
-	err := configstruct.Set(m, opt)
-	if err != nil {
+	if err := configstruct.Set(m, opt); err != nil {
 		return nil, err
 	}
-	err = checkUploadChunkSize(opt.ChunkSize)
-	if err != nil {
-		return nil, fmt.Errorf("pikpak: chunk size: %w", err)
-	}
-	err = checkUploadCutoff(opt.UploadCutoff)
-	if err != nil {
-		return nil, fmt.Errorf("pikpak: upload cutoff: %w", err)
+	if opt.ChunkSize < minChunkSize {
+		return nil, fmt.Errorf("chunk size must be at least %s", minChunkSize)
 	}
 
 	root := parsePath(path)
@@ -1316,7 +1260,9 @@ func (f *Fs) uploadByForm(ctx context.Context, in io.Reader, name string, size i
 	return
 }
 
-func (f *Fs) newS3Client(ctx context.Context, p *api.ResumableParams) (s3Client *s3.Client, err error) {
+func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable) (err error) {
+	p := resumable.Params
+
 	// Create a credentials provider
 	creds := credentials.NewStaticCredentialsProvider(p.AccessKeyID, p.AccessKeySecret, p.SecurityToken)
 
@@ -1326,64 +1272,22 @@ func (f *Fs) newS3Client(ctx context.Context, p *api.ResumableParams) (s3Client
 	if err != nil {
 		return
 	}
-	ci := fs.GetConfig(ctx)
-	cfg.RetryMaxAttempts = ci.LowLevelRetries
-	cfg.HTTPClient = getClient(ctx, &f.opt)
 
 	client := s3.NewFromConfig(cfg, func(o *s3.Options) {
 		o.BaseEndpoint = aws.String("https://mypikpak.com/")
-		o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
-		o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
 	})
-	return client, nil
-}
+	partSize := chunksize.Calculator(name, size, int(manager.MaxUploadParts), f.opt.ChunkSize)
 
-func (f *Fs) uploadByResumable(ctx context.Context, in io.Reader, name string, size int64, resumable *api.Resumable, options ...fs.OpenOption) (err error) {
-	p := resumable.Params
-
-	if size < 0 || size >= int64(f.opt.UploadCutoff) {
-		mu, err := f.newChunkWriter(ctx, name, size, p, in, options...)
-		if err != nil {
-			return fmt.Errorf("multipart upload failed to initialise: %w", err)
-		}
-		return mu.Upload(ctx)
-	}
-
-	// upload singlepart
-	client, err := f.newS3Client(ctx, p)
-	if err != nil {
-		return fmt.Errorf("failed to create upload client: %w", err)
-	}
-	req := &s3.PutObjectInput{
+	// Create an uploader with custom options
+	uploader := manager.NewUploader(client, func(u *manager.Uploader) {
+		u.PartSize = int64(partSize)
+		u.Concurrency = f.opt.UploadConcurrency
+	})
+	// Perform an upload
+	_, err = uploader.Upload(ctx, &s3.PutObjectInput{
 		Bucket: &p.Bucket,
 		Key:    &p.Key,
-		Body:   io.NopCloser(in),
-	}
-	// Apply upload options
-	for _, option := range options {
-		key, value := option.Header()
-		lowerKey := strings.ToLower(key)
-		switch lowerKey {
-		case "":
-			// ignore
-		case "cache-control":
-			req.CacheControl = aws.String(value)
-		case "content-disposition":
-			req.ContentDisposition = aws.String(value)
-		case "content-encoding":
-			req.ContentEncoding = aws.String(value)
-		case "content-type":
-			req.ContentType = aws.String(value)
-		}
-	}
-	var s3opts = []func(*s3.Options){}
-	// Can't retry single part uploads as only have an io.Reader
-	s3opts = append(s3opts, func(o *s3.Options) {
-		o.RetryMaxAttempts = 1
-	})
-	err = f.pacer.CallNoRetry(func() (bool, error) {
-		_, err = client.PutObject(ctx, req, s3opts...)
-		return f.shouldRetry(ctx, nil, err)
+		Body:   in,
 	})
 	return
 }
@@ -1441,7 +1345,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, leaf, dirID, gcid string,
 	if uploadType == api.UploadTypeForm && new.Form != nil {
 		err = f.uploadByForm(ctx, in, req.Name, size, new.Form, options...)
 	} else if uploadType == api.UploadTypeResumable && new.Resumable != nil {
-		err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable, options...)
+		err = f.uploadByResumable(ctx, in, leaf, size, new.Resumable)
 	} else {
 		err = fmt.Errorf("no method available for uploading: %+v", new)
 	}

backend/pikpak/pikpak_test.go

@@ -1,10 +1,10 @@
 // Test PikPak filesystem interface
-package pikpak
+package pikpak_test
 
 import (
 	"testing"
 
-	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/backend/pikpak"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -12,23 +12,6 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestPikPak:",
-		NilObject:  (*Object)(nil),
-		ChunkedUpload: fstests.ChunkedUploadConfig{
-			MinChunkSize: minChunkSize,
-			MaxChunkSize: maxChunkSize,
-		},
+		NilObject:  (*pikpak.Object)(nil),
 	})
 }
-
-func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadChunkSize(cs)
-}
-
-func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
-	return f.setUploadCutoff(cs)
-}
-
-var (
-	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
-	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
-)

backend/s3/s3.go

@@ -101,12 +101,6 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Dreamhost",
 		Help:  "Dreamhost DreamObjects",
-	}, {
-		Value: "Exaba",
-		Help:  "Exaba Object Storage",
-	}, {
-		Value: "FlashBlade",
-		Help:  "Pure Storage FlashBlade Object Storage",
 	}, {
 		Value: "GCS",
 		Help:  "Google Cloud Storage",
@@ -137,9 +131,6 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Magalu",
 		Help:  "Magalu Object Storage",
-	}, {
-		Value: "Mega",
-		Help:  "MEGA S4 Object Storage",
 	}, {
 		Value: "Minio",
 		Help:  "Minio Object Storage",
@@ -185,9 +176,6 @@ var providerOption = fs.Option{
 	}, {
 		Value: "Qiniu",
 		Help:  "Qiniu Object Storage (Kodo)",
-	}, {
-		Value: "Zata",
-		Help:  "Zata (S3 compatible Gateway)",
 	}, {
 		Value: "Other",
 		Help:  "Any other S3 compatible provider",
@@ -493,14 +481,6 @@ func init() {
 				Value: "ap-northeast-1",
 				Help:  "Northeast Asia Region 1.\nNeeds location constraint ap-northeast-1.",
 			}},
-		}, {
-			Name:     "region",
-			Help:     "Region where you can connect with.\n",
-			Provider: "Zata",
-			Examples: []fs.OptionExample{{
-				Value: "us-east-1",
-				Help:  "Indore, Madhya Pradesh, India",
-			}},
 		}, {
 			Name:     "region",
 			Help:     "Region where your bucket will be created and your data stored.\n",
@@ -587,7 +567,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,FlashBlade,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive,Mega,Zata",
+			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Linode,Magalu,Qiniu,RackCorp,Scaleway,Selectel,Storj,Synology,TencentCOS,HuaweiOBS,IDrive",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -1023,12 +1003,6 @@ func init() {
 				Value: "us-iad-1.linodeobjects.com",
 				Help:  "Washington, DC, (USA), us-iad-1",
 			}},
-		}, {
-			// Lyve Cloud endpoints
-			Name:     "endpoint",
-			Help:     "Endpoint for Lyve Cloud S3 API.\nRequired when using an S3 clone. Please type in your LyveCloud endpoint.\nExamples:\n- s3.us-west-1.{account_name}.lyve.seagate.com (US West 1 - California)\n- s3.eu-west-1.{account_name}.lyve.seagate.com (EU West 1 - Ireland)",
-			Provider: "LyveCloud",
-			Required: true,
 		}, {
 			// Magalu endpoints: https://docs.magalu.cloud/docs/object-storage/how-to/copy-url
 			Name:     "endpoint",
@@ -1391,14 +1365,6 @@ func init() {
 				Value: "s3-ap-northeast-1.qiniucs.com",
 				Help:  "Northeast Asia Endpoint 1",
 			}},
-		}, {
-			Name:     "endpoint",
-			Help:     "Endpoint for Zata Object Storage.",
-			Provider: "Zata",
-			Examples: []fs.OptionExample{{
-				Value: "idr01.zata.ai",
-				Help:  "South Asia Endpoint",
-			}},
 		}, {
 			// Selectel endpoints: https://docs.selectel.ru/en/cloud/object-storage/manage/domains/#s3-api-domains
 			Name:     "endpoint",
@@ -1411,7 +1377,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,LyveCloud,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox,Zata",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Linode,Magalu,Scaleway,Selectel,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1460,6 +1426,18 @@ func init() {
 				Value:    "localhost:8333",
 				Help:     "SeaweedFS S3 localhost",
 				Provider: "SeaweedFS",
+			}, {
+				Value:    "s3.us-east-1.lyvecloud.seagate.com",
+				Help:     "Seagate Lyve Cloud US East 1 (Virginia)",
+				Provider: "LyveCloud",
+			}, {
+				Value:    "s3.us-west-1.lyvecloud.seagate.com",
+				Help:     "Seagate Lyve Cloud US West 1 (California)",
+				Provider: "LyveCloud",
+			}, {
+				Value:    "s3.ap-southeast-1.lyvecloud.seagate.com",
+				Help:     "Seagate Lyve Cloud AP Southeast 1 (Singapore)",
+				Provider: "LyveCloud",
 			}, {
 				Value:    "oos.eu-west-2.outscale.com",
 				Help:     "Outscale EU West 2 (Paris)",
@@ -1548,22 +1526,6 @@ func init() {
 				Value:    "s3.ir-tbz-sh1.arvanstorage.ir",
 				Help:     "ArvanCloud Tabriz Iran (Shahriar) endpoint",
 				Provider: "ArvanCloud",
-			}, {
-				Value:    "s3.eu-central-1.s4.mega.io",
-				Help:     "Mega S4 eu-central-1 (Amsterdam)",
-				Provider: "Mega",
-			}, {
-				Value:    "s3.eu-central-2.s4.mega.io",
-				Help:     "Mega S4 eu-central-2 (Bettembourg)",
-				Provider: "Mega",
-			}, {
-				Value:    "s3.ca-central-1.s4.mega.io",
-				Help:     "Mega S4 ca-central-1 (Montreal)",
-				Provider: "Mega",
-			}, {
-				Value:    "s3.ca-west-1.s4.mega.io",
-				Help:     "Mega S4 ca-west-1 (Vancouver)",
-				Provider: "Mega",
 			}},
 		}, {
 			Name:     "location_constraint",
@@ -1946,7 +1908,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,FlashBlade,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox,Mega",
+			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Linode,Magalu,Outscale,Qiniu,RackCorp,Scaleway,Selectel,StackPath,Storj,TencentCOS,Petabox",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1961,7 +1923,7 @@ doesn't copy the ACL from the source but rather writes a fresh one.
 If the acl is an empty string then no X-Amz-Acl: header is added and
 the default (private) will be used.
 `,
-			Provider: "!Storj,Selectel,Synology,Cloudflare,FlashBlade,Mega",
+			Provider: "!Storj,Selectel,Synology,Cloudflare",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
 				Help:     "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -2019,7 +1981,6 @@ isn't set then "acl" is used instead.
 If the "acl" and "bucket_acl" are empty strings then no X-Amz-Acl:
 header is added and the default (private) will be used.
 `,
-			Provider: "!Storj,Selectel,Synology,Cloudflare,FlashBlade",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
 				Value: "private",
@@ -2635,7 +2596,7 @@ The parameter should be a date, "2006-01-02", datetime "2006-01-02
 Note that when using this no file write operations are permitted,
 so you can't upload files or delete them.
 
-See [the time option docs](/docs/#time-options) for valid formats.
+See [the time option docs](/docs/#time-option) for valid formats.
 `,
 			Default:  fs.Time{},
 			Advanced: true,
@@ -3155,9 +3116,6 @@ func parsePath(path string) (root string) {
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
 	bucketName, bucketPath = bucket.Split(bucket.Join(f.root, rootRelativePath))
-	if f.opt.DirectoryMarkers && strings.HasSuffix(bucketPath, "//") {
-		bucketPath = bucketPath[:len(bucketPath)-1]
-	}
 	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
@@ -3537,9 +3495,6 @@ func setQuirks(opt *Options) {
 	case "Dreamhost":
 		urlEncodeListings = false
 		useAlreadyExists = false // untested
-	case "FlashBlade":
-		mightGzip = false        // Never auto gzips objects
-		virtualHostStyle = false // supports vhost but defaults to paths
 	case "IBMCOS":
 		listObjectsV2 = false // untested
 		virtualHostStyle = false
@@ -3572,14 +3527,6 @@ func setQuirks(opt *Options) {
 		urlEncodeListings = false
 		useMultipartEtag = false
 		useAlreadyExists = false
-	case "Mega":
-		listObjectsV2 = true
-		virtualHostStyle = false
-		urlEncodeListings = true
-		useMultipartEtag = false
-		useAlreadyExists = false
-		// Multipart server side copies not supported
-		opt.CopyCutoff = math.MaxInt64
 	case "Minio":
 		virtualHostStyle = false
 	case "Netease":
@@ -3650,13 +3597,6 @@ func setQuirks(opt *Options) {
 		urlEncodeListings = false
 		virtualHostStyle = false
 		useAlreadyExists = false // untested
-	case "Zata":
-		useMultipartEtag = false
-		mightGzip = false
-		useUnsignedPayload = false
-		useAlreadyExists = false
-	case "Exaba":
-		virtualHostStyle = false
 	case "GCS":
 		// Google break request Signature by mutating accept-encoding HTTP header
 		// https://github.com/rclone/rclone/issues/6670
@@ -4485,7 +4425,7 @@ func (f *Fs) list(ctx context.Context, opt listOpt, fn listFn) error {
 			remote = remote[len(opt.prefix):]
 			if isDirectory {
 				// process directory markers as directories
-				remote, _ = strings.CutSuffix(remote, "/")
+				remote = strings.TrimRight(remote, "/")
 			}
 			if opt.addBucket {
 				remote = bucket.Join(opt.bucket, remote)
@@ -4800,7 +4740,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 // mkdirParent creates the parent bucket/directory if it doesn't exist
 func (f *Fs) mkdirParent(ctx context.Context, remote string) error {
-	remote, _ = strings.CutSuffix(remote, "/")
+	remote = strings.TrimRight(remote, "/")
 	dir := path.Dir(remote)
 	if dir == "/" || dir == "." {
 		dir = ""

backend/seafile/seafile.go

@@ -111,8 +111,7 @@ func init() {
 				encoder.EncodeSlash |
 				encoder.EncodeBackSlash |
 				encoder.EncodeDoubleQuote |
-				encoder.EncodeInvalidUtf8 |
-				encoder.EncodeDot),
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }

backend/sftp/sftp.go

@@ -222,45 +222,15 @@ E.g. the second example above should be rewritten as:
 					Help:  "Windows Command Prompt",
 				},
 			},
-		}, {
-			Name:     "hashes",
-			Help:     `Comma separated list of supported checksum types.`,
-			Default:  fs.CommaSepList{},
-			Advanced: true,
 		}, {
 			Name:     "md5sum_command",
 			Default:  "",
-			Help:     "The command used to read MD5 hashes.\n\nLeave blank for autodetect.",
+			Help:     "The command used to read md5 hashes.\n\nLeave blank for autodetect.",
 			Advanced: true,
 		}, {
 			Name:     "sha1sum_command",
 			Default:  "",
-			Help:     "The command used to read SHA-1 hashes.\n\nLeave blank for autodetect.",
-			Advanced: true,
-		}, {
-			Name:     "crc32sum_command",
-			Default:  "",
-			Help:     "The command used to read CRC-32 hashes.\n\nLeave blank for autodetect.",
-			Advanced: true,
-		}, {
-			Name:     "sha256sum_command",
-			Default:  "",
-			Help:     "The command used to read SHA-256 hashes.\n\nLeave blank for autodetect.",
-			Advanced: true,
-		}, {
-			Name:     "blake3sum_command",
-			Default:  "",
-			Help:     "The command used to read BLAKE3 hashes.\n\nLeave blank for autodetect.",
-			Advanced: true,
-		}, {
-			Name:     "xxh3sum_command",
-			Default:  "",
-			Help:     "The command used to read XXH3 hashes.\n\nLeave blank for autodetect.",
-			Advanced: true,
-		}, {
-			Name:     "xxh128sum_command",
-			Default:  "",
-			Help:     "The command used to read XXH128 hashes.\n\nLeave blank for autodetect.",
+			Help:     "The command used to read sha1 hashes.\n\nLeave blank for autodetect.",
 			Advanced: true,
 		}, {
 			Name:     "skip_links",
@@ -565,14 +535,8 @@ type Options struct {
 	PathOverride            string          `config:"path_override"`
 	SetModTime              bool            `config:"set_modtime"`
 	ShellType               string          `config:"shell_type"`
-	Hashes                  fs.CommaSepList `config:"hashes"`
 	Md5sumCommand           string          `config:"md5sum_command"`
 	Sha1sumCommand          string          `config:"sha1sum_command"`
-	Crc32sumCommand         string          `config:"crc32sum_command"`
-	Sha256sumCommand        string          `config:"sha256sum_command"`
-	Blake3sumCommand        string          `config:"blake3sum_command"`
-	Xxh3sumCommand          string          `config:"xxh3sum_command"`
-	Xxh128sumCommand        string          `config:"xxh128sum_command"`
 	SkipLinks               bool            `config:"skip_links"`
 	Subsystem               string          `config:"subsystem"`
 	ServerCommand           string          `config:"server_command"`
@@ -621,18 +585,13 @@ type Fs struct {
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
 type Object struct {
-	fs        *Fs
-	remote    string
-	size      int64       // size of the object
-	modTime   uint32      // modification time of the object as unix time
-	mode      os.FileMode // mode bits from the file
-	md5sum    *string     // Cached MD5 checksum
-	sha1sum   *string     // Cached SHA-1 checksum
-	crc32sum  *string     // Cached CRC-32 checksum
-	sha256sum *string     // Cached SHA-256 checksum
-	blake3sum *string     // Cached BLAKE3 checksum
-	xxh3sum   *string     // Cached XXH3 checksum
-	xxh128sum *string     // Cached XXH128 checksum
+	fs      *Fs
+	remote  string
+	size    int64       // size of the object
+	modTime uint32      // modification time of the object as unix time
+	mode    os.FileMode // mode bits from the file
+	md5sum  *string     // Cached MD5 checksum
+	sha1sum *string     // Cached SHA1 checksum
 }
 
 // conn encapsulates an ssh client and corresponding sftp client
@@ -932,7 +891,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		User:            opt.User,
 		Auth:            []ssh.AuthMethod{},
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		Timeout:         time.Duration(f.ci.ConnectTimeout),
+		Timeout:         f.ci.ConnectTimeout,
 		ClientVersion:   "SSH-2.0-" + f.ci.UserAgent,
 	}
 
@@ -1684,112 +1643,14 @@ func (f *Fs) Hashes() hash.Set {
 		return *f.cachedHashes
 	}
 
-	hashTypesSupported := hash.NewHashSet()
-	f.cachedHashes = &hashTypesSupported
+	hashSet := hash.NewHashSet()
+	f.cachedHashes = &hashSet
 
 	if f.opt.DisableHashCheck || f.shellType == shellTypeNotSupported {
-		return hashTypesSupported
-	}
-
-	hashTypes := hash.NewHashSet()
-	if len(f.opt.Hashes) > 0 {
-		for _, hashName := range f.opt.Hashes {
-			var hashType hash.Type
-			if err := hashType.Set(hashName); err != nil {
-				fs.Infof(nil, "Invalid token %q in hash string %q", hashName, f.opt.Hashes.String())
-			}
-			hashTypes.Add(hashType)
-		}
-	} else {
-		hashTypes.Add(hash.MD5, hash.SHA1)
-	}
-
-	hashCommands := map[hash.Type]struct {
-		option       *string
-		emptyHash    string
-		hashCommands []struct{ hashFile, hashEmpty string }
-	}{
-		hash.MD5: {
-			&f.opt.Md5sumCommand,
-			"d41d8cd98f00b204e9800998ecf8427e",
-			[]struct{ hashFile, hashEmpty string }{
-				{"md5sum", "md5sum"},
-				{"md5 -r", "md5 -r"},
-				{"rclone md5sum", "rclone md5sum"},
-			},
-		},
-		hash.SHA1: {
-			&f.opt.Sha1sumCommand,
-			"da39a3ee5e6b4b0d3255bfef95601890afd80709",
-			[]struct{ hashFile, hashEmpty string }{
-				{"sha1sum", "sha1sum"},
-				{"sha1 -r", "sha1 -r"},
-				{"rclone sha1sum", "rclone sha1sum"},
-			},
-		},
-		hash.CRC32: {
-			&f.opt.Sha1sumCommand,
-			"00000000",
-			[]struct{ hashFile, hashEmpty string }{
-				{"crc32", "crc32"},
-				{"rclone hashsum crc32", "rclone hashsum crc32"},
-			},
-		},
-		hash.SHA256: {
-			&f.opt.Sha256sumCommand,
-			"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
-			[]struct{ hashFile, hashEmpty string }{
-				{"sha256sum", "sha1sum"},
-				{"sha256 -r", "sha1 -r"},
-				{"rclone hashsum sha256", "rclone hashsum sha256"},
-			},
-		},
-		hash.BLAKE3: {
-			&f.opt.Blake3sumCommand,
-			"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
-			[]struct{ hashFile, hashEmpty string }{
-				{"b3sum", "b3sum"},
-				{"rclone hashsum blake3", "rclone hashsum blake3"},
-			},
-		},
-		hash.XXH3: {
-			&f.opt.Xxh3sumCommand,
-			"2d06800538d394c2",
-			[]struct{ hashFile, hashEmpty string }{
-				// The xxhsum tool uses a non-standard prefix "XXH3_" preceding the hash output for the 64-bit variant
-				// of XXH3, to avoid confusion with the older 64-bit algorithm XXH64. This was introduced in version
-				// 0.8.3 released Dec 30, 2024. Older versions only supported the alternative BSD style output format,
-				// otherwise optional with argument --tag. We are currently not expecting these output formats and can
-				// therefore not use the "xxhsum -H3" command or its xxh3sum alias directly.
-				//{"xxh3sum", "xxh3sum"},
-				//{"xxhsum -H3", "xxhsum -H3"},
-				{"rclone hashsum xxh3", "rclone hashsum xxh3"},
-			},
-		},
-		hash.XXH128: {
-			&f.opt.Xxh128sumCommand,
-			"99aa06d3014798d86001c324468d497f",
-			[]struct{ hashFile, hashEmpty string }{
-				{"xxh128sum", "xxh128sum"},
-				{"xxhsum -H2", "xxhsum -H2"},
-				{"rclone hashsum xxh128", "rclone hashsum xxh128"},
-			},
-		},
-	}
-	if f.shellType == "powershell" {
-		for _, hashType := range []hash.Type{hash.MD5, hash.SHA1, hash.SHA256} {
-			if entry, ok := hashCommands[hashType]; ok {
-				entry.hashCommands = append(hashCommands[hashType].hashCommands, struct {
-					hashFile, hashEmpty string
-				}{
-					fmt.Sprintf("&{param($Path);Get-FileHash -Algorithm %v -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower())  ${Path}\"}}", hashType),
-					fmt.Sprintf("Get-FileHash -Algorithm %v -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}", hashType),
-				})
-				hashCommands[hashType] = entry
-			}
-		}
+		return hashSet
 	}
 
+	// look for a hash command which works
 	checkHash := func(hashType hash.Type, commands []struct{ hashFile, hashEmpty string }, expected string, hashCommand *string, changed *bool) bool {
 		if *hashCommand == hashCommandNotSupported {
 			return false
@@ -1818,25 +1679,55 @@ func (f *Fs) Hashes() hash.Set {
 	}
 
 	changed := false
-	for _, hashType := range hashTypes.Array() {
-		if entry, ok := hashCommands[hashType]; ok {
-			if works := checkHash(hashType, entry.hashCommands, entry.emptyHash, entry.option, &changed); works {
-				hashTypesSupported.Add(hashType)
-			}
-		}
+	md5Commands := []struct {
+		hashFile, hashEmpty string
+	}{
+		{"md5sum", "md5sum"},
+		{"md5 -r", "md5 -r"},
+		{"rclone md5sum", "rclone md5sum"},
 	}
+	sha1Commands := []struct {
+		hashFile, hashEmpty string
+	}{
+		{"sha1sum", "sha1sum"},
+		{"sha1 -r", "sha1 -r"},
+		{"rclone sha1sum", "rclone sha1sum"},
+	}
+	if f.shellType == "powershell" {
+		md5Commands = append(md5Commands, struct {
+			hashFile, hashEmpty string
+		}{
+			"&{param($Path);Get-FileHash -Algorithm MD5 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower())  ${Path}\"}}",
+			"Get-FileHash -Algorithm MD5 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
+		})
+
+		sha1Commands = append(sha1Commands, struct {
+			hashFile, hashEmpty string
+		}{
+			"&{param($Path);Get-FileHash -Algorithm SHA1 -LiteralPath $Path -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{\"$($_.ToLower())  ${Path}\"}}",
+			"Get-FileHash -Algorithm SHA1 -InputStream ([System.IO.MemoryStream]::new()) -ErrorAction Stop|Select-Object -First 1 -ExpandProperty Hash|ForEach-Object{$_.ToLower()}",
+		})
+	}
+
+	md5Works := checkHash(hash.MD5, md5Commands, "d41d8cd98f00b204e9800998ecf8427e", &f.opt.Md5sumCommand, &changed)
+	sha1Works := checkHash(hash.SHA1, sha1Commands, "da39a3ee5e6b4b0d3255bfef95601890afd80709", &f.opt.Sha1sumCommand, &changed)
 
 	if changed {
 		// Save permanently in config to avoid the extra work next time
-		for _, hashType := range hashTypes.Array() {
-			if entry, ok := hashCommands[hashType]; ok {
-				fs.Debugf(f, "Setting hash command for %v to %q (set %vsum_command to override)", hashType, *entry.option, hashType)
-				f.m.Set(fmt.Sprintf("%vsum_command", hashType), *entry.option)
-			}
-		}
+		fs.Debugf(f, "Setting hash command for %v to %q (set sha1sum_command to override)", hash.MD5, f.opt.Md5sumCommand)
+		f.m.Set("md5sum_command", f.opt.Md5sumCommand)
+		fs.Debugf(f, "Setting hash command for %v to %q (set md5sum_command to override)", hash.SHA1, f.opt.Sha1sumCommand)
+		f.m.Set("sha1sum_command", f.opt.Sha1sumCommand)
 	}
 
-	return hashTypesSupported
+	if sha1Works {
+		hashSet.Add(hash.SHA1)
+	}
+	if md5Works {
+		hashSet.Add(hash.MD5)
+	}
+
+	return hashSet
 }
 
 // About gets usage stats
@@ -1971,43 +1862,17 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	_ = o.fs.Hashes()
 
 	var hashCmd string
-	switch r {
-	case hash.MD5:
+	if r == hash.MD5 {
 		if o.md5sum != nil {
 			return *o.md5sum, nil
 		}
 		hashCmd = o.fs.opt.Md5sumCommand
-	case hash.SHA1:
+	} else if r == hash.SHA1 {
 		if o.sha1sum != nil {
 			return *o.sha1sum, nil
 		}
 		hashCmd = o.fs.opt.Sha1sumCommand
-	case hash.CRC32:
-		if o.crc32sum != nil {
-			return *o.crc32sum, nil
-		}
-		hashCmd = o.fs.opt.Crc32sumCommand
-	case hash.SHA256:
-		if o.sha256sum != nil {
-			return *o.sha256sum, nil
-		}
-		hashCmd = o.fs.opt.Sha256sumCommand
-	case hash.BLAKE3:
-		if o.blake3sum != nil {
-			return *o.blake3sum, nil
-		}
-		hashCmd = o.fs.opt.Blake3sumCommand
-	case hash.XXH3:
-		if o.xxh3sum != nil {
-			return *o.xxh3sum, nil
-		}
-		hashCmd = o.fs.opt.Xxh3sumCommand
-	case hash.XXH128:
-		if o.xxh128sum != nil {
-			return *o.xxh128sum, nil
-		}
-		hashCmd = o.fs.opt.Xxh128sumCommand
-	default:
+	} else {
 		return "", hash.ErrUnsupported
 	}
 	if hashCmd == "" || hashCmd == hashCommandNotSupported {
@@ -2024,21 +1889,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	}
 	hashString := parseHash(outBytes)
 	fs.Debugf(o, "Parsed hash: %s", hashString)
-	switch r {
-	case hash.MD5:
+	if r == hash.MD5 {
 		o.md5sum = &hashString
-	case hash.SHA1:
+	} else if r == hash.SHA1 {
 		o.sha1sum = &hashString
-	case hash.CRC32:
-		o.crc32sum = &hashString
-	case hash.SHA256:
-		o.sha256sum = &hashString
-	case hash.BLAKE3:
-		o.blake3sum = &hashString
-	case hash.XXH3:
-		o.xxh3sum = &hashString
-	case hash.XXH128:
-		o.xxh128sum = &hashString
 	}
 	return hashString, nil
 }
@@ -2103,7 +1957,7 @@ func (f *Fs) remoteShellPath(remote string) string {
 }
 
 // Converts a byte array from the SSH session returned by
-// an invocation of hash command to a hash string
+// an invocation of md5sum/sha1sum to a hash string
 // as expected by the rest of this application
 func parseHash(bytes []byte) string {
 	// For strings with backslash *sum writes a leading \
@@ -2332,11 +2186,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Clear the hash cache since we are about to update the object
 	o.md5sum = nil
 	o.sha1sum = nil
-	o.crc32sum = nil
-	o.sha256sum = nil
-	o.blake3sum = nil
-	o.xxh3sum = nil
-	o.xxh128sum = nil
 	c, err := o.fs.getSftpConnection(ctx)
 	if err != nil {
 		return fmt.Errorf("Update: %w", err)

backend/smb/connpool.go

@@ -38,7 +38,7 @@ func (f *Fs) dial(ctx context.Context, network, addr string) (*conn, error) {
 
 	d := &smb2.Dialer{}
 	if f.opt.UseKerberos {
-		cl, err := createKerberosClient(f.opt.KerberosCCache)
+		cl, err := getKerberosClient()
 		if err != nil {
 			return nil, err
 		}

backend/smb/kerberos.go

@@ -14,81 +14,65 @@ import (
 )
 
 var (
-	kerberosClient sync.Map // map[string]*client.Client
-	kerberosErr    sync.Map // map[string]error
+	kerberosClient *client.Client
+	kerberosErr    error
+	kerberosOnce   sync.Once
 )
 
-func resolveCcachePath(ccachePath string) (string, error) {
-	if ccachePath == "" {
-		ccachePath = os.Getenv("KRB5CCNAME")
+// getKerberosClient returns a Kerberos client that can be used to authenticate.
+func getKerberosClient() (*client.Client, error) {
+	if kerberosClient == nil || kerberosErr == nil {
+		kerberosOnce.Do(func() {
+			kerberosClient, kerberosErr = createKerberosClient()
+		})
 	}
 
-	switch {
-	case strings.Contains(ccachePath, ":"):
-		parts := strings.SplitN(ccachePath, ":", 2)
-		prefix, path := parts[0], parts[1]
-		switch prefix {
-		case "FILE":
-			return path, nil
-		case "DIR":
-			primary, err := os.ReadFile(filepath.Join(path, "primary"))
-			if err != nil {
-				return "", err
-			}
-			return filepath.Join(path, strings.TrimSpace(string(primary))), nil
-		default:
-			return "", fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
-		}
-	case ccachePath == "":
-		u, err := user.Current()
-		if err != nil {
-			return "", err
-		}
-		return "/tmp/krb5cc_" + u.Uid, nil
-	default:
-		return ccachePath, nil
-	}
+	return kerberosClient, kerberosErr
 }
 
-func loadKerberosConfig() (*config.Config, error) {
+// createKerberosClient creates a new Kerberos client.
+func createKerberosClient() (*client.Client, error) {
 	cfgPath := os.Getenv("KRB5_CONFIG")
 	if cfgPath == "" {
 		cfgPath = "/etc/krb5.conf"
 	}
-	return config.Load(cfgPath)
-}
 
-// createKerberosClient creates a new Kerberos client.
-func createKerberosClient(ccachePath string) (*client.Client, error) {
-	ccachePath, err := resolveCcachePath(ccachePath)
+	cfg, err := config.Load(cfgPath)
 	if err != nil {
 		return nil, err
 	}
 
-	// check if we already have a client or an error for this ccache path
-	if errVal, ok := kerberosErr.Load(ccachePath); ok {
-		return nil, errVal.(error)
-	}
-	if clientVal, ok := kerberosClient.Load(ccachePath); ok {
-		return clientVal.(*client.Client), nil
+	// Determine the ccache location from the environment, falling back to the
+	// default location.
+	ccachePath := os.Getenv("KRB5CCNAME")
+	switch {
+	case strings.Contains(ccachePath, ":"):
+		parts := strings.SplitN(ccachePath, ":", 2)
+		switch parts[0] {
+		case "FILE":
+			ccachePath = parts[1]
+		case "DIR":
+			primary, err := os.ReadFile(filepath.Join(parts[1], "primary"))
+			if err != nil {
+				return nil, err
+			}
+			ccachePath = filepath.Join(parts[1], strings.TrimSpace(string(primary)))
+		default:
+			return nil, fmt.Errorf("unsupported KRB5CCNAME: %s", ccachePath)
+		}
+	case ccachePath == "":
+		u, err := user.Current()
+		if err != nil {
+			return nil, err
+		}
+
+		ccachePath = "/tmp/krb5cc_" + u.Uid
 	}
 
-	// create a new client if not found in the map
-	cfg, err := loadKerberosConfig()
-	if err != nil {
-		kerberosErr.Store(ccachePath, err)
-		return nil, err
-	}
 	ccache, err := credentials.LoadCCache(ccachePath)
 	if err != nil {
-		kerberosErr.Store(ccachePath, err)
 		return nil, err
 	}
-	cl, err := client.NewFromCCache(ccache, cfg)
-	if err != nil {
-		kerberosErr.Store(ccachePath, err)
-		return nil, err
-	}
-	kerberosClient.Store(ccachePath, cl)
-	return cl, nil
+
+	return client.NewFromCCache(ccache, cfg)
 }

backend/smb/kerberos_test.go

@@ -1,79 +0,0 @@
-package smb
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestResolveCcachePath(t *testing.T) {
-	tmpDir := t.TempDir()
-
-	// Setup: files for FILE and DIR modes
-	fileCcache := filepath.Join(tmpDir, "file_ccache")
-	err := os.WriteFile(fileCcache, []byte{}, 0600)
-	assert.NoError(t, err)
-
-	dirCcache := filepath.Join(tmpDir, "dir_ccache")
-	err = os.Mkdir(dirCcache, 0755)
-	assert.NoError(t, err)
-	err = os.WriteFile(filepath.Join(dirCcache, "primary"), []byte("ticket"), 0600)
-	assert.NoError(t, err)
-	dirCcacheTicket := filepath.Join(dirCcache, "ticket")
-	err = os.WriteFile(dirCcacheTicket, []byte{}, 0600)
-	assert.NoError(t, err)
-
-	tests := []struct {
-		name          string
-		ccachePath    string
-		envKRB5CCNAME string
-		expected      string
-		expectError   bool
-	}{
-		{
-			name:          "FILE: prefix from env",
-			ccachePath:    "",
-			envKRB5CCNAME: "FILE:" + fileCcache,
-			expected:      fileCcache,
-		},
-		{
-			name:          "DIR: prefix from env",
-			ccachePath:    "",
-			envKRB5CCNAME: "DIR:" + dirCcache,
-			expected:      dirCcacheTicket,
-		},
-		{
-			name:          "Unsupported prefix",
-			ccachePath:    "",
-			envKRB5CCNAME: "MEMORY:/bad/path",
-			expectError:   true,
-		},
-		{
-			name:       "Direct file path (no prefix)",
-			ccachePath: "/tmp/myccache",
-			expected:   "/tmp/myccache",
-		},
-		{
-			name:          "Default to /tmp/krb5cc_<uid>",
-			ccachePath:    "",
-			envKRB5CCNAME: "",
-			expected:      "/tmp/krb5cc_",
-		},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			t.Setenv("KRB5CCNAME", tt.envKRB5CCNAME)
-			result, err := resolveCcachePath(tt.ccachePath)
-
-			if tt.expectError {
-				assert.Error(t, err)
-			} else {
-				assert.NoError(t, err)
-				assert.Contains(t, result, tt.expected)
-			}
-		})
-	}
-}

backend/smb/smb.go

@@ -107,20 +107,6 @@ Set to 0 to keep connections indefinitely.
 			Help:     "Whether the server is configured to be case-insensitive.\n\nAlways true on Windows shares.",
 			Default:  true,
 			Advanced: true,
-		}, {
-			Name: "kerberos_ccache",
-			Help: `Path to the Kerberos credential cache (krb5cc).
-
-Overrides the default KRB5CCNAME environment variable and allows this
-instance of the SMB backend to use a different Kerberos cache file.
-This is useful when mounting multiple SMB with different credentials
-or running in multi-user environments.
-
-Supported formats:
-  - FILE:/path/to/ccache   – Use the specified file.
-  - DIR:/path/to/ccachedir – Use the primary file inside the specified directory.
-  - /path/to/ccache        – Interpreted as a file path.`,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -151,7 +137,6 @@ type Options struct {
 	Domain          string      `config:"domain"`
 	SPN             string      `config:"spn"`
 	UseKerberos     bool        `config:"use_kerberos"`
-	KerberosCCache  string      `config:"kerberos_ccache"`
 	HideSpecial     bool        `config:"hide_special_share"`
 	CaseInsensitive bool        `config:"case_insensitive"`
 	IdleTimeout     fs.Duration `config:"idle_timeout"`

backend/smb/smb_test.go

@@ -6,7 +6,6 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/smb"
-	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
@@ -19,9 +18,6 @@ func TestIntegration(t *testing.T) {
 }
 
 func TestIntegration2(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("skipping as -remote is set")
-	}
 	krb5Dir := t.TempDir()
 	t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
 	t.Setenv("KRB5CCNAME", filepath.Join(krb5Dir, "ccache"))
@@ -30,24 +26,3 @@ func TestIntegration2(t *testing.T) {
 		NilObject:  (*smb.Object)(nil),
 	})
 }
-
-func TestIntegration3(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("skipping as -remote is set")
-	}
-
-	krb5Dir := t.TempDir()
-	t.Setenv("KRB5_CONFIG", filepath.Join(krb5Dir, "krb5.conf"))
-	ccache := filepath.Join(krb5Dir, "ccache")
-	t.Setenv("RCLONE_TEST_CUSTOM_CCACHE_LOCATION", ccache)
-
-	name := "TestSMBKerberosCcache"
-
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":rclone",
-		NilObject:  (*smb.Object)(nil),
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "kerberos_ccache", Value: ccache},
-		},
-	})
-}

backend/swift/swift.go

@@ -491,8 +491,8 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
 		ApplicationCredentialName:   opt.ApplicationCredentialName,
 		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
 		EndpointType:                swift.EndpointType(opt.EndpointType),
-		ConnectTimeout:              time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
-		Timeout:                     time.Duration(10 * ci.Timeout),        // Use the timeouts in the transport
+		ConnectTimeout:              10 * ci.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:                     10 * ci.Timeout,        // Use the timeouts in the transport
 		Transport:                   fshttp.NewTransport(ctx),
 		FetchUntilEmptyPage:         opt.FetchUntilEmptyPage,
 		PartialPageFetchThreshold:   opt.PartialPageFetchThreshold,

backend/webdav/webdav.go

@@ -1550,7 +1550,7 @@ func (o *Object) extraHeaders(ctx context.Context, src fs.ObjectInfo) map[string
 	extraHeaders := map[string]string{}
 	if o.fs.useOCMtime || o.fs.hasOCMD5 || o.fs.hasOCSHA1 {
 		if o.fs.useOCMtime {
-			extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
+			extraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", o.modTime.Unix())
 		}
 		// Set one upload checksum
 		// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5

bin/.ignore-emails

@@ -12,5 +12,4 @@
 <seb•ɑƬ•chezwam•ɖɵʈ•org>
 <allllaboutyou@gmail.com>
 <psycho@feltzv.fr>
-<afw5059@gmail.com>
-<piyushgarg80>
+<afw5059@gmail.com>

bin/check_autogenerated_edits.py

@@ -77,10 +77,6 @@ def check_file(file):
     viol = False
     new_lines = get_file_content("HEAD", file)
     old_lines = get_file_content("HEAD~1", file)
-
-    # If old file did not exist or was empty then don't check
-    if not old_lines:
-        return
     
     # Entire autogenerated file check.
     if any("autogenerated - DO NOT EDIT" in l for l in new_lines[:10]):

bin/make_manual.py

@@ -41,10 +41,8 @@ docs = [
     "crypt.md",
     "compress.md",
     "combine.md",
-    "doi.md",
     "dropbox.md",
     "filefabric.md",
-    "filelu.md",
     "filescom.md",
     "ftp.md",
     "gofile.md",

cmd/bisync/bilib/output.go

@@ -3,22 +3,20 @@ package bilib
 
 import (
 	"bytes"
-	"log/slog"
+	"log"
 
-	"github.com/rclone/rclone/fs/log"
+	"github.com/sirupsen/logrus"
 )
 
-// CaptureOutput runs a function capturing its output at log level INFO.
+// CaptureOutput runs a function capturing its output.
 func CaptureOutput(fun func()) []byte {
+	logSave := log.Writer()
+	logrusSave := logrus.StandardLogger().Out
 	buf := &bytes.Buffer{}
-	oldLevel := log.Handler.SetLevel(slog.LevelInfo)
-	log.Handler.SetOutput(func(level slog.Level, text string) {
-		buf.WriteString(text)
-	})
-	defer func() {
-		log.Handler.ResetOutput()
-		log.Handler.SetLevel(oldLevel)
-	}()
+	log.SetOutput(buf)
+	logrus.SetOutput(buf)
 	fun()
+	log.SetOutput(logSave)
+	logrus.SetOutput(logrusSave)
 	return buf.Bytes()
 }

cmd/bisync/bisync_test.go

@@ -278,7 +278,7 @@ func testBisync(t *testing.T, path1, path2 string) {
 	}
 	bisync.Colors = true
 	time.Local = bisync.TZ
-	ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour)
+	ci.FsCacheExpireDuration = 5 * time.Hour
 
 	baseDir, err := os.Getwd()
 	require.NoError(t, err, "get current directory")

cmd/bisync/cmd.go

@@ -55,7 +55,7 @@ type Options struct {
 	Compare               CompareOpt
 	CompareFlag           string
 	DebugName             string
-	MaxLock               fs.Duration
+	MaxLock               time.Duration
 	ConflictResolve       Prefer
 	ConflictLoser         ConflictLoserAction
 	ConflictSuffixFlag    string
@@ -146,7 +146,7 @@ func init() {
 	flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")
 	flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")
 	flags.BoolVarP(cmdFlags, &Opt.Compare.DownloadHash, "download-hash", "", Opt.Compare.DownloadHash, "Compute hash by downloading when otherwise unavailable. (warning: may be slow and use lots of data!)", "")
-	flags.FVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
+	flags.DurationVarP(cmdFlags, &Opt.MaxLock, "max-lock", "", Opt.MaxLock, "Consider lock files older than this to be expired (default: 0 (never expire)) (minimum: 2m)", "")
 	flags.FVarP(cmdFlags, &Opt.ConflictResolve, "conflict-resolve", "", "Automatically resolve conflicts by preferring the version that is: "+ConflictResolveList+" (default: none)", "")
 	flags.FVarP(cmdFlags, &Opt.ConflictLoser, "conflict-loser", "", "Action to take on the loser of a sync conflict (when there is a winner) or on both files (when there is no winner): "+ConflictLoserList+" (default: num)", "")
 	flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")

cmd/bisync/lockfile.go

@@ -14,7 +14,7 @@ import (
 	"github.com/rclone/rclone/lib/terminal"
 )
 
-const basicallyforever = fs.Duration(200 * 365 * 24 * time.Hour)
+const basicallyforever = 200 * 365 * 24 * time.Hour
 
 var stopRenewal func()
 
@@ -66,9 +66,9 @@ func (b *bisyncRun) removeLockFile() {
 }
 
 func (b *bisyncRun) setLockFileExpiration() {
-	if b.opt.MaxLock > 0 && b.opt.MaxLock < fs.Duration(2*time.Minute) {
+	if b.opt.MaxLock > 0 && b.opt.MaxLock < 2*time.Minute {
 		fs.Logf(nil, Color(terminal.YellowFg, "--max-lock cannot be shorter than 2 minutes (unless 0.) Changing --max-lock from %v to %v"), b.opt.MaxLock, 2*time.Minute)
-		b.opt.MaxLock = fs.Duration(2 * time.Minute)
+		b.opt.MaxLock = 2 * time.Minute
 	} else if b.opt.MaxLock <= 0 {
 		b.opt.MaxLock = basicallyforever
 	}
@@ -80,7 +80,7 @@ func (b *bisyncRun) renewLockFile() {
 		data.Session = b.basePath
 		data.PID = strconv.Itoa(os.Getpid())
 		data.TimeRenewed = time.Now()
-		data.TimeExpires = time.Now().Add(time.Duration(b.opt.MaxLock))
+		data.TimeExpires = time.Now().Add(b.opt.MaxLock)
 
 		// save data file
 		df, err := os.Create(b.lockFile)
@@ -131,7 +131,7 @@ func (b *bisyncRun) startLockRenewal() func() {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		ticker := time.NewTicker(time.Duration(b.opt.MaxLock) - time.Minute)
+		ticker := time.NewTicker(b.opt.MaxLock - time.Minute)
 		for {
 			select {
 			case <-ticker.C:

cmd/bisync/operations.go

@@ -138,7 +138,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
 					if b.SyncCI != nil {
 						fs.Infoc(nil, Color(terminal.YellowFg, "Telling Sync to wrap up early."))
 						b.SyncCI.MaxTransfer = 1
-						b.SyncCI.MaxDuration = fs.Duration(1 * time.Second)
+						b.SyncCI.MaxDuration = 1 * time.Second
 						b.SyncCI.CutoffMode = fs.CutoffModeSoft
 						gracePeriod := 30 * time.Second // TODO: flag to customize this?
 						if !waitFor("Canceling Sync if not done in", gracePeriod, func() bool { return b.CleanupCompleted }) {

cmd/bisync/queue.go

@@ -376,8 +376,8 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
 	return files.Save(queueFile)
 }
 
-func naptime(totalWait fs.Duration) {
-	expireTime := time.Now().Add(time.Duration(totalWait))
+func naptime(totalWait time.Duration) {
+	expireTime := time.Now().Add(totalWait)
 	fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))
 	for i := 0; time.Until(expireTime) > 0; i++ {
 		if i > 0 && i%10 == 0 {

cmd/check/check.go

@@ -73,7 +73,7 @@ you what happened to it. These are reminiscent of diff files.
 - |* path| means path was present in source and destination but different.
 - |! path| means there was an error reading or hashing the source or dest.
 
-The default number of parallel checks is 8. See the [--checkers](/docs/#checkers-int)
+The default number of parallel checks is 8. See the [--checkers=N](/docs/#checkers-n)
 option for more information.
 `, "|", "`")
 

cmd/cmd.go

@@ -288,7 +288,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 			accounting.GlobalStats().ResetErrors()
 		}
 		if ci.RetriesInterval > 0 {
-			time.Sleep(time.Duration(ci.RetriesInterval))
+			time.Sleep(ci.RetriesInterval)
 		}
 	}
 	stopStats()

cmd/convmv/convmv.go

@@ -34,15 +34,17 @@ var commandDefinition = &cobra.Command{
 	Long: strings.ReplaceAll(`
 convmv supports advanced path name transformations for converting and renaming files and directories by applying prefixes, suffixes, and other alterations.
 
-`+transform.Help()+`Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
+`+transform.SprintList()+`
+
+Multiple transformations can be used in sequence, applied in the order they are specified on the command line.
 
 The ¡--name-transform¡ flag is also available in ¡sync¡, ¡copy¡, and ¡move¡.
 
-## Files vs Directories
+## Files vs Directories ##
 
 By default ¡--name-transform¡ will only apply to file names. The means only the leaf file name will be transformed.
 However some of the transforms would be better applied to the whole path or just directories.
-To choose which which part of the file path is affected some tags can be added to the ¡--name-transform¡.
+To choose which which part of the file path is affected some tags can be added to the ¡--name-transform¡
 
 | Tag | Effect |
 |------|------|
@@ -52,11 +54,11 @@ To choose which which part of the file path is affected some tags can be added t
 
 This is used by adding the tag into the transform name like this: ¡--name-transform file,prefix=ABC¡ or ¡--name-transform dir,prefix=DEF¡.
 
-For some conversions using all is more likely to be useful, for example ¡--name-transform all,nfc¡.
+For some conversions using all is more likely to be useful, for example ¡--name-transform all,nfc¡
 
 Note that ¡--name-transform¡ may not add path separators ¡/¡ to the name. This will cause an error.
 
-## Ordering and Conflicts
+## Ordering and Conflicts ##
 
 * Transformations will be applied in the order specified by the user.
     * If the ¡file¡ tag is in use (the default) then only the leaf name of files will be transformed.
@@ -71,19 +73,19 @@ user, allowing for intentional use cases (e.g., trimming one prefix before addin
     * Users should be aware that certain combinations may lead to unexpected results and should verify
 transformations using ¡--dry-run¡ before execution.
 
-## Race Conditions and Non-Deterministic Behavior
+## Race Conditions and Non-Deterministic Behavior ##
 
 Some transformations, such as ¡replace=old:new¡, may introduce conflicts where multiple source files map to the same destination name.
 This can lead to race conditions when performing concurrent transfers. It is up to the user to anticipate these.
 * If two files from the source are transformed into the same name at the destination, the final state may be non-deterministic.
 * Running rclone check after a sync using such transformations may erroneously report missing or differing files due to overwritten results.
 
-To minimize risks, users should:
-* Carefully review transformations that may introduce conflicts.
-* Use ¡--dry-run¡ to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
-* Avoid transformations that cause multiple distinct source files to map to the same destination name.
-* Consider disabling concurrency with ¡--transfers=1¡ if necessary.
-* Certain transformations (e.g. ¡prefix¡) will have a multiplying effect every time they are used. Avoid these when using ¡bisync¡.
+* To minimize risks, users should:
+    * Carefully review transformations that may introduce conflicts.
+    * Use ¡--dry-run¡ to inspect changes before executing a sync (but keep in mind that it won't show the effect of non-deterministic transformations).
+    * Avoid transformations that cause multiple distinct source files to map to the same destination name.
+    * Consider disabling concurrency with ¡--transfers=1¡ if necessary.
+    * Certain transformations (e.g. ¡prefix¡) will have a multiplying effect every time they are used. Avoid these when using ¡bisync¡.
 
 	`, "¡", "`"),
 	Annotations: map[string]string{

cmd/convmv/convmv_test.go

@@ -141,7 +141,7 @@ func TestTransform(t *testing.T) {
 }
 
 // const alphabet = "ƀɀɠʀҠԀڀڠݠހ߀ကႠᄀᄠᅀᆀᇠሀሠበዠጠᎠᏀᐠᑀᑠᒀᒠᓀᓠᔀᔠᕀᕠᖀᖠᗀᗠᘀᘠᙀᚠᛀកᠠᡀᣀᦀ᧠ᨠᯀᰀᴀ⇠⋀⍀⍠⎀⎠⏀␀─┠╀╠▀■◀◠☀☠♀♠⚀⚠⛀⛠✀✠❀➀➠⠀⠠⡀⡠⢀⢠⣀⣠⤀⤠⥀⥠⦠⨠⩀⪀⪠⫠⬀⬠⭀ⰀⲀⲠⳀⴀⵀ⺠⻀㇀㐀㐠㑀㑠㒀㒠㓀㓠㔀㔠㕀㕠㖀㖠㗀㗠㘀㘠㙀㙠㚀㚠㛀㛠㜀㜠㝀㝠㞀㞠㟀㟠㠀㠠㡀㡠㢀㢠㣀㣠㤀㤠㥀㥠㦀㦠㧀㧠㨀㨠㩀㩠㪀㪠㫀㫠㬀㬠㭀㭠㮀㮠㯀㯠㰀㰠㱀㱠㲀㲠㳀㳠㴀㴠㵀㵠㶀㶠㷀㷠㸀㸠㹀㹠㺀㺠㻀㻠㼀㼠㽀㽠㾀㾠㿀㿠䀀䀠䁀䁠䂀䂠䃀䃠䄀䄠䅀䅠䆀䆠䇀䇠䈀䈠䉀䉠䊀䊠䋀䋠䌀䌠䍀䍠䎀䎠䏀䏠䐀䐠䑀䑠䒀䒠䓀䓠䔀䔠䕀䕠䖀䖠䗀䗠䘀䘠䙀䙠䚀䚠䛀䛠䜀䜠䝀䝠䞀䞠䟀䟠䠀䠠䡀䡠䢀䢠䣀䣠䤀䤠䥀䥠䦀䦠䧀䧠䨀䨠䩀䩠䪀䪠䫀䫠䬀䬠䭀䭠䮀䮠䯀䯠䰀䰠䱀䱠䲀䲠䳀䳠䴀䴠䵀䵠䶀䷀䷠一丠乀习亀亠什仠伀传佀你侀侠俀俠倀倠偀偠傀傠僀僠儀儠兀兠冀冠净几刀删剀剠劀加勀勠匀匠區占厀厠叀叠吀吠呀呠咀咠哀哠唀唠啀啠喀喠嗀嗠嘀嘠噀噠嚀嚠囀因圀圠址坠垀垠埀埠堀堠塀塠墀墠壀壠夀夠奀奠妀妠姀姠娀娠婀婠媀媠嫀嫠嬀嬠孀孠宀宠寀寠尀尠局屠岀岠峀峠崀崠嵀嵠嶀嶠巀巠帀帠幀幠庀庠廀廠开张彀彠往徠忀忠怀怠恀恠悀悠惀惠愀愠慀慠憀憠懀懠戀戠所扠技抠拀拠挀挠捀捠掀掠揀揠搀搠摀摠撀撠擀擠攀攠敀敠斀斠旀无昀映晀晠暀暠曀曠最朠杀杠枀枠柀柠栀栠桀桠梀梠检棠椀椠楀楠榀榠槀槠樀樠橀橠檀檠櫀櫠欀欠歀歠殀殠毀毠氀氠汀池沀沠泀泠洀洠浀浠涀涠淀淠渀渠湀湠満溠滀滠漀漠潀潠澀澠激濠瀀瀠灀灠炀炠烀烠焀焠煀煠熀熠燀燠爀爠牀牠犀犠狀狠猀猠獀獠玀玠珀珠琀琠瑀瑠璀璠瓀瓠甀甠畀畠疀疠痀痠瘀瘠癀癠皀皠盀盠眀眠着睠瞀瞠矀矠砀砠础硠碀碠磀磠礀礠祀祠禀禠秀秠稀稠穀穠窀窠竀章笀笠筀筠简箠節篠簀簠籀籠粀粠糀糠紀素絀絠綀綠緀締縀縠繀繠纀纠绀绠缀缠罀罠羀羠翀翠耀耠聀聠肀肠胀胠脀脠腀腠膀膠臀臠舀舠艀艠芀芠苀苠茀茠荀荠莀莠菀菠萀萠葀葠蒀蒠蓀蓠蔀蔠蕀蕠薀薠藀藠蘀蘠虀虠蚀蚠蛀蛠蜀蜠蝀蝠螀螠蟀蟠蠀蠠血衠袀袠裀裠褀褠襀襠覀覠觀觠言訠詀詠誀誠諀諠謀謠譀譠讀讠诀诠谀谠豀豠貀負賀賠贀贠赀赠趀趠跀跠踀踠蹀蹠躀躠軀軠輀輠轀轠辀辠迀迠退造遀遠邀邠郀郠鄀鄠酀酠醀醠釀釠鈀鈠鉀鉠銀銠鋀鋠錀錠鍀鍠鎀鎠鏀鏠鐀鐠鑀鑠钀钠铀铠销锠镀镠門閠闀闠阀阠陀陠隀隠雀雠需霠靀靠鞀鞠韀韠頀頠顀顠颀颠飀飠餀餠饀饠馀馠駀駠騀騠驀驠骀骠髀髠鬀鬠魀魠鮀鮠鯀鯠鰀鰠鱀鱠鲀鲠鳀鳠鴀鴠鵀鵠鶀鶠鷀鷠鸀鸠鹀鹠麀麠黀黠鼀鼠齀齠龀龠ꀀꀠꁀꁠꂀꂠꃀꃠꄀꄠꅀꅠꆀꆠꇀꇠꈀꈠꉀꉠꊀꊠꋀꋠꌀꌠꍀꍠꎀꎠꏀꏠꐀꐠꑀꑠ꒠ꔀꔠꕀꕠꖀꖠꗀꗠꙀꚠꛀ꜀꜠ꝀꞀꡀ測試_Русский___ě_áñ"
-const alphabet = "abcdefg123456789Ü"
+const alphabet = "abcdefg123456789"
 
 var extras = []string{"apple", "banana", "appleappleapplebanana", "splitbananasplit"}
 
@@ -251,25 +251,3 @@ func detectEncoding(s string) string {
 	}
 	return "OTHER"
 }
-
-func TestUnicodeEquivalence(t *testing.T) {
-	r := fstest.NewRun(t)
-	defer r.Finalise()
-
-	ctx := context.Background()
-	r.Mkdir(ctx, r.Fremote)
-	const remote = "Über"
-	item := r.WriteObject(ctx, remote, "", t1)
-
-	obj, err := r.Fremote.NewObject(ctx, remote) // can't use r.CheckRemoteListing here as it forces NFC
-	require.NoError(t, err)
-	require.NotEmpty(t, obj)
-
-	err = transform.SetOptions(ctx, "all,nfc")
-	require.NoError(t, err)
-
-	err = sync.Transform(ctx, r.Fremote, true, true)
-	assert.NoError(t, err)
-	item.Path = norm.NFC.String(item.Path)
-	r.CheckRemoteListing(t, []fstest.Item{item}, nil)
-}

cmd/copy/copy.go

@@ -8,23 +8,18 @@ import (
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
 
 var (
 	createEmptySrcDirs = false
-	loggerOpt          = operations.LoggerOpt{}
-	loggerFlagsOpt     = operationsflags.AddLoggerFlagsOptions{}
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after copy", "")
-	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
-	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -95,30 +90,19 @@ for more info.
 **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
 
 **Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
-
-`, "|", "`") + operationsflags.Help(),
+`, "|", "`"),
 	Annotations: map[string]string{
 		"groups": "Copy,Filter,Listing,Important",
 	},
 	Run: func(command *cobra.Command, args []string) {
+
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
-			ctx := context.Background()
-			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
-			if err != nil {
-				return err
-			}
-			defer close()
-
-			if loggerFlagsOpt.AnySet() {
-				ctx = operations.WithSyncLogger(ctx, loggerOpt)
-			}
-
 			if srcFileName == "" {
-				return sync.CopyDir(ctx, fdst, fsrc, createEmptySrcDirs)
+				return sync.CopyDir(context.Background(), fdst, fsrc, createEmptySrcDirs)
 			}
-			return operations.CopyFile(ctx, fdst, fsrc, srcFileName, srcFileName)
+			return operations.CopyFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
 		})
 	},
 }

cmd/copyto/copyto.go

@@ -6,21 +6,12 @@ import (
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
 
-var (
-	loggerOpt      = operations.LoggerOpt{}
-	loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
-)
-
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
-	cmdFlags := commandDefinition.Flags()
-	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
-	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -55,8 +46,7 @@ the destination.
 *If you are looking to copy just a byte range of a file, please see 'rclone cat --offset X --count Y'*
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
-
-` + operationsflags.Help(),
+`,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.35",
 		"groups":            "Copy,Filter,Listing,Important",
@@ -65,21 +55,10 @@ the destination.
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 		cmd.Run(true, true, command, func() error {
-			ctx := context.Background()
-			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
-			if err != nil {
-				return err
-			}
-			defer close()
-
-			if loggerFlagsOpt.AnySet() {
-				ctx = operations.WithSyncLogger(ctx, loggerOpt)
-			}
-
 			if srcFileName == "" {
-				return sync.CopyDir(ctx, fdst, fsrc, false)
+				return sync.CopyDir(context.Background(), fdst, fsrc, false)
 			}
-			return operations.CopyFile(ctx, fdst, fsrc, dstFileName, srcFileName)
+			return operations.CopyFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
 		})
 	},
 }

cmd/mountlib/mount.md

@@ -240,7 +240,7 @@ to run as the SYSTEM account. A third alternative is to use the
 Read more in the [install documentation](https://rclone.org/install/).
 Note that when running rclone as another user, it will not use
 the configuration file from your profile unless you tell it to
-with the [`--config`](https://rclone.org/docs/#config-string) option.
+with the [`--config`](https://rclone.org/docs/#config-config-file) option.
 Note also that it is now the SYSTEM account that will have the owner
 permissions, and other accounts will have permissions according to the
 group or others scopes. As mentioned above, these will then not get the

cmd/move/move.go

@@ -8,7 +8,6 @@ import (
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
@@ -17,8 +16,6 @@ import (
 var (
 	deleteEmptySrcDirs = false
 	createEmptySrcDirs = false
-	loggerOpt          = operations.LoggerOpt{}
-	loggerFlagsOpt     = operationsflags.AddLoggerFlagsOptions{}
 )
 
 func init() {
@@ -26,8 +23,6 @@ func init() {
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &deleteEmptySrcDirs, "delete-empty-src-dirs", "", deleteEmptySrcDirs, "Delete empty source dirs after move", "")
 	flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after move", "")
-	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
-	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -71,8 +66,7 @@ for more info.
 |--dry-run| or the |--interactive|/|-i| flag.
 
 **Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
-
-`, "|", "`") + operationsflags.Help(),
+`, "|", "`"),
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.19",
 		"groups":            "Filter,Listing,Important,Copy",
@@ -81,21 +75,10 @@ for more info.
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
-			ctx := context.Background()
-			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
-			if err != nil {
-				return err
-			}
-			defer close()
-
-			if loggerFlagsOpt.AnySet() {
-				ctx = operations.WithSyncLogger(ctx, loggerOpt)
-			}
-
 			if srcFileName == "" {
-				return sync.MoveDir(ctx, fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
+				return sync.MoveDir(context.Background(), fdst, fsrc, deleteEmptySrcDirs, createEmptySrcDirs)
 			}
-			return operations.MoveFile(ctx, fdst, fsrc, srcFileName, srcFileName)
+			return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName)
 		})
 	},
 }

cmd/moveto/moveto.go

@@ -6,21 +6,12 @@ import (
 
 	"github.com/rclone/rclone/cmd"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/operations/operationsflags"
 	"github.com/rclone/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
 
-var (
-	loggerOpt      = operations.LoggerOpt{}
-	loggerFlagsOpt = operationsflags.AddLoggerFlagsOptions{}
-)
-
 func init() {
 	cmd.Root.AddCommand(commandDefinition)
-	cmdFlags := commandDefinition.Flags()
-	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
-	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
 }
 
 var commandDefinition = &cobra.Command{
@@ -56,8 +47,7 @@ successful transfer.
 ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
 
 **Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
-
-` + operationsflags.Help(),
+`,
 	Annotations: map[string]string{
 		"versionIntroduced": "v1.35",
 		"groups":            "Filter,Listing,Important,Copy",
@@ -67,21 +57,10 @@ successful transfer.
 		fsrc, srcFileName, fdst, dstFileName := cmd.NewFsSrcDstFiles(args)
 
 		cmd.Run(true, true, command, func() error {
-			ctx := context.Background()
-			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
-			if err != nil {
-				return err
-			}
-			defer close()
-
-			if loggerFlagsOpt.AnySet() {
-				ctx = operations.WithSyncLogger(ctx, loggerOpt)
-			}
-
 			if srcFileName == "" {
-				return sync.MoveDir(ctx, fdst, fsrc, false, false)
+				return sync.MoveDir(context.Background(), fdst, fsrc, false, false)
 			}
-			return operations.MoveFile(ctx, fdst, fsrc, dstFileName, srcFileName)
+			return operations.MoveFile(context.Background(), fdst, fsrc, dstFileName, srcFileName)
 		})
 	},
 }

cmd/ncdu/ncdu.go

@@ -6,8 +6,6 @@ package ncdu
 import (
 	"context"
 	"fmt"
-	"log/slog"
-	"os"
 	"path"
 	"reflect"
 	"sort"
@@ -927,19 +925,23 @@ func (u *UI) Run() error {
 		return fmt.Errorf("screen init: %w", err)
 	}
 
-	// Hijack log output so that it doesn't corrupt the screen.
-	if !log.Redirected() {
-		var logs []string
-		log.Handler.SetOutput(func(level slog.Level, text string) {
+	// Hijack fs.LogOutput so that it doesn't corrupt the screen.
+	if logOutput := fs.LogOutput; !log.Redirected() {
+		type log struct {
+			text  string
+			level fs.LogLevel
+		}
+		var logs []log
+		fs.LogOutput = func(level fs.LogLevel, text string) {
 			if len(logs) > 100 {
 				logs = logs[len(logs)-100:]
 			}
-			logs = append(logs, text)
-		})
+			logs = append(logs, log{level: level, text: text})
+		}
 		defer func() {
-			log.Handler.ResetOutput()
-			for _, text := range logs {
-				_, _ = os.Stderr.WriteString(text)
+			fs.LogOutput = logOutput
+			for i := range logs {
+				logOutput(logs[i].level, logs[i].text)
 			}
 		}()
 	}

cmd/progress.go

@@ -5,11 +5,11 @@ package cmd
 import (
 	"bytes"
 	"fmt"
-	"log/slog"
 	"strings"
 	"sync"
 	"time"
 
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/operations"
@@ -19,6 +19,8 @@ import (
 const (
 	// interval between progress prints
 	defaultProgressInterval = 500 * time.Millisecond
+	// time format for logging
+	logTimeFormat = "2006/01/02 15:04:05"
 )
 
 // startProgress starts the progress bar printing
@@ -26,13 +28,15 @@ const (
 // It returns a func which should be called to stop the stats.
 func startProgress() func() {
 	stopStats := make(chan struct{})
+	oldLogOutput := fs.LogOutput
 	oldSyncPrint := operations.SyncPrintf
 
 	if !log.Redirected() {
 		// Intercept the log calls if not logging to file or syslog
-		log.Handler.SetOutput(func(level slog.Level, text string) {
-			printProgress(text)
-		})
+		fs.LogOutput = func(level fs.LogLevel, text string) {
+			printProgress(fmt.Sprintf("%s %-6s: %s", time.Now().Format(logTimeFormat), level, text))
+
+		}
 	}
 
 	// Intercept output from functions such as HashLister to stdout
@@ -56,10 +60,7 @@ func startProgress() func() {
 			case <-stopStats:
 				ticker.Stop()
 				printProgress("")
-				if !log.Redirected() {
-					// Reset intercept of the log calls
-					log.Handler.ResetOutput()
-				}
+				fs.LogOutput = oldLogOutput
 				operations.SyncPrintf = oldSyncPrint
 				fmt.Println("")
 				return

cmd/serve/s3/list.go

@@ -28,8 +28,7 @@ func (b *s3Backend) entryListR(_vfs *vfs.VFS, bucket, fdPath, name string, addPr
 
 		if entry.IsDir() {
 			if addPrefix {
-				prefixWithTrailingSlash := objectPath + "/"
-				response.AddPrefix(prefixWithTrailingSlash)
+				response.AddPrefix(objectPath)
 				continue
 			}
 			err := b.entryListR(_vfs, bucket, path.Join(fdPath, object), "", false, response)

cmd/serve/s3/utils.go

@@ -45,10 +45,6 @@ func getFileHashByte(node any, hashType hash.Type) []byte {
 }
 
 func getFileHash(node any, hashType hash.Type) string {
-	if hashType == hash.None {
-		return ""
-	}
-
 	var o fs.Object
 
 	switch b := node.(type) {

cmd/serve/sftp/connection.go

@@ -95,44 +95,61 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
 		if err != nil {
 			return fmt.Errorf("send output failed: %w", err)
 		}
-	case "md5sum":
-		return c.handleHashsumCommand(ctx, out, hash.MD5, args)
-	case "sha1sum":
-		return c.handleHashsumCommand(ctx, out, hash.SHA1, args)
-	case "crc32":
-		return c.handleHashsumCommand(ctx, out, hash.CRC32, args)
-	case "sha256sum":
-		return c.handleHashsumCommand(ctx, out, hash.SHA256, args)
-	case "b3sum":
-		return c.handleHashsumCommand(ctx, out, hash.BLAKE3, args)
-	case "xxh128sum":
-		return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
-	case "xxhsum":
-		argv := strings.SplitN(args, " ", 2)
-		if len(argv) == 0 || argv[0] != "-H2" {
-			return fmt.Errorf("%q not implemented", command)
+	case "md5sum", "sha1sum":
+		ht := hash.MD5
+		if binary == "sha1sum" {
+			ht = hash.SHA1
 		}
-		if len(argv) > 1 {
-			args = argv[1]
-		} else {
-			args = ""
+		if !c.vfs.Fs().Hashes().Contains(ht) {
+			return fmt.Errorf("%v hash not supported", ht)
 		}
-		return c.handleHashsumCommand(ctx, out, hash.XXH128, args)
-	case "rclone":
-		argv := strings.SplitN(args, " ", 3)
-		if len(argv) > 1 && argv[0] == "hashsum" {
-			var ht hash.Type
-			if err := ht.Set(argv[1]); err != nil {
-				return err
-			}
-			if len(argv) > 2 {
-				args = argv[2]
+		var hashSum string
+		if args == "" {
+			// empty hash for no input
+			if ht == hash.MD5 {
+				hashSum = "d41d8cd98f00b204e9800998ecf8427e"
 			} else {
-				args = ""
+				hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
+			}
+			args = "-"
+		} else {
+			node, err := c.vfs.Stat(args)
+			if err != nil {
+				return fmt.Errorf("hash failed finding file %q: %w", args, err)
+			}
+			if node.IsDir() {
+				return errors.New("can't hash directory")
+			}
+			o, ok := node.DirEntry().(fs.ObjectInfo)
+			if !ok {
+				fs.Debugf(args, "File uploading - reading hash from VFS cache")
+				in, err := node.Open(os.O_RDONLY)
+				if err != nil {
+					return fmt.Errorf("hash vfs open failed: %w", err)
+				}
+				defer func() {
+					_ = in.Close()
+				}()
+				h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+				if err != nil {
+					return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
+				}
+				_, err = io.Copy(h, in)
+				if err != nil {
+					return fmt.Errorf("hash vfs copy failed: %w", err)
+				}
+				hashSum = h.Sums()[ht]
+			} else {
+				hashSum, err = o.Hash(ctx, ht)
+				if err != nil {
+					return fmt.Errorf("hash failed: %w", err)
+				}
 			}
-			return c.handleHashsumCommand(ctx, out, ht, args)
 		}
-		return fmt.Errorf("%q not implemented", command)
+		_, err = fmt.Fprintf(out, "%s  %s\n", hashSum, args)
+		if err != nil {
+			return fmt.Errorf("send output failed: %w", err)
+		}
 	case "echo":
 		// Special cases for legacy rclone command detection.
 		// Before rclone v1.49.0 the sftp backend used "echo 'abc' | md5sum" when
@@ -172,74 +189,6 @@ func (c *conn) execCommand(ctx context.Context, out io.Writer, command string) (
 	return nil
 }
 
-// handleHashsumCommand is a helper to execCommand for common functionality of hashsum related commands
-func (c *conn) handleHashsumCommand(ctx context.Context, out io.Writer, ht hash.Type, args string) (err error) {
-	if !c.vfs.Fs().Hashes().Contains(ht) {
-		return fmt.Errorf("%v hash not supported", ht)
-	}
-	var hashSum string
-	if args == "" {
-		// empty hash for no input
-		switch ht {
-		case hash.MD5:
-			hashSum = "d41d8cd98f00b204e9800998ecf8427e"
-		case hash.SHA1:
-			hashSum = "da39a3ee5e6b4b0d3255bfef95601890afd80709"
-		case hash.CRC32:
-			hashSum = "00000000"
-		case hash.SHA256:
-			hashSum = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
-		case hash.BLAKE3:
-			hashSum = "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"
-		case hash.XXH3:
-			hashSum = "2d06800538d394c2"
-		case hash.XXH128:
-			hashSum = "99aa06d3014798d86001c324468d497f"
-		default:
-			return fmt.Errorf("%v hash not implemented", ht)
-		}
-		args = "-"
-	} else {
-		node, err := c.vfs.Stat(args)
-		if err != nil {
-			return fmt.Errorf("hash failed finding file %q: %w", args, err)
-		}
-		if node.IsDir() {
-			return errors.New("can't hash directory")
-		}
-		o, ok := node.DirEntry().(fs.ObjectInfo)
-		if !ok {
-			fs.Debugf(args, "File uploading - reading hash from VFS cache")
-			in, err := node.Open(os.O_RDONLY)
-			if err != nil {
-				return fmt.Errorf("hash vfs open failed: %w", err)
-			}
-			defer func() {
-				_ = in.Close()
-			}()
-			h, err := hash.NewMultiHasherTypes(hash.NewHashSet(ht))
-			if err != nil {
-				return fmt.Errorf("hash vfs create multi-hasher failed: %w", err)
-			}
-			_, err = io.Copy(h, in)
-			if err != nil {
-				return fmt.Errorf("hash vfs copy failed: %w", err)
-			}
-			hashSum = h.Sums()[ht]
-		} else {
-			hashSum, err = o.Hash(ctx, ht)
-			if err != nil {
-				return fmt.Errorf("hash failed: %w", err)
-			}
-		}
-	}
-	_, err = fmt.Fprintf(out, "%s  %s\n", hashSum, args)
-	if err != nil {
-		return fmt.Errorf("send output failed: %w", err)
-	}
-	return nil
-}
-
 // handle a new incoming channel request
 func (c *conn) handleChannel(newChannel ssh.NewChannel) {
 	fs.Debugf(c.what, "Incoming channel: %s\n", newChannel.ChannelType())

cmd/sync/sync.go

@@ -3,9 +3,13 @@ package sync
 
 import (
 	"context"
-	"strings"
+	"io"
+	"os"
+
+	mutex "sync" // renamed as "sync" already in use
 
 	"github.com/rclone/rclone/cmd"
+	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/flags"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/operations/operationsflags"
@@ -15,7 +19,7 @@ import (
 
 var (
 	createEmptySrcDirs = false
-	loggerOpt          = operations.LoggerOpt{}
+	opt                = operations.LoggerOpt{}
 	loggerFlagsOpt     = operationsflags.AddLoggerFlagsOptions{}
 )
 
@@ -23,15 +27,119 @@ func init() {
 	cmd.Root.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
 	flags.BoolVarP(cmdFlags, &createEmptySrcDirs, "create-empty-src-dirs", "", createEmptySrcDirs, "Create empty source dirs on destination after sync", "")
-	operationsflags.AddLoggerFlags(cmdFlags, &loggerOpt, &loggerFlagsOpt)
-	loggerOpt.LoggerFn = operations.NewDefaultLoggerFn(&loggerOpt)
+	operationsflags.AddLoggerFlags(cmdFlags, &opt, &loggerFlagsOpt)
+	// TODO: add same flags to move and copy
+}
+
+var lock mutex.Mutex
+
+func syncLoggerFn(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	if err == fs.ErrorIsDir && !opt.FilesOnly && opt.DestAfter != nil {
+		opt.PrintDestAfter(ctx, sigil, src, dst, err)
+		return
+	}
+
+	_, srcOk := src.(fs.Object)
+	_, dstOk := dst.(fs.Object)
+	var filename string
+	if !srcOk && !dstOk {
+		return
+	} else if srcOk && !dstOk {
+		filename = src.String()
+	} else {
+		filename = dst.String()
+	}
+
+	if sigil.Writer(opt) != nil {
+		operations.SyncFprintf(sigil.Writer(opt), "%s\n", filename)
+	}
+	if opt.Combined != nil {
+		operations.SyncFprintf(opt.Combined, "%c %s\n", sigil, filename)
+		fs.Debugf(nil, "Sync Logger: %s: %c %s\n", sigil.String(), sigil, filename)
+	}
+	if opt.DestAfter != nil {
+		opt.PrintDestAfter(ctx, sigil, src, dst, err)
+	}
+}
+
+// GetSyncLoggerOpt gets the options corresponding to the logger flags
+func GetSyncLoggerOpt(ctx context.Context, fdst fs.Fs, command *cobra.Command) (operations.LoggerOpt, func(), error) {
+	closers := []io.Closer{}
+
+	opt.LoggerFn = syncLoggerFn
+	if opt.TimeFormat == "max" {
+		opt.TimeFormat = operations.FormatForLSFPrecision(fdst.Precision())
+	}
+	opt.SetListFormat(ctx, command.Flags())
+	opt.NewListJSON(ctx, fdst, "")
+
+	open := func(name string, pout *io.Writer) error {
+		if name == "" {
+			return nil
+		}
+		if name == "-" {
+			*pout = os.Stdout
+			return nil
+		}
+		out, err := os.Create(name)
+		if err != nil {
+			return err
+		}
+		*pout = out
+		closers = append(closers, out)
+		return nil
+	}
+
+	if err := open(loggerFlagsOpt.Combined, &opt.Combined); err != nil {
+		return opt, nil, err
+	}
+	if err := open(loggerFlagsOpt.MissingOnSrc, &opt.MissingOnSrc); err != nil {
+		return opt, nil, err
+	}
+	if err := open(loggerFlagsOpt.MissingOnDst, &opt.MissingOnDst); err != nil {
+		return opt, nil, err
+	}
+	if err := open(loggerFlagsOpt.Match, &opt.Match); err != nil {
+		return opt, nil, err
+	}
+	if err := open(loggerFlagsOpt.Differ, &opt.Differ); err != nil {
+		return opt, nil, err
+	}
+	if err := open(loggerFlagsOpt.ErrFile, &opt.Error); err != nil {
+		return opt, nil, err
+	}
+	if err := open(loggerFlagsOpt.DestAfter, &opt.DestAfter); err != nil {
+		return opt, nil, err
+	}
+
+	close := func() {
+		for _, closer := range closers {
+			err := closer.Close()
+			if err != nil {
+				fs.Errorf(nil, "Failed to close report output: %v", err)
+			}
+		}
+	}
+
+	return opt, close, nil
+}
+
+func anyNotBlank(s ...string) bool {
+	for _, x := range s {
+		if x != "" {
+			return true
+		}
+	}
+	return false
 }
 
 var commandDefinition = &cobra.Command{
 	Use:   "sync source:path dest:path",
 	Short: `Make source and dest identical, modifying destination only.`,
-	// Warning! "|" will be replaced by backticks below
-	Long: strings.ReplaceAll(`Sync the source to the destination, changing the destination
+	Long: `Sync the source to the destination, changing the destination
 only.  Doesn't transfer files that are identical on source and
 destination, testing by size and modification time or MD5SUM.
 Destination is updated to match source, including deleting files
@@ -40,7 +148,7 @@ want to delete files from destination, use the
 [copy](/commands/rclone_copy/) command instead.
 
 **Important**: Since this can cause data loss, test first with the
-|--dry-run| or the |--interactive|/|i| flag.
+` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
 
     rclone sync --interactive SOURCE remote:DESTINATION
 
@@ -63,18 +171,55 @@ destination that is inside the source directory.
 
 Rclone will sync the modification times of files and directories if
 the backend supports it. If metadata syncing is required then use the
-|--metadata| flag.
+` + "`--metadata`" + ` flag.
 
 Note that the modification time and metadata for the root directory
 will **not** be synced. See https://github.com/rclone/rclone/issues/7652
 for more info.
 
-**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics
+**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
 
-**Note**: Use the |rclone dedupe| command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
+**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
 See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14372) for more info.
 
-`, "|", "`") + operationsflags.Help(),
+## Logger Flags
+
+The ` + "`--differ`" + `, ` + "`--missing-on-dst`" + `, ` + "`--missing-on-src`" + `, ` +
+		"`--match`" + ` and ` + "`--error`" + ` flags write paths, one per line, to the file name (or
+stdout if it is ` + "`-`" + `) supplied. What they write is described in the
+help below. For example ` + "`--differ`" + ` will write all paths which are
+present on both the source and destination but different.
+
+The ` + "`--combined`" + ` flag will write a file (or stdout) which contains all
+file paths with a symbol and then a space and then the path to tell
+you what happened to it. These are reminiscent of diff files.
+
+- ` + "`= path`" + ` means path was found in source and destination and was identical
+- ` + "`- path`" + ` means path was missing on the source, so only in the destination
+- ` + "`+ path`" + ` means path was missing on the destination, so only in the source
+- ` + "`* path`" + ` means path was present in source and destination but different.
+- ` + "`! path`" + ` means there was an error reading or hashing the source or dest.
+
+The ` + "`--dest-after`" + ` flag writes a list file using the same format flags
+as [` + "`lsf`" + `](/commands/rclone_lsf/#synopsis) (including [customizable options
+for hash, modtime, etc.](/commands/rclone_lsf/#synopsis))
+Conceptually it is similar to rsync's ` + "`--itemize-changes`" + `, but not identical
+-- it should output an accurate list of what will be on the destination
+after the sync.
+
+Note that these logger flags have a few limitations, and certain scenarios
+are not currently supported:
+
+- ` + "`--max-duration`" + ` / ` + "`CutoffModeHard`" + `
+- ` + "`--compare-dest`" + ` / ` + "`--copy-dest`" + `
+- server-side moves of an entire dir at once
+- High-level retries, because there would be duplicates (use ` + "`--retries 1`" + ` to disable)
+- Possibly some unusual error scenarios
+
+Note also that each file is logged during the sync, as opposed to after, so it
+is most useful as a predictor of what SHOULD happen to each file
+(which may or may not match what actually DID.)
+`,
 	Annotations: map[string]string{
 		"groups": "Sync,Copy,Filter,Listing,Important",
 	},
@@ -83,14 +228,15 @@ See [this forum post](https://forum.rclone.org/t/sync-not-clearing-duplicates/14
 		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
 			ctx := context.Background()
-			close, err := operationsflags.ConfigureLoggers(ctx, fdst, command, &loggerOpt, loggerFlagsOpt)
+			opt, close, err := GetSyncLoggerOpt(ctx, fdst, command)
 			if err != nil {
 				return err
 			}
 			defer close()
 
-			if loggerFlagsOpt.AnySet() {
-				ctx = operations.WithSyncLogger(ctx, loggerOpt)
+			if anyNotBlank(loggerFlagsOpt.Combined, loggerFlagsOpt.MissingOnSrc, loggerFlagsOpt.MissingOnDst,
+				loggerFlagsOpt.Match, loggerFlagsOpt.Differ, loggerFlagsOpt.ErrFile, loggerFlagsOpt.DestAfter) {
+				ctx = operations.WithSyncLogger(ctx, opt)
 			}
 
 			if srcFileName == "" {

cmd/test/changenotify/changenotify.go

@@ -14,13 +14,13 @@ import (
 )
 
 var (
-	pollInterval = fs.Duration(10 * time.Second)
+	pollInterval = 10 * time.Second
 )
 
 func init() {
 	test.Command.AddCommand(commandDefinition)
 	cmdFlags := commandDefinition.Flags()
-	flags.FVarP(cmdFlags, &pollInterval, "poll-interval", "", "Time to wait between polling for changes", "")
+	flags.DurationVarP(cmdFlags, &pollInterval, "poll-interval", "", pollInterval, "Time to wait between polling for changes", "")
 }
 
 var commandDefinition = &cobra.Command{
@@ -39,7 +39,7 @@ var commandDefinition = &cobra.Command{
 		if do := features.ChangeNotify; do != nil {
 			pollChan := make(chan time.Duration)
 			do(ctx, changeNotify, pollChan)
-			pollChan <- time.Duration(pollInterval)
+			pollChan <- pollInterval
 			fs.Logf(nil, "Waiting for changes, polling every %v", pollInterval)
 		} else {
 			return errors.New("poll-interval is not supported by this remote")

cmd/test/info/info.go

@@ -40,7 +40,7 @@ var (
 	checkStreaming     bool
 	checkBase32768     bool
 	all                bool
-	uploadWait         fs.Duration
+	uploadWait         time.Duration
 	positionLeftRe     = regexp.MustCompile(`(?s)^(.*)-position-left-([[:xdigit:]]+)$`)
 	positionMiddleRe   = regexp.MustCompile(`(?s)^position-middle-([[:xdigit:]]+)-(.*)-$`)
 	positionRightRe    = regexp.MustCompile(`(?s)^position-right-([[:xdigit:]]+)-(.*)$`)
@@ -52,7 +52,7 @@ func init() {
 	flags.StringVarP(cmdFlags, &writeJSON, "write-json", "", "", "Write results to file", "")
 	flags.BoolVarP(cmdFlags, &checkNormalization, "check-normalization", "", false, "Check UTF-8 Normalization", "")
 	flags.BoolVarP(cmdFlags, &checkControl, "check-control", "", false, "Check control characters", "")
-	flags.FVarP(cmdFlags, &uploadWait, "upload-wait", "", "Wait after writing a file", "")
+	flags.DurationVarP(cmdFlags, &uploadWait, "upload-wait", "", 0, "Wait after writing a file", "")
 	flags.BoolVarP(cmdFlags, &checkLength, "check-length", "", false, "Check max filename length", "")
 	flags.BoolVarP(cmdFlags, &checkStreaming, "check-streaming", "", false, "Check uploads with indeterminate file size", "")
 	flags.BoolVarP(cmdFlags, &checkBase32768, "check-base32768", "", false, "Check can store all possible base32768 characters", "")
@@ -204,7 +204,7 @@ func (r *results) writeFile(path string) (fs.Object, error) {
 	src := object.NewStaticObjectInfo(path, time.Now(), int64(len(contents)), true, nil, r.f)
 	obj, err := r.f.Put(r.ctx, bytes.NewBufferString(contents), src)
 	if uploadWait > 0 {
-		time.Sleep(time.Duration(uploadWait))
+		time.Sleep(uploadWait)
 	}
 	return obj, err
 }

cmdtest/environment_test.go

@@ -48,7 +48,7 @@ func TestEnvironmentVariables(t *testing.T) {
 	env = "RCLONE_LOG_LEVEL=DEBUG"
 	out, err = rcloneEnv(env, "version", "--quiet")
 	if assert.Error(t, err) {
-		assert.Contains(t, out, " DEBUG ")
+		assert.Contains(t, out, " DEBUG : ")
 		assert.Contains(t, out, "Can't set -q and --log-level")
 		assert.Contains(t, "exit status 1", err.Error())
 	}
@@ -329,7 +329,7 @@ func TestEnvironmentVariables(t *testing.T) {
 	jsonLogOK := func() {
 		t.Helper()
 		if assert.NoError(t, err) {
-			assert.Contains(t, out, `"level":"debug"`)
+			assert.Contains(t, out, `{"level":"debug",`)
 			assert.Contains(t, out, `"msg":"Version `)
 			assert.Contains(t, out, `"}`)
 		}

docs/content/_index.md

@@ -123,9 +123,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Dropbox" home="https://www.dropbox.com/" config="/dropbox/" >}}
 {{< provider name="Enterprise File Fabric" home="https://storagemadeeasy.com/about/" config="/filefabric/" >}}
 {{< provider name="Fastmail Files" home="https://www.fastmail.com/" config="/webdav/#fastmail-files" >}}
-{{< provider name="FileLu Cloud Storage" home="https://filelu.com/" config="/filelu/" >}}
 {{< provider name="Files.com" home="https://www.files.com/" config="/filescom/" >}}
-{{< provider name="FlashBlade" home="https://www.purestorage.com/products/unstructured-data-storage.html" config="/s3/#pure-storage-flashblade" >}}
 {{< provider name="FTP" home="https://en.wikipedia.org/wiki/File_Transfer_Protocol" config="/ftp/" >}}
 {{< provider name="Gofile" home="https://gofile.io/" config="/gofile/" >}}
 {{< provider name="Google Cloud Storage" home="https://cloud.google.com/storage/" config="/googlecloudstorage/" >}}
@@ -150,8 +148,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="Magalu" home="https://magalu.cloud/object-storage/" config="/s3/#magalu" >}}
 {{< provider name="Mail.ru Cloud" home="https://cloud.mail.ru/" config="/mailru/" >}}
 {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}}
-{{< provider name="MEGA" home="https://mega.nz/" config="/mega/" >}}
-{{< provider name="MEGA S4" home="https://mega.io/objectstorage" config="/s3/#mega" >}}
+{{< provider name="Mega" home="https://mega.nz/" config="/mega/" >}}
 {{< provider name="Memory" home="/memory/" config="/memory/" >}}
 {{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}}
 {{< provider name="Microsoft Azure Files Storage" home="https://azure.microsoft.com/en-us/services/storage/files/" config="/azurefiles/" >}}
@@ -197,7 +194,6 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="WebDAV" home="https://en.wikipedia.org/wiki/WebDAV" config="/webdav/" >}}
 {{< provider name="Yandex Disk" home="https://disk.yandex.com/" config="/yandex/" >}}
 {{< provider name="Zoho WorkDrive" home="https://www.zoho.com/workdrive/" config="/zoho/" >}}
-{{< provider name="Zata" home="https://zata.ai/" config="/s3/#Zata" end="true" >}}
 {{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
 {{< /provider_list >}}
 

docs/content/authors.md

@@ -788,6 +788,7 @@ put them back in again.` >}}
   * Adithya Kumar <akumar42@protonmail.com>
   * Tayo-pasedaRJ <138471223+Tayo-pasedaRJ@users.noreply.github.com>
   * Peter Kreuser <logo@kreuser.name>
+  * Piyush <piyushgarg80>
   * fotile96 <fotile96@users.noreply.github.com>
   * Luc Ritchie <luc.ritchie@gmail.com>
   * cynful <cynful@users.noreply.github.com>
@@ -966,28 +967,3 @@ put them back in again.` >}}
   * Christian Richter <crichter@owncloud.com> <1058116+dragonchaser@users.noreply.github.com>
   * Ralf Haferkamp <r.haferkamp@opencloud.eu>
   * Jugal Kishore <me@devjugal.com>
-  * Tho Neyugn <nguyentruongtho@users.noreply.github.com>
-  * Ben Boeckel <mathstuf@users.noreply.github.com>
-  * Clément Wehrung <cwehrung@nurves.com>
-  * Jeff Geerling <geerlingguy@mac.com>
-  * Germán Casares <german.casares.march+github@gmail.com>
-  * fhuber <florian.huber@noris.de>
-  * wbulot <wbulot@hotmail.com>
-  * Jeremy Daer <jeremydaer@gmail.com>
-  * Oleksiy Stashok <ostashok@tesla.com>
-  * PrathameshLakawade <prathameshlakawade@gmail.com>
-  * Nathanael Demacon <7271496+quantumsheep@users.noreply.github.com>
-  * ahxxm <ahxxm@users.noreply.github.com>
-  * Flora Thiebaut <johann.thiebaut@gmail.com>
-  * kingston125 <support@filelu.com>
-  * Ser-Bul <30335009+Ser-Bul@users.noreply.github.com>
-  * jinjingroad <jinjingroad@sina.com>
-  * necaran <55765083+necaran@users.noreply.github.com>
-  * Marvin Rösch <marvinroesch99@gmail.com>
-  * curlwget <curlwget@icloud.com>
-  * $@M@RTH_ <165886349+Samarth1015@users.noreply.github.com>
-  * Ali Zein Yousuf <azy5030@gmail.com>
-  * Davide Bizzarri <davide.bizzarri@willhaben.at>
-  * Ross Smith II <ross@smithii.com>
-  * Vikas Bhansali <64532198+vibhansa-msft@users.noreply.github.com>
-  * Sudipto Baral <sudiptobaral.me@gmail.com>

docs/content/azureblob.md

@@ -297,16 +297,6 @@ be explicitly specified using exactly one of the `msi_object_id`,
 If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
 set, this is is equivalent to using `env_auth`.
 
-#### Fedrated Identity Credentials 
-
-If these variables are set, rclone will authenticate with fedrated identity.
-
-- `tenant_id`: tenant_id to authenticate in storage
-- `client_id`: client ID of the application the user will authenticate to storage
-- `msi_client_id`: managed identity client ID of the application the user will authenticate to
-
-By default "api://AzureADTokenExchange" is used as scope for token retrieval over MSI. This token is then exchanged for actual storage token using 'tenant_id' and 'client_id'.
-
 #### Azure CLI tool `az` {#use_az}
 
 Set to use the [Azure CLI tool `az`](https://learn.microsoft.com/en-us/cli/azure/)
@@ -729,65 +719,6 @@ Properties:
 - Type:        int
 - Default:     16
 
-#### --azureblob-copy-cutoff
-
-Cutoff for switching to multipart copy.
-
-Any files larger than this that need to be server-side copied will be
-copied in chunks of chunk_size using the put block list API.
-
-Files smaller than this limit will be copied with the Copy Blob API.
-
-Properties:
-
-- Config:      copy_cutoff
-- Env Var:     RCLONE_AZUREBLOB_COPY_CUTOFF
-- Type:        SizeSuffix
-- Default:     8Mi
-
-#### --azureblob-copy-concurrency
-
-Concurrency for multipart copy.
-
-This is the number of chunks of the same file that are copied
-concurrently.
-
-These chunks are not buffered in memory and Microsoft recommends
-setting this value to greater than 1000 in the azcopy documentation.
-
-https://learn.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-optimize#increase-concurrency
-
-In tests, copy speed increases almost linearly with copy
-concurrency.
-
-Properties:
-
-- Config:      copy_concurrency
-- Env Var:     RCLONE_AZUREBLOB_COPY_CONCURRENCY
-- Type:        int
-- Default:     512
-
-#### --azureblob-use-copy-blob
-
-Whether to use the Copy Blob API when copying to the same storage account.
-
-If true (the default) then rclone will use the Copy Blob API for
-copies to the same storage account even when the size is above the
-copy_cutoff.
-
-Rclone assumes that the same storage account means the same config
-and does not check for the same storage account in different configs.
-
-There should be no need to change this value.
-
-
-Properties:
-
-- Config:      use_copy_blob
-- Env Var:     RCLONE_AZUREBLOB_USE_COPY_BLOB
-- Type:        bool
-- Default:     true
-
 #### --azureblob-list-chunk
 
 Size of blob list.

docs/content/azurefiles.md

@@ -295,16 +295,6 @@ be explicitly specified using exactly one of the `msi_object_id`,
 
 If none of `msi_object_id`, `msi_client_id`, or `msi_mi_res_id` is
 set, this is is equivalent to using `env_auth`.
-
-#### Fedrated Identity Credentials 
-
-If these variables are set, rclone will authenticate with fedrated identity.
-
-- `tenant_id`: tenant_id to authenticate in storage
-- `client_id`: client ID of the application the user will authenticate to storage
-- `msi_client_id`: managed identity client ID of the application the user will authenticate to
-
-By default "api://AzureADTokenExchange" is used as scope for token retrieval over MSI. This token is then exchanged for actual storage token using 'tenant_id' and 'client_id'.
 	
 #### Azure CLI tool `az` {#use_az}
 Set to use the [Azure CLI tool `az`](https://learn.microsoft.com/en-us/cli/azure/)
@@ -625,42 +615,6 @@ Properties:
 - Type:        string
 - Required:    false
 
-#### --azurefiles-disable-instance-discovery
-
-Skip requesting Microsoft Entra instance metadata
-This should be set true only by applications authenticating in
-disconnected clouds, or private clouds such as Azure Stack.
-It determines whether rclone requests Microsoft Entra instance
-metadata from `https://login.microsoft.com/` before
-authenticating.
-Setting this to true will skip this request, making you responsible
-for ensuring the configured authority is valid and trustworthy.
-
-
-Properties:
-
-- Config:      disable_instance_discovery
-- Env Var:     RCLONE_AZUREFILES_DISABLE_INSTANCE_DISCOVERY
-- Type:        bool
-- Default:     false
-
-#### --azurefiles-use-az
-
-Use Azure CLI tool az for authentication
-Set to use the [Azure CLI tool az](https://learn.microsoft.com/en-us/cli/azure/)
-as the sole means of authentication.
-Setting this can be useful if you wish to use the az CLI on a host with
-a System Managed Identity that you do not want to use.
-Don't set env_auth at the same time.
-
-
-Properties:
-
-- Config:      use_az
-- Env Var:     RCLONE_AZUREFILES_USE_AZ
-- Type:        bool
-- Default:     false
-
 #### --azurefiles-endpoint
 
 Endpoint for the service.

docs/content/bisync.md

@@ -635,7 +635,7 @@ directory separators.) To address this particular issue, an additional
 `2006-01-02 0304PM`.
 
 Note that `--conflict-suffix` is entirely separate from rclone's main
-[`--sufix`](/docs/#suffix-string) flag. This is intentional, as users may wish
+[`--sufix`](/docs/#suffix-suffix) flag. This is intentional, as users may wish
 to use both flags simultaneously, if also using
 [`--backup-dir`](#backup-dir1-and-backup-dir2).
 
@@ -810,7 +810,7 @@ without requiring the user to get involved and run a `--resync`. (See also:
 
 ### --backup-dir1 and --backup-dir2
 
-As of `v1.66`, [`--backup-dir`](/docs/#backup-dir-string) is supported in bisync.
+As of `v1.66`, [`--backup-dir`](/docs/#backup-dir-dir) is supported in bisync.
 Because `--backup-dir` must be a non-overlapping path on the same remote,
 Bisync has introduced new `--backup-dir1` and `--backup-dir2` flags to support
 separate backup-dirs for `Path1` and `Path2` (bisyncing between different
@@ -841,7 +841,7 @@ In the event of a [rename due to a sync conflict](#conflict-loser), the
 rename is not considered a delete, unless a previous conflict with the same
 name already exists and would get overwritten.
 
-See also: [`--suffix`](/docs/#suffix-string),
+See also: [`--suffix`](/docs/#suffix-suffix),
 [`--suffix-keep-extension`](/docs/#suffix-keep-extension)
 
 ## Operation
@@ -1831,7 +1831,7 @@ about _Unison_ and synchronization in general.
 * Final listings are now generated from sync results, to avoid needing to re-list
 * Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes
 * Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time.
-* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color) (`AUTO`|`NEVER`|`ALWAYS`)
+* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)
 * Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,
 for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).
 * Fixed handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)

docs/content/changelog.md

@@ -5,178 +5,6 @@ description: "Rclone Changelog"
 
 # Changelog
 
-## v1.70.3 - 2025-07-09
-
-[See commits](https://github.com/rclone/rclone/compare/v1.70.2...v1.70.3)
-
-* Bug Fixes
-    * check: Fix difference report (was reporting error counts) (albertony)
-    * march: Fix deadlock when using `--no-traverse` (Nick Craig-Wood)
-    * doc fixes (albertony, Nick Craig-Wood)
-* Azure Blob
-    * Fix server side copy error "requires exactly one scope" (Nick Craig-Wood)
-* B2
-    * Fix finding objects when using `--b2-version-at` (Davide Bizzarri)
-* Linkbox
-    * Fix upload error "user upload file not exist" (Nick Craig-Wood)
-* Pikpak
-    * Improve error handling for missing links and unrecoverable 500s (wiserain)
-* WebDAV
-    * Fix setting modtime to that of local object instead of remote (WeidiDeng)
-
-## v1.70.2 - 2025-06-27
-
-[See commits](https://github.com/rclone/rclone/compare/v1.70.1...v1.70.2)
-
-* Bug Fixes
-    * convmv: Make --dry-run logs less noisy (nielash)
-    * sync: Avoid copying dir metadata to itself (nielash)
-    * build: Bump github.com/go-chi/chi/v5 from 5.2.1 to 5.2.2 to fix GHSA-vrw8-fxc6-2r93 (dependabot[bot])
-    * convmv: Fix moving to unicode-equivalent name (nielash)
-    * log: Fix deadlock when using systemd logging (Nick Craig-Wood)
-    * pacer: Fix nil pointer deref in RetryError (Nick Craig-Wood)
-    * doc fixes (Ali Zein Yousuf, Nick Craig-Wood)
-* Local
-    * Fix --skip-links on Windows when skipping Junction points (Nick Craig-Wood)
-* Combine
-    * Fix directory not found errors with ListP interface (Nick Craig-Wood)
-* Mega
-    * Fix tls handshake failure (necaran)
-* Pikpak
-    * Fix uploads fail with "aws-chunked encoding is not supported" error (Nick Craig-Wood)
-
-## v1.70.1 - 2025-06-19
-
-[See commits](https://github.com/rclone/rclone/compare/v1.70.0...v1.70.1)
-
-* Bug Fixes
-    * convmv: Fix spurious "error running command echo" on Windows (Nick Craig-Wood)
-    * doc fixes (albertony, Ed Craig-Wood, jinjingroad)
-
-## v1.70.0 - 2025-06-17
-
-[See commits](https://github.com/rclone/rclone/compare/v1.69.0...v1.70.0)
-
-* New backends
-    * [DOI](/doi/) (Flora Thiebaut)
-    * [FileLu](/filelu/) (kingston125)
-    * New S3 providers:
-        * [MEGA S4](/s3/#mega) (Nick Craig-Wood)
-        * [Pure Storage FlashBlade](/s3/#pure-storage-flashblade) (Jeremy Daer)
-* New commands
-    * [convmv](/commands/rclone_convmv/): for moving and transforming files (nielash)
-* New Features
-    * Add [`--max-connections`](/docs/#max-connections-n) to control maximum backend concurrency (Nick Craig-Wood)
-    * Add [`--max-buffer-memory`](/docs/#max-buffer-memory) to limit total buffer memory usage (Nick Craig-Wood)
-    * Add transform library and [`--name-transform`](/docs/#name-transform-stringarray) flag (nielash)
-    * sync: Implement [`--list-cutoff`](/docs/#list-cutoff) to allow on disk sorting for reduced memory use (Nick Craig-Wood)
-    * accounting: Add listed stat for number of directory entries listed (Nick Craig-Wood)
-    * backend: Skip hash calculation when the hashType is None (Oleksiy Stashok)
-    * build
-        * Update to go1.24 and make go1.22 the minimum required version (Nick Craig-Wood)
-        * Disable docker builds on PRs & add missing dockerfile changes (Anagh Kumar Baranwal)
-        * Modernize Go usage (Nick Craig-Wood)
-        * Update all dependencies (Nick Craig-Wood)
-    * cmd/authorize: Show required arguments in help text (simwai)
-    * cmd/config: add `--no-output` option (Jess)
-    * cmd/gitannex
-        * Tweak parsing of "rcloneremotename" config (Dan McArdle)
-        * Permit remotes with options (Dan McArdle)
-        * Reject unknown layout modes in INITREMOTE (Dan McArdle)
-    * docker image: Add label org.opencontainers.image.source for release notes in Renovate dependency updates (Robin Schneider)
-    * doc fixes (albertony, Andrew Kreimer, Ben Boeckel, Christoph Berger, Danny Garside, Dimitri Papadopoulos, eccoisle, Ed Craig-Wood, Fernando Fernández, jack, Jeff Geerling, Jugal Kishore, kingston125, luzpaz, Markus Gerstel, Matt Ickstadt, Michael Kebe, Nick Craig-Wood, PrathameshLakawade, Ser-Bul, simonmcnair, Tim White, Zachary Vorhies)
-    * filter:
-        * Add `--hash-filter` to deterministically select a subset of files (Nick Craig-Wood)
-        * Show `--min-size` and `--max-size` in `--dump` filters (Nick Craig-Wood)
-    * hash: Add SHA512 support for file hashes (Enduriel)
-    * http servers: Add `--user-from-header` to use for authentication (Moises Lima)
-    * lib/batcher: Deprecate unused option: batch_commit_timeout (Dan McArdle)
-    * log:
-        * Remove github.com/sirupsen/logrus and replace with log/slog (Nick Craig-Wood)
-        * Add `--windows-event-log-level` to support Windows Event Log (Nick Craig-Wood)
-    * rc
-        * Add add `short` parameter to `core/stats` to not return transferring and checking (Nick Craig-Wood)
-        * In `options/info` make FieldName contain a "." if it should be nested (Nick Craig-Wood)
-        * Add rc control for serve commands (Nick Craig-Wood)
-        * rcserver: Improve content-type check (Jonathan Giannuzzi)
-    * serve nfs
-        * Update docs to note Windows is not supported (Zachary Vorhies)
-        * Change the format of `--nfs-cache-type symlink` file handles (Nick Craig-Wood)
-        * Make metadata files have special file handles (Nick Craig-Wood)
-    * touch: Make touch obey `--transfers` (Nick Craig-Wood)
-    * version: Add `--deps` flag to show dependencies and other build info (Nick Craig-Wood)
-* Bug Fixes
-    * serve s3:
-        * Fix ListObjectsV2 response (fhuber)
-        * Remove redundant handler initialization (Tho Neyugn)
-    * stats: Fix goroutine leak and improve stats accounting process (Nathanael Demacon)
-* VFS
-    * Add `--vfs-metadata-extension` to expose metadata sidecar files (Nick Craig-Wood)
-* Azure Blob
-    * Add support for `x-ms-tags` header (Trevor Starick)
-    * Cleanup uncommitted blocks on upload errors (Nick Craig-Wood)
-    * Speed up server side copies for small files (Nick Craig-Wood)
-    * Implement multipart server side copy (Nick Craig-Wood)
-    * Remove uncommitted blocks on InvalidBlobOrBlock error (Nick Craig-Wood)
-    * Fix handling of objects with // in (Nick Craig-Wood)
-    * Handle retry error codes more carefully (Nick Craig-Wood)
-    * Fix errors not being retried when doing single part copy (Nick Craig-Wood)
-    * Fix multipart server side copies of 0 sized files (Nick Craig-Wood)
-* Azurefiles
-    * Add `--azurefiles-use-az` and `--azurefiles-disable-instance-discovery` (b-wimmer)
-* B2
-    * Add SkipDestructive handling to backend commands (Pat Patterson)
-    * Use file id from listing when not presented in headers (ahxxm)
-* Cloudinary
-    * Automatically add/remove known media files extensions (yuval-cloudinary)
-    * Var naming convention (yuval-cloudinary)
-* Drive
-    * Added `backend moveid` command (Spencer McCullough)
-* Dropbox
-    * Support Dropbox Paper (Dave Vasilevsky)
-* FTP
-    * Add `--ftp-http-proxy` to connect via HTTP CONNECT proxy
-* Gofile
-    * Update to use new direct upload endpoint (wbulot)
-* Googlephotos
-    * Update read only and read write scopes to meet Google's requirements. (Germán Casares)
-* Iclouddrive
-    * Fix panic and files potentially downloaded twice (Clément Wehrung)
-* Internetarchive
-    * Add `--internetarchive-metadata="key=value"` for setting item metadata (Corentin Barreau)
-* Onedrive
-    * Fix "The upload session was not found" errors (Nick Craig-Wood)
-    * Re-add `--onedrive-upload-cutoff` flag (Nick Craig-Wood)
-    * Fix crash if no metadata was updated (Nick Craig-Wood)
-* Opendrive
-    * Added `--opendrive-access` flag to handle permissions (Joel K Biju)
-* Pcloud
-    * Fix "Access denied. You do not have permissions to perform this operation" on large uploads (Nick Craig-Wood)
-* S3
-    * Fix handling of objects with // in (Nick Craig-Wood)
-    * Add IBM IAM signer (Alexander Minbaev)
-    * Split the GCS quirks into `--s3-use-x-id` and `--s3-sign-accept-encoding` (Nick Craig-Wood)
-    * Implement paged listing interface ListP (Nick Craig-Wood)
-    * Add Pure Storage FlashBlade provider support (Jeremy Daer)
-    * Require custom endpoint for Lyve Cloud v2 support (PrathameshLakawade)
-    * MEGA S4 support (Nick Craig-Wood)
-* SFTP
-    * Add `--sftp-http-proxy` to connect via HTTP CONNECT proxy (Nick Craig-Wood)
-* Smb
-    * Add support for kerberos authentication (Jonathan Giannuzzi)
-    * Improve connection pooling efficiency (Jonathan Giannuzzi)
-* WebDAV
-    * Retry propfind on 425 status (Jörn Friedrich Dreyer)
-    * Add an ownCloud Infinite Scale vendor that enables tus chunked upload support (Klaas Freitag)
-
-## v1.69.3 - 2025-05-21
-
-[See commits](https://github.com/rclone/rclone/compare/v1.69.2...v1.69.3)
-
-* Bug Fixes
-    * build: Reapply update github.com/golang-jwt/jwt/v5 from 5.2.1 to 5.2.2 to fix CVE-2025-30204 (dependabot[bot])
-    * build: Update github.com/ebitengine/purego to work around bug in go1.24.3 (Nick Craig-Wood)
-
 ## v1.69.2 - 2025-05-01
 
 [See commits](https://github.com/rclone/rclone/compare/v1.69.1...v1.69.2)
@@ -777,7 +605,7 @@ description: "Rclone Changelog"
     * Final listings are now generated from sync results, to avoid needing to re-list (nielash)
     * Bisync is now much more resilient to changes that happen during a bisync run, and far less prone to critical errors / undetected changes (nielash)
     * Bisync is now capable of rolling a file listing back in cases of uncertainty, essentially marking the file as needing to be rechecked next time. (nielash)
-    * A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-autoneveralways) (`AUTO`|`NEVER`|`ALWAYS`) (nielash)
+    * A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`) (nielash)
     * Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`, for performance improvements and less risk of error. (nielash)
     * `--resync` is now much more efficient (especially for users of `--create-empty-src-dirs`) (nielash)
     * Google Docs (and other files of unknown size) are now supported (with the same options as in `sync`) (nielash)
@@ -789,7 +617,7 @@ instead of of `--size-only`, when `check` is not available.
     * A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted. (nielash)
     * Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags. (nielash)
     * A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`. (nielash)
-    * Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-duration) (when [`--resilient`](#resilient) is set.) (nielash)
+    * Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.) (nielash)
     * Clarify file operation directions in dry-run logs (Kyle Reynolds)
 * Local
     * Fix cleanRootPath on Windows after go1.21.4 stdlib update (nielash)
@@ -3040,7 +2868,7 @@ instead of of `--size-only`, when `check` is not available.
 * New Features
     * The [VFS layer](/commands/rclone_mount/#vfs-virtual-file-system) was heavily reworked for this release - see below for more details
     * Interactive mode [-i/--interactive](/docs/#interactive) for destructive operations (fishbullet)
-    * Add [--bwlimit-file](/docs/#bwlimit-file-bwtimetable) flag to limit speeds of individual file transfers (Nick Craig-Wood)
+    * Add [--bwlimit-file](/docs/#bwlimit-file-bandwidth-spec) flag to limit speeds of individual file transfers (Nick Craig-Wood)
     * Transfers are sorted by start time in the stats and progress output (Max Sum)
     * Make sure backends expand `~` and environment vars in file names they use (Nick Craig-Wood)
     * Add [--refresh-times](/docs/#refresh-times) flag to set modtimes on hashless backends (Nick Craig-Wood)

docs/content/commands/rclone.md

@@ -37,8 +37,6 @@ rclone [flags]
       --azureblob-client-id string                          The ID of the client in use
       --azureblob-client-secret string                      One of the service principal's client secrets
       --azureblob-client-send-certificate-chain             Send the certificate chain when using certificate auth
-      --azureblob-copy-concurrency int                      Concurrency for multipart copy (default 512)
-      --azureblob-copy-cutoff SizeSuffix                    Cutoff for switching to multipart copy (default 8Mi)
       --azureblob-delete-snapshots string                   Set to specify how to deal with snapshots on blob deletion
       --azureblob-description string                        Description of the remote
       --azureblob-directory-markers                         Upload an empty object with a trailing slash when a new directory is created
@@ -62,7 +60,6 @@ rclone [flags]
       --azureblob-upload-concurrency int                    Concurrency for multipart uploads (default 16)
       --azureblob-upload-cutoff string                      Cutoff for switching to chunked upload (<= 256 MiB) (deprecated)
       --azureblob-use-az                                    Use Azure CLI tool az for authentication
-      --azureblob-use-copy-blob                             Whether to use the Copy Blob API when copying to the same storage account (default true)
       --azureblob-use-emulator                              Uses local storage emulator if provided as 'true'
       --azureblob-use-msi                                   Use a managed service identity to authenticate (only works in Azure)
       --azureblob-username string                           User name (usually an email address)
@@ -75,7 +72,6 @@ rclone [flags]
       --azurefiles-client-send-certificate-chain            Send the certificate chain when using certificate auth
       --azurefiles-connection-string string                 Azure Files Connection String
       --azurefiles-description string                       Description of the remote
-      --azurefiles-disable-instance-discovery               Skip requesting Microsoft Entra instance metadata
       --azurefiles-encoding Encoding                        The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,RightPeriod,InvalidUtf8,Dot)
       --azurefiles-endpoint string                          Endpoint for the service
       --azurefiles-env-auth                                 Read credentials from runtime (environment variables, CLI or MSI)
@@ -90,7 +86,6 @@ rclone [flags]
       --azurefiles-share-name string                        Azure Files Share Name
       --azurefiles-tenant string                            ID of the service principal's tenant. Also called its directory ID
       --azurefiles-upload-concurrency int                   Concurrency for multipart uploads (default 16)
-      --azurefiles-use-az                                   Use Azure CLI tool az for authentication
       --azurefiles-use-msi                                  Use a managed service identity to authenticate (only works in Azure)
       --azurefiles-username string                          User name (usually an email address)
       --b2-account string                                   Account ID or Application Key ID
@@ -165,14 +160,12 @@ rclone [flags]
       --chunker-remote string                               Remote to chunk/unchunk
       --client-cert string                                  Client SSL certificate (PEM) for mutual TLS auth
       --client-key string                                   Client SSL private key (PEM) for mutual TLS auth
-      --cloudinary-adjust-media-files-extensions            Cloudinary handles media formats as a file attribute and strips it from the name, which is unlike most other file systems (default true)
       --cloudinary-api-key string                           Cloudinary API Key
       --cloudinary-api-secret string                        Cloudinary API Secret
       --cloudinary-cloud-name string                        Cloudinary Environment Name
       --cloudinary-description string                       Description of the remote
       --cloudinary-encoding Encoding                        The encoding for the backend (default Slash,LtGt,DoubleQuote,Question,Asterisk,Pipe,Hash,Percent,BackSlash,Del,Ctl,RightSpace,InvalidUtf8,Dot)
       --cloudinary-eventually-consistent-delay Duration     Wait N seconds for eventual consistency of the databases that support the backend operation (default 0s)
-      --cloudinary-media-extensions stringArray             Cloudinary supported media extensions (default 3ds,3g2,3gp,ai,arw,avi,avif,bmp,bw,cr2,cr3,djvu,dng,eps3,fbx,flif,flv,gif,glb,gltf,hdp,heic,heif,ico,indd,jp2,jpe,jpeg,jpg,jxl,jxr,m2ts,mov,mp4,mpeg,mts,mxf,obj,ogv,pdf,ply,png,psd,svg,tga,tif,tiff,ts,u3ma,usdz,wdp,webm,webp,wmv)
       --cloudinary-upload-prefix string                     Specify the API endpoint for environments out of the US
       --cloudinary-upload-preset string                     Upload Preset to select asset manipulation on upload
       --color AUTO|NEVER|ALWAYS                             When to show colors (and other ANSI codes) AUTO|NEVER|ALWAYS (default AUTO)
@@ -211,10 +204,6 @@ rclone [flags]
       --disable string                                      Disable a comma separated list of features (use --disable help to see a list)
       --disable-http-keep-alives                            Disable HTTP keep-alives and use each connection once
       --disable-http2                                       Disable HTTP/2 in the global transport
-      --doi-description string                              Description of the remote
-      --doi-doi string                                      The DOI or the doi.org URL
-      --doi-doi-resolver-api-url string                     The URL of the DOI resolver API to use
-      --doi-provider string                                 DOI provider
       --drive-acknowledge-abuse                             Set to allow files which return cannotDownloadAbusiveFile to be downloaded
       --drive-allow-import-name-change                      Allow the filetype to change when uploading Google docs
       --drive-auth-owner-only                               Only consider files owned by the authenticated user
@@ -266,6 +255,7 @@ rclone [flags]
       --drive-use-trash                                     Send files to the trash instead of deleting permanently (default true)
       --drive-v2-download-min-size SizeSuffix               If Object's are greater, use drive v2 API to download (default off)
       --dropbox-auth-url string                             Auth server URL
+      --dropbox-batch-commit-timeout Duration               Max time to wait for a batch to finish committing (default 10m0s)
       --dropbox-batch-mode string                           Upload file batching sync|async|off (default "sync")
       --dropbox-batch-size int                              Max number of files in upload batch
       --dropbox-batch-timeout Duration                      Max time to allow an idle upload batch before uploading (default 0s)
@@ -275,14 +265,11 @@ rclone [flags]
       --dropbox-client-secret string                        OAuth Client Secret
       --dropbox-description string                          Description of the remote
       --dropbox-encoding Encoding                           The encoding for the backend (default Slash,BackSlash,Del,RightSpace,InvalidUtf8,Dot)
-      --dropbox-export-formats CommaSepList                 Comma separated list of preferred formats for exporting files (default html,md)
       --dropbox-impersonate string                          Impersonate this user when using a business account
       --dropbox-pacer-min-sleep Duration                    Minimum time to sleep between API calls (default 10ms)
       --dropbox-root-namespace string                       Specify a different Dropbox namespace ID to use as the root for all paths
       --dropbox-shared-files                                Instructs rclone to work on individual shared files
       --dropbox-shared-folders                              Instructs rclone to work on shared folders
-      --dropbox-show-all-exports                            Show all exportable files in listings
-      --dropbox-skip-exports                                Skip exportable files in all listings
       --dropbox-token string                                OAuth Access Token as a JSON blob
       --dropbox-token-url string                            Token server url
   -n, --dry-run                                             Do a trial run with no permanent changes
@@ -311,9 +298,6 @@ rclone [flags]
       --filefabric-token-expiry string                      Token expiry time
       --filefabric-url string                               URL of the Enterprise File Fabric to connect to
       --filefabric-version string                           Version read from the file fabric
-      --filelu-description string                           Description of the remote
-      --filelu-encoding Encoding                            The encoding for the backend (default Slash,LtGt,DoubleQuote,SingleQuote,BackQuote,Dollar,Colon,Question,Asterisk,Pipe,Hash,Percent,BackSlash,CrLf,Del,Ctl,LeftSpace,LeftPeriod,LeftTilde,LeftCrLfHtVt,RightSpace,RightPeriod,RightCrLfHtVt,InvalidUtf8,Dot,SquareBracket,Semicolon,Exclamation)
-      --filelu-key string                                   Your FileLu Rclone key from My Account
       --files-from stringArray                              Read list of source-file names from file (use - to read from stdin)
       --files-from-raw stringArray                          Read list of source-file names from file without any processing of lines (use - to read from stdin)
       --filescom-api-key string                             The API key used to authenticate with Files.com
@@ -380,6 +364,7 @@ rclone [flags]
       --gofile-list-chunk int                               Number of items to list in each call (default 1000)
       --gofile-root-folder-id string                        ID of the root folder
       --gphotos-auth-url string                             Auth server URL
+      --gphotos-batch-commit-timeout Duration               Max time to wait for a batch to finish committing (default 10m0s)
       --gphotos-batch-mode string                           Upload file batching sync|async|off (default "sync")
       --gphotos-batch-size int                              Max number of files in upload batch
       --gphotos-batch-timeout Duration                      Max time to allow an idle upload batch before uploading (default 0s)
@@ -395,7 +380,6 @@ rclone [flags]
       --gphotos-start-year int                              Year limits the photos to be downloaded to those which are uploaded after the given year (default 2000)
       --gphotos-token string                                OAuth Access Token as a JSON blob
       --gphotos-token-url string                            Token server url
-      --hash-filter string                                  Partition filenames by hash k/n or randomly @/n
       --hasher-auto-size SizeSuffix                         Auto-update checksum for files smaller than this size (disabled by default)
       --hasher-description string                           Description of the remote
       --hasher-hashes CommaSepList                          Comma separated list of supported checksum types (default md5,sha1)
@@ -465,8 +449,6 @@ rclone [flags]
       --internetarchive-encoding Encoding                   The encoding for the backend (default Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot)
       --internetarchive-endpoint string                     IAS3 Endpoint (default "https://s3.us.archive.org")
       --internetarchive-front-endpoint string               Host of InternetArchive Frontend (default "https://archive.org")
-      --internetarchive-item-derive                         Whether to trigger derive on the IA item or not. If set to false, the item will not be derived by IA upon upload (default true)
-      --internetarchive-item-metadata stringArray           Metadata to be set on the IA item, this is different from file-level metadata that can be set using --metadata-set
       --internetarchive-secret-access-key string            IAS3 Secret Key (password)
       --internetarchive-wait-archive Duration               Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish (default 0s)
       --jottacloud-auth-url string                          Auth server URL
@@ -494,7 +476,6 @@ rclone [flags]
       --linkbox-description string                          Description of the remote
       --linkbox-token string                                Token from https://www.linkbox.to/admin/account
   -l, --links                                               Translate symlinks to/from regular files with a '.rclonelink' extension
-      --list-cutoff int                                     To save memory, sort directory listings on disk above this threshold (default 1000000)
       --local-case-insensitive                              Force the filesystem to report itself as case insensitive
       --local-case-sensitive                                Force the filesystem to report itself as case sensitive
       --local-description string                            Description of the remote
@@ -510,7 +491,7 @@ rclone [flags]
       --local-unicode-normalization                         Apply unicode NFC normalization to paths and filenames
       --local-zero-size-links                               Assume the Stat size of links is zero (and read them instead) (deprecated)
       --log-file string                                     Log everything to this file
-      --log-format Bits                                     Comma separated list of log format options (default date,time)
+      --log-format string                                   Comma separated list of log format options (default "date,time")
       --log-level LogLevel                                  Log level DEBUG|INFO|NOTICE|ERROR (default NOTICE)
       --log-systemd                                         Activate systemd integration for the logger
       --low-level-retries int                               Number of low level retries to do (default 10)
@@ -531,8 +512,6 @@ rclone [flags]
       --mailru-user string                                  User name (usually email)
       --max-age Duration                                    Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y (default off)
       --max-backlog int                                     Maximum number of objects in sync or check backlog (default 10000)
-      --max-buffer-memory SizeSuffix                        If set, don't allocate more than this amount of memory as buffers (default off)
-      --max-connections int                                 Maximum number of simultaneous backend API connections, 0 for unlimited
       --max-delete int                                      When synchronizing, limit the number of deletes (default -1)
       --max-delete-size SizeSuffix                          When synchronizing, limit the total size of deletes (default off)
       --max-depth int                                       If set limits the recursion depth to this (default -1)
@@ -574,7 +553,6 @@ rclone [flags]
       --metrics-server-write-timeout Duration               Timeout for server writing data (default 1h0m0s)
       --metrics-template string                             User-specified template
       --metrics-user string                                 User name for authentication
-      --metrics-user-from-header string                     User name from a defined HTTP header
       --min-age Duration                                    Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y (default off)
       --min-size SizeSuffix                                 Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P (default off)
       --modify-window Duration                              Max time diff to be considered the same (default 1ns)
@@ -582,7 +560,6 @@ rclone [flags]
       --multi-thread-cutoff SizeSuffix                      Use multi-thread downloads for files above this size (default 256Mi)
       --multi-thread-streams int                            Number of streams to use for multi-thread downloads (default 4)
       --multi-thread-write-buffer-size SizeSuffix           In memory buffer size for writing when in multi-thread mode (default 128Ki)
-      --name-transform stringArray                          Transform paths during the copy process
       --netstorage-account string                           Set the NetStorage account name
       --netstorage-description string                       Description of the remote
       --netstorage-host string                              Domain+path of NetStorage host to connect to
@@ -624,7 +601,6 @@ rclone [flags]
       --onedrive-tenant string                              ID of the service principal's tenant. Also called its directory ID
       --onedrive-token string                               OAuth Access Token as a JSON blob
       --onedrive-token-url string                           Token server url
-      --onedrive-upload-cutoff SizeSuffix                   Cutoff for switching to chunked upload (default off)
       --oos-attempt-resume-upload                           If true attempt to resume previously started multipart upload for the object
       --oos-chunk-size SizeSuffix                           Chunk size to use for uploading (default 5Mi)
       --oos-compartment string                              Specify compartment OCID, if you need to list buckets
@@ -650,7 +626,6 @@ rclone [flags]
       --oos-storage-tier string                             The storage class to use when storing new objects in storage. https://docs.oracle.com/en-us/iaas/Content/Object/Concepts/understandingstoragetiers.htm (default "Standard")
       --oos-upload-concurrency int                          Concurrency for multipart uploads (default 10)
       --oos-upload-cutoff SizeSuffix                        Cutoff for switching to chunked upload (default 200Mi)
-      --opendrive-access string                             Files and folders will be uploaded with this access permission (default private) (default "private")
       --opendrive-chunk-size SizeSuffix                     Files will be uploaded in chunks this size (default 10Mi)
       --opendrive-description string                        Description of the remote
       --opendrive-encoding Encoding                         The encoding for the backend (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
@@ -761,7 +736,6 @@ rclone [flags]
       --rc-server-write-timeout Duration                    Timeout for server writing data (default 1h0m0s)
       --rc-template string                                  User-specified template
       --rc-user string                                      User name for authentication
-      --rc-user-from-header string                          User name from a defined HTTP header
       --rc-web-fetch-url string                             URL to fetch the releases for webgui (default "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest")
       --rc-web-gui                                          Launch WebGUI on localhost
       --rc-web-gui-force-update                             Force update to latest version of web gui
@@ -786,8 +760,6 @@ rclone [flags]
       --s3-endpoint string                                  Endpoint for S3 API
       --s3-env-auth                                         Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars)
       --s3-force-path-style                                 If true use path style access if false use virtual hosted style (default true)
-      --s3-ibm-api-key string                               IBM API Key to be used to obtain IAM token
-      --s3-ibm-resource-instance-id string                  IBM service instance id
       --s3-leave-parts-on-error                             If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery
       --s3-list-chunk int                                   Size of listing chunk (response list for each ListObject S3 request) (default 1000)
       --s3-list-url-encode Tristate                         Whether to url encode listings: true/false/unset (default unset)
@@ -808,7 +780,6 @@ rclone [flags]
       --s3-server-side-encryption string                    The server-side encryption algorithm used when storing this object in S3
       --s3-session-token string                             An AWS session token
       --s3-shared-credentials-file string                   Path to the shared credentials file
-      --s3-sign-accept-encoding Tristate                    Set if rclone should include Accept-Encoding as part of the signature (default unset)
       --s3-sse-customer-algorithm string                    If using SSE-C, the server-side encryption algorithm used when storing this object in S3
       --s3-sse-customer-key string                          To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data
       --s3-sse-customer-key-base64 string                   If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data
@@ -825,7 +796,6 @@ rclone [flags]
       --s3-use-multipart-uploads Tristate                   Set if rclone should use multipart uploads (default unset)
       --s3-use-presigned-request                            Whether to use a presigned request or PutObject for single part uploads
       --s3-use-unsigned-payload Tristate                    Whether to use an unsigned payload in PutObject (default unset)
-      --s3-use-x-id Tristate                                Set if rclone should add x-id URL parameters (default unset)
       --s3-v2-auth                                          If true use v2 authentication
       --s3-version-at Time                                  Show file versions as they were at the specified time (default off)
       --s3-version-deleted                                  Show deleted file markers when using versions
@@ -852,7 +822,6 @@ rclone [flags]
       --sftp-disable-hashcheck                              Disable the execution of SSH commands to determine if remote file hashing is available
       --sftp-host string                                    SSH host to connect to
       --sftp-host-key-algorithms SpaceSepList               Space separated list of host key algorithms, ordered by preference
-      --sftp-http-proxy string                              URL for HTTP CONNECT proxy
       --sftp-idle-timeout Duration                          Max time before closing idle connections (default 1m0s)
       --sftp-key-exchange SpaceSepList                      Space separated list of key exchange algorithms, ordered by preference
       --sftp-key-file string                                Path to PEM-encoded private key file
@@ -908,7 +877,6 @@ rclone [flags]
       --smb-pass string                                     SMB password (obscured)
       --smb-port int                                        SMB port number (default 445)
       --smb-spn string                                      Service principal name
-      --smb-use-kerberos                                    Use Kerberos authentication
       --smb-user string                                     SMB username (default "$USER")
       --stats Duration                                      Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable) (default 1m0s)
       --stats-file-name-length int                          Max file name length in stats (0 for no limit) (default 45)
@@ -997,7 +965,7 @@ rclone [flags]
       --use-json-log                                        Use json log format
       --use-mmap                                            Use mmap allocator (see docs)
       --use-server-modtime                                  Use server modified time instead of object metadata
-      --user-agent string                                   Set the user-agent to a specified string (default "rclone/v1.70.0")
+      --user-agent string                                   Set the user-agent to a specified string (default "rclone/v1.69.0")
   -v, --verbose count                                       Print lots more stuff (repeat for more)
   -V, --version                                             Print the version number
       --webdav-auth-redirect                                Preserve authentication on redirect
@@ -1049,7 +1017,6 @@ rclone [flags]
 * [rclone cleanup](/commands/rclone_cleanup/)	 - Clean up the remote if possible.
 * [rclone completion](/commands/rclone_completion/)	 - Output completion script for a given shell.
 * [rclone config](/commands/rclone_config/)	 - Enter an interactive configuration session.
-* [rclone convmv](/commands/rclone_convmv/)	 - Convert file and directory names in place.
 * [rclone copy](/commands/rclone_copy/)	 - Copy files from source to dest, skipping identical files.
 * [rclone copyto](/commands/rclone_copyto/)	 - Copy files from source to dest, skipping identical files.
 * [rclone copyurl](/commands/rclone_copyurl/)	 - Copy the contents of the URL supplied content to dest:path.

docs/content/commands/rclone_authorize.md

@@ -14,18 +14,13 @@ Remote authorization. Used to authorize a remote or headless
 rclone from a machine with a browser - use as instructed by
 rclone config.
 
-The command requires 1-3 arguments:
-  - fs name (e.g., "drive", "s3", etc.)
-  - Either a base64 encoded JSON blob obtained from a previous rclone config session
-  - Or a client_id and client_secret pair obtained from the remote service
-
 Use --auth-no-open-browser to prevent rclone to open auth
 link in default browser automatically.
 
 Use --template to generate HTML output via a custom Go template. If a blank string is provided as an argument to this flag, the default template is used.
 
 ```
-rclone authorize <fs name> [base64_json_blob | client_id client_secret] [flags]
+rclone authorize [flags]
 ```
 
 ## Options

docs/content/commands/rclone_bisync.md

@@ -93,7 +93,6 @@ Flags for anything which can copy a file
       --multi-thread-cutoff SizeSuffix              Use multi-thread downloads for files above this size (default 256Mi)
       --multi-thread-streams int                    Number of streams to use for multi-thread downloads (default 4)
       --multi-thread-write-buffer-size SizeSuffix   In memory buffer size for writing when in multi-thread mode (default 128Ki)
-      --name-transform stringArray                  Transform paths during the copy process
       --no-check-dest                               Don't check the destination, copy regardless
       --no-traverse                                 Don't traverse destination file system on copy
       --no-update-dir-modtime                       Don't update directory modification times
@@ -130,7 +129,6 @@ Flags for filtering directory listings
       --files-from-raw stringArray          Read list of source-file names from file without any processing of lines (use - to read from stdin)
   -f, --filter stringArray                  Add a file filtering rule
       --filter-from stringArray             Read file filtering patterns from a file (use - to read from stdin)
-      --hash-filter string                  Partition filenames by hash k/n or randomly @/n
       --ignore-case                         Ignore case in filters (case insensitive)
       --include stringArray                 Include files matching pattern
       --include-from stringArray            Read file include patterns from file (use - to read from stdin)