Make http servers obey --dump headers,bodies

Before this change, rclone always expected --sftp-path-override to be
the absolute SSH path to remote:path/subpath which effectively made it
unusable for wrapped remotes (for example, when used with a crypt
remote, the user would need to provide the full decrypted path.)

After this change, the old behavior remains the default, but dynamic
paths are now also supported, if the user adds '@' as the first
character of --sftp-path-override. Rclone will ignore the '@' and
treat the rest of the string as the path to the SFTP remote's root.
Rclone will then add any relative subpaths automatically (including
unwrapping/decrypting remotes as necessary).

In other words, the path_override config parameter can now be used to
specify the difference between the SSH and SFTP paths. Once specified
in the config, it is no longer necessary to re-specify for each
command.

See: https://forum.rclone.org/t/sftp-path-override-breaks-on-wrapped-remotes/40025

.github/workflows/build.yml

@@ -27,12 +27,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.18', 'go1.19']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -43,14 +43,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
             os: macos-11
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -59,14 +59,14 @@ jobs:
 
           - job_name: mac_arm64
             os: macos-11
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -76,23 +76,23 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.21.0-rc.3'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.18
-            os: ubuntu-latest
-            go: '1.18'
-            quicktest: true
-            racequicktest: true
-
           - job_name: go1.19
             os: ubuntu-latest
             go: '1.19'
             quicktest: true
             racequicktest: true
 
+          - job_name: go1.20
+            os: ubuntu-latest
+            go: '1.20'
+            quicktest: true
+            racequicktest: true
+
     name: ${{ matrix.job_name }}
 
     runs-on: ${{ matrix.os }}
@@ -130,12 +130,13 @@ jobs:
       - name: Install Libraries on macOS
         shell: bash
         run: |
-          brew untap homebrew/core homebrew/cask
-          brew config
+          # https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
+          # https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
+          unset HOMEBREW_NO_INSTALL_FROM_API
+          brew untap --force homebrew/core
+          brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
-        env:
-          HOMEBREW_NO_INSTALL_FROM_API: # No value, so brew doesn't rely on homebrew-core / homebrew-cask
         if: matrix.os == 'macos-11'
 
       - name: Install Libraries on Windows
@@ -243,7 +244,7 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21.0-rc.3'
           check-latest: true
 
       - name: Install govulncheck
@@ -268,7 +269,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: '1.20'
+          go-version: '1.21.0-rc.3'
 
       - name: Go module cache
         uses: actions/cache@v3

CONTRIBUTING.md

@@ -419,7 +419,7 @@ remote or an fs.
 
 Research
 
-  * Look at the interfaces defined in `fs/fs.go`
+  * Look at the interfaces defined in `fs/types.go`
   * Study one or more of the existing remotes
 
 Getting going

MAINTAINERS.md

@@ -18,6 +18,7 @@ Current active maintainers of rclone are:
 | Caleb Case       | @calebcase        | storj backend                |
 | wiserain         | @wiserain         | pikpak backend               |
 | albertony        | @albertony        |                              |
+| Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
 
 **This is a work in progress Draft**
 

README.md

@@ -51,6 +51,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * IONOS Cloud [:page_facing_up:](https://rclone.org/s3/#ionos)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
+  * Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
   * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
@@ -84,6 +85,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * Storj [:page_facing_up:](https://rclone.org/storj/)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
+  * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
   * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)

backend/all/all.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/pikpak"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
+	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"

backend/azureblob/azureblob.go

@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
@@ -95,6 +95,7 @@ Leave blank to use SAS URL or Emulator, otherwise it needs to be set.
 If this is blank and if env_auth is set it will be read from the
 environment variable ` + "`AZURE_STORAGE_ACCOUNT_NAME`" + ` if possible.
 `,
+			Sensitive: true,
 		}, {
 			Name: "env_auth",
 			Help: `Read credentials from runtime (environment variables, CLI or MSI).
@@ -106,11 +107,13 @@ See the [authentication docs](/azureblob#authentication) for full info.`,
 			Help: `Storage Account Shared Key.
 
 Leave blank to use SAS URL or Emulator.`,
+			Sensitive: true,
 		}, {
 			Name: "sas_url",
 			Help: `SAS URL for container level access only.
 
 Leave blank if using account/key or Emulator.`,
+			Sensitive: true,
 		}, {
 			Name: "tenant",
 			Help: `ID of the service principal's tenant. Also called its directory ID.
@@ -120,6 +123,7 @@ Set this if using
 - Service principal with certificate
 - User with username and password
 `,
+			Sensitive: true,
 		}, {
 			Name: "client_id",
 			Help: `The ID of the client in use.
@@ -129,6 +133,7 @@ Set this if using
 - Service principal with certificate
 - User with username and password
 `,
+			Sensitive: true,
 		}, {
 			Name: "client_secret",
 			Help: `One of the service principal's client secrets
@@ -136,6 +141,7 @@ Set this if using
 Set this if using
 - Service principal with client secret
 `,
+			Sensitive: true,
 		}, {
 			Name: "client_certificate_path",
 			Help: `Path to a PEM or PKCS12 certificate file including the private key.
@@ -173,7 +179,8 @@ Optionally set this if using
 Set this if using
 - User with username and password
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "password",
 			Help: `The user's password
@@ -216,17 +223,20 @@ msi_client_id, or msi_mi_res_id parameters.`,
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name:     "msi_object_id",
-			Help:     "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
-			Advanced: true,
+			Name:      "msi_object_id",
+			Help:      "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_mi_res_id specified.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "msi_client_id",
-			Help:     "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.",
-			Advanced: true,
+			Name:      "msi_client_id",
+			Help:      "Object ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_object_id or msi_mi_res_id specified.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "msi_mi_res_id",
-			Help:     "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
-			Advanced: true,
+			Name:      "msi_mi_res_id",
+			Help:      "Azure resource ID of the user-assigned MSI to use, if any.\n\nLeave blank if msi_client_id or msi_object_id specified.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "use_emulator",
 			Help:     "Uses local storage emulator if provided as 'true'.\n\nLeave blank if using real azure storage endpoint.",

backend/azureblob/azureblob_internal_test.go

@@ -1,5 +1,5 @@
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,7 +1,7 @@
 // Test AzureBlob filesystem interface
 
-//go:build !plan9 && !solaris && !js && go1.18
-// +build !plan9,!solaris,!js,go1.18
+//go:build !plan9 && !solaris && !js
+// +build !plan9,!solaris,!js
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,7 +1,7 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-//go:build plan9 || solaris || js || !go1.18
-// +build plan9 solaris js !go1.18
+//go:build plan9 || solaris || js
+// +build plan9 solaris js
 
 package azureblob

backend/b2/b2.go

@@ -75,13 +75,15 @@ func init() {
 		Description: "Backblaze B2",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "account",
-			Help:     "Account ID or Application Key ID.",
-			Required: true,
+			Name:      "account",
+			Help:      "Account ID or Application Key ID.",
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:     "key",
-			Help:     "Application Key.",
-			Required: true,
+			Name:      "key",
+			Help:      "Application Key.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for the service.\n\nLeave blank normally.",

backend/box/box.go

@@ -77,7 +77,7 @@ var (
 )
 
 type boxCustomClaims struct {
-	jwt.RegisteredClaims
+	jwt.StandardClaims
 	BoxSubType string `json:"box_sub_type,omitempty"`
 }
 
@@ -107,16 +107,18 @@ func init() {
 			return nil, nil
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
-			Name:     "root_folder_id",
-			Help:     "Fill in for rclone to use a non root folder as its starting point.",
-			Default:  "0",
-			Advanced: true,
+			Name:      "root_folder_id",
+			Help:      "Fill in for rclone to use a non root folder as its starting point.",
+			Default:   "0",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "box_config_file",
 			Help: "Box App config.json location\n\nLeave blank normally." + env.ShellExpandHelp,
 		}, {
-			Name: "access_token",
-			Help: "Box App Primary Access Token\n\nLeave blank normally.",
+			Name:      "access_token",
+			Help:      "Box App Primary Access Token\n\nLeave blank normally.",
+			Sensitive: true,
 		}, {
 			Name:    "box_sub_type",
 			Default: "user",
@@ -206,12 +208,14 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *boxCustomC
 	}
 
 	claims = &boxCustomClaims{
-		RegisteredClaims: jwt.RegisteredClaims{
-			ID:        val,
+		//lint:ignore SA1019 since we need to use jwt.StandardClaims even if deprecated in jwt-go v4 until a more permanent solution is ready in time before jwt-go v5 where it is removed entirely
+		//nolint:staticcheck // Don't include staticcheck when running golangci-lint to avoid SA1019
+		StandardClaims: jwt.StandardClaims{
+			Id:        val,
 			Issuer:    boxConfig.BoxAppSettings.ClientID,
 			Subject:   boxConfig.EnterpriseID,
-			Audience:  jwt.ClaimStrings{tokenURL},
-			ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 45)),
+			Audience:  tokenURL,
+			ExpiresAt: time.Now().Add(time.Second * 45).Unix(),
 		},
 		BoxSubType: boxSubType,
 	}

backend/cache/cache.go

@@ -76,17 +76,19 @@ func init() {
 			Name: "plex_url",
 			Help: "The URL of the Plex server.",
 		}, {
-			Name: "plex_username",
-			Help: "The username of the Plex user.",
+			Name:      "plex_username",
+			Help:      "The username of the Plex user.",
+			Sensitive: true,
 		}, {
 			Name:       "plex_password",
 			Help:       "The password of the Plex user.",
 			IsPassword: true,
 		}, {
-			Name:     "plex_token",
-			Help:     "The plex token for authentication - auto set normally.",
-			Hide:     fs.OptionHideBoth,
-			Advanced: true,
+			Name:      "plex_token",
+			Help:      "The plex token for authentication - auto set normally.",
+			Hide:      fs.OptionHideBoth,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "plex_insecure",
 			Help:     "Skip all certificate verification when connecting to the Plex server.",

backend/cache/cache_upload_test.go

@@ -160,11 +160,11 @@ func TestInternalUploadQueueMoreFiles(t *testing.T) {
 	minSize := 5242880
 	maxSize := 10485760
 	totalFiles := 10
-	rand.Seed(time.Now().Unix())
+	randInstance := rand.New(rand.NewSource(time.Now().Unix()))
 
 	lastFile := ""
 	for i := 0; i < totalFiles; i++ {
-		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		size := int64(randInstance.Intn(maxSize-minSize) + minSize)
 		testReader := runInstance.randomReader(t, size)
 		remote := "test/" + strconv.Itoa(i) + ".bin"
 		runInstance.writeRemoteReader(t, rootFs, remote, testReader)

backend/drive/drive.go

@@ -277,20 +277,23 @@ Leave blank normally.
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
-			Name:     "service_account_credentials",
-			Help:     "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
-			Hide:     fs.OptionHideConfigurator,
-			Advanced: true,
+			Name:      "service_account_credentials",
+			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Hide:      fs.OptionHideConfigurator,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "team_drive",
-			Help:     "ID of the Shared Drive (Team Drive).",
-			Hide:     fs.OptionHideConfigurator,
-			Advanced: true,
+			Name:      "team_drive",
+			Help:      "ID of the Shared Drive (Team Drive).",
+			Hide:      fs.OptionHideConfigurator,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "auth_owner_only",
 			Default:  false,
@@ -416,10 +419,11 @@ date is used.`,
 			Help:     "Size of listing chunk 100-1000, 0 to disable.",
 			Advanced: true,
 		}, {
-			Name:     "impersonate",
-			Default:  "",
-			Help:     `Impersonate this user when using a service account.`,
-			Advanced: true,
+			Name:      "impersonate",
+			Default:   "",
+			Help:      `Impersonate this user when using a service account.`,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:    "alternate_export",
 			Default: false,
@@ -592,7 +596,8 @@ Note also that opening the folder once in the web interface (with the
 user you've authenticated rclone with) seems to be enough so that the
 resource key is no needed.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,

backend/dropbox/dropbox.go

@@ -182,8 +182,9 @@ client_secret) to use this option as currently rclone's default set of
 permissions doesn't include "members.read". This can be added once
 v1.55 or later is in use everywhere.
 `,
-			Default:  "",
-			Advanced: true,
+			Default:   "",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "shared_files",
 			Help: `Instructs rclone to work on individual shared files.

backend/fichier/api.go

@@ -408,6 +408,32 @@ func (f *Fs) moveFile(ctx context.Context, url string, folderID int, rename stri
 	return response, nil
 }
 
+func (f *Fs) moveDir(ctx context.Context, folderID int, newLeaf string, destinationFolderID int) (response *MoveDirResponse, err error) {
+	request := &MoveDirRequest{
+		FolderID:            folderID,
+		DestinationFolderID: destinationFolderID,
+		Rename:              newLeaf,
+		// DestinationUser:     destinationUser,
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/folder/mv.cgi",
+	}
+
+	response = &MoveDirResponse{}
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
+		return shouldRetry(ctx, resp, err)
+	})
+
+	if err != nil {
+		return nil, fmt.Errorf("couldn't move dir: %w", err)
+	}
+
+	return response, nil
+}
+
 func (f *Fs) copyFile(ctx context.Context, url string, folderID int, rename string) (response *CopyFileResponse, err error) {
 	request := &CopyFileRequest{
 		URLs:     []string{url},

backend/fichier/fichier.go

@@ -38,8 +38,9 @@ func init() {
 		Description: "1Fichier",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Help: "Your API Key, get it from https://1fichier.com/console/params.pl.",
-			Name: "api_key",
+			Help:      "Your API Key, get it from https://1fichier.com/console/params.pl.",
+			Name:      "api_key",
+			Sensitive: true,
 		}, {
 			Help:     "If you want to download a shared folder, add this parameter.",
 			Name:     "shared_folder",
@@ -487,6 +488,51 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return dstObj, nil
 }
 
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove.
+//
+// If destination exists then return fs.ErrorDirExists.
+//
+// This is complicated by the fact that we can't use moveDir to move
+// to a different directory AND rename at the same time as it can
+// overwrite files in the source directory.
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
+	if err != nil {
+		return err
+	}
+	srcIDnumeric, err := strconv.Atoi(srcID)
+	if err != nil {
+		return err
+	}
+	dstDirectoryIDnumeric, err := strconv.Atoi(dstDirectoryID)
+	if err != nil {
+		return err
+	}
+
+	var resp *MoveDirResponse
+	resp, err = f.moveDir(ctx, srcIDnumeric, dstLeaf, dstDirectoryIDnumeric)
+	if err != nil {
+		return fmt.Errorf("couldn't rename leaf: %w", err)
+	}
+	if resp.Status != "OK" {
+		return fmt.Errorf("couldn't rename leaf: %s", resp.Message)
+	}
+
+	srcFs.dirCache.FlushDir(srcRemote)
+	return nil
+}
+
 // Copy src to this remote using server side move operations.
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	srcObj, ok := src.(*Object)
@@ -560,6 +606,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 var (
 	_ fs.Fs              = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
+	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)

backend/fichier/structs.go

@@ -70,6 +70,22 @@ type MoveFileResponse struct {
 	URLs    []string `json:"urls"`
 }
 
+// MoveDirRequest is the request structure of the corresponding request
+type MoveDirRequest struct {
+	FolderID            int    `json:"folder_id"`
+	DestinationFolderID int    `json:"destination_folder_id,omitempty"`
+	DestinationUser     string `json:"destination_user"`
+	Rename              string `json:"rename,omitempty"`
+}
+
+// MoveDirResponse is the response structure of the corresponding request
+type MoveDirResponse struct {
+	Status  string `json:"status"`
+	Message string `json:"message"`
+	OldName string `json:"old_name"`
+	NewName string `json:"new_name"`
+}
+
 // CopyFileRequest is the request structure of the corresponding request
 type CopyFileRequest struct {
 	URLs     []string `json:"urls"`

backend/filefabric/filefabric.go

@@ -84,6 +84,7 @@ Leave blank normally.
 
 Fill in to make rclone start with directory of a given ID.
 `,
+			Sensitive: true,
 		}, {
 			Name: "permanent_token",
 			Help: `Permanent Authentication Token.
@@ -97,6 +98,7 @@ These tokens are normally valid for several years.
 
 For more info see: https://docs.storagemadeeasy.com/organisationcloud/api-tokens
 `,
+			Sensitive: true,
 		}, {
 			Name: "token",
 			Help: `Session Token.
@@ -106,7 +108,8 @@ usually valid for 1 hour.
 
 Don't set this value - rclone will set it automatically.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "token_expiry",
 			Help: `Token expiry time.

backend/ftp/ftp.go

@@ -28,6 +28,7 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/proxy"
 	"github.com/rclone/rclone/lib/readers"
 )
 
@@ -48,13 +49,15 @@ func init() {
 		Description: "FTP",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "host",
-			Help:     "FTP host to connect to.\n\nE.g. \"ftp.example.com\".",
-			Required: true,
+			Name:      "host",
+			Help:      "FTP host to connect to.\n\nE.g. \"ftp.example.com\".",
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:    "user",
-			Help:    "FTP username.",
-			Default: currentUser,
+			Name:      "user",
+			Help:      "FTP username.",
+			Default:   currentUser,
+			Sensitive: true,
 		}, {
 			Name:    "port",
 			Help:    "FTP port number.",
@@ -172,6 +175,18 @@ Enabled by default. Use 0 to disable.`,
 If this is set and no password is supplied then rclone will ask for a password
 `,
 			Advanced: true,
+		}, {
+			Name:    "socks_proxy",
+			Default: "",
+			Help: `Socks 5 proxy host.
+		
+		Supports the format user:pass@host:port, user@host:port, host:port.
+		
+		Example:
+		
+			myUser:myPass@localhost:9005
+		`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -216,6 +231,7 @@ type Options struct {
 	ShutTimeout       fs.Duration          `config:"shut_timeout"`
 	AskPassword       bool                 `config:"ask_password"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
+	SocksProxy        string               `config:"socks_proxy"`
 }
 
 // Fs represents a remote FTP server
@@ -357,7 +373,12 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 		defer func() {
 			fs.Debugf(f, "> dial: conn=%T, err=%v", conn, err)
 		}()
-		conn, err = fshttp.NewDialer(ctx).Dial(network, address)
+		baseDialer := fshttp.NewDialer(ctx)
+		if f.opt.SocksProxy != "" {
+			conn, err = proxy.SOCKS5Dial(network, address, f.opt.SocksProxy, baseDialer)
+		} else {
+			conn, err = baseDialer.Dial(network, address)
+		}
 		if err != nil {
 			return nil, err
 		}

backend/googlecloudstorage/googlecloudstorage.go

@@ -91,18 +91,21 @@ func init() {
 			})
 		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
-			Name: "project_number",
-			Help: "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
+			Name:      "project_number",
+			Help:      "Project number.\n\nOptional - needed only for list/create/delete buckets - see your developer console.",
+			Sensitive: true,
 		}, {
-			Name: "user_project",
-			Help: "User project.\n\nOptional - needed only for requester pays.",
+			Name:      "user_project",
+			Help:      "User project.\n\nOptional - needed only for requester pays.",
+			Sensitive: true,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
-			Name: "service_account_credentials",
-			Help: "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
-			Hide: fs.OptionHideBoth,
+			Name:      "service_account_credentials",
+			Help:      "Service Account Credentials JSON blob.\n\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Hide:      fs.OptionHideBoth,
+			Sensitive: true,
 		}, {
 			Name:    "anonymous",
 			Help:    "Access public buckets and objects without credentials.\n\nSet to 'true' if you just want to download files and don't configure credentials.",

backend/hdfs/hdfs.go

@@ -19,9 +19,10 @@ func init() {
 		Description: "Hadoop distributed file system",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "namenode",
-			Help:     "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
-			Required: true,
+			Name:      "namenode",
+			Help:      "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name: "username",
 			Help: "Hadoop user name.",
@@ -29,6 +30,7 @@ func init() {
 				Value: "root",
 				Help:  "Connect to hdfs as root.",
 			}},
+			Sensitive: true,
 		}, {
 			Name: "service_principal_name",
 			Help: `Kerberos service principal name for the namenode.
@@ -36,7 +38,8 @@ func init() {
 Enables KERBEROS authentication. Specifies the Service Principal Name
 (SERVICE/FQDN) for the namenode. E.g. \"hdfs/namenode.hadoop.docker\"
 for namenode running as service 'hdfs' with FQDN 'namenode.hadoop.docker'.`,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "data_transfer_protection",
 			Help: `Kerberos data transfer protection: authentication|integrity|privacy.

backend/internetarchive/internetarchive.go

@@ -133,11 +133,13 @@ Owner is able to add custom keys. Metadata feature grabs all the keys including
 		},
 
 		Options: []fs.Option{{
-			Name: "access_key_id",
-			Help: "IAS3 Access Key.\n\nLeave blank for anonymous access.\nYou can find one here: https://archive.org/account/s3.php",
+			Name:      "access_key_id",
+			Help:      "IAS3 Access Key.\n\nLeave blank for anonymous access.\nYou can find one here: https://archive.org/account/s3.php",
+			Sensitive: true,
 		}, {
-			Name: "secret_access_key",
-			Help: "IAS3 Secret Key (password).\n\nLeave blank for anonymous access.",
+			Name:      "secret_access_key",
+			Help:      "IAS3 Secret Key (password).\n\nLeave blank for anonymous access.",
+			Sensitive: true,
 		}, {
 			// their official client (https://github.com/jjjake/internetarchive) hardcodes following the two
 			Name:     "endpoint",

backend/jottacloud/jottacloud.go

@@ -88,7 +88,7 @@ func init() {
 		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config:      Config,
-		Options: []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
@@ -123,7 +123,7 @@ func init() {
 			Default: (encoder.Display |
 				encoder.EncodeWin | // :?"*<>|
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 

backend/koofr/koofr.go

@@ -61,9 +61,10 @@ func init() {
 			Default:  true,
 			Advanced: true,
 		}, {
-			Name:     "user",
-			Help:     "Your user name.",
-			Required: true,
+			Name:      "user",
+			Help:      "Your user name.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:       "password",
 			Help:       "Your password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password).",

backend/local/local.go

@@ -516,7 +516,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 								continue
 							}
 						}
-						err = fmt.Errorf("failed to read directory %q: %w", namepath, fierr)
+						fierr = fmt.Errorf("failed to get info about directory entry %q: %w", namepath, fierr)
 						fs.Errorf(dir, "%v", fierr)
 						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue

backend/local/local_internal_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/fs/filter"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
@@ -514,3 +515,43 @@ func TestFilterSymlinkCopyLinks(t *testing.T) {
 func TestFilterSymlinkLinks(t *testing.T) {
 	testFilterSymlink(t, false)
 }
+
+func TestCopySymlink(t *testing.T) {
+	ctx := context.Background()
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	when := time.Now()
+	f := r.Flocal.(*Fs)
+
+	// Create a file and a symlink to it
+	r.WriteFile("src/file.txt", "hello world", when)
+	require.NoError(t, os.Symlink("file.txt", filepath.Join(r.LocalName, "src", "link.txt")))
+	defer func() {
+		// Reset -L/-l mode
+		f.opt.FollowSymlinks = false
+		f.opt.TranslateSymlinks = false
+		f.lstat = os.Lstat
+	}()
+
+	// Set fs into "-l/--links" mode
+	f.opt.FollowSymlinks = false
+	f.opt.TranslateSymlinks = true
+	f.lstat = os.Lstat
+
+	// Create dst
+	require.NoError(t, f.Mkdir(ctx, "dst"))
+
+	// Do copy from src into dst
+	src, err := f.NewObject(ctx, "src/link.txt.rclonelink")
+	require.NoError(t, err)
+	require.NotNil(t, src)
+	dst, err := operations.Copy(ctx, f, nil, "dst/link.txt.rclonelink", src)
+	require.NoError(t, err)
+	require.NotNil(t, dst)
+
+	// Test that we made a symlink and it has the right contents
+	dstPath := filepath.Join(r.LocalName, "dst", "link.txt")
+	linkContents, err := os.Readlink(dstPath)
+	require.NoError(t, err)
+	assert.Equal(t, "file.txt", linkContents)
+}

backend/mailru/mailru.go

@@ -85,10 +85,11 @@ func init() {
 		Name:        "mailru",
 		Description: "Mail.ru Cloud",
 		NewFs:       NewFs,
-		Options: []fs.Option{{
-			Name:     "user",
-			Help:     "User name (usually email).",
-			Required: true,
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
+			Name:      "user",
+			Help:      "User name (usually email).",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name: "pass",
 			Help: `Password.
@@ -213,7 +214,7 @@ Supported quirks: atomicmkdir binlist unknowndirs`,
 				encoder.EncodeWin | // :?"*<>|
 				encoder.EncodeBackSlash |
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 

backend/mega/mega.go

@@ -58,9 +58,10 @@ func init() {
 		Description: "Mega",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "user",
-			Help:     "User name.",
-			Required: true,
+			Name:      "user",
+			Help:      "User name.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:       "pass",
 			Help:       "Password.",

backend/netstorage/netstorage.go

@@ -65,11 +65,13 @@ HTTP is provided primarily for debugging purposes.`,
 			Help: `Domain+path of NetStorage host to connect to.
 
 Format should be ` + "`<domain>/<internal folders>`",
-			Required: true,
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:     "account",
-			Help:     "Set the NetStorage account name",
-			Required: true,
+			Name:      "account",
+			Help:      "Set the NetStorage account name",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name: "secret",
 			Help: `Set the NetStorage account secret/G2O key for authentication.

backend/onedrive/onedrive.go

@@ -131,10 +131,11 @@ Note that the chunks will be buffered into memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
-			Name:     "drive_id",
-			Help:     "The ID of the drive to use.",
-			Default:  "",
-			Advanced: true,
+			Name:      "drive_id",
+			Help:      "The ID of the drive to use.",
+			Default:   "",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "drive_type",
 			Help:     "The type of the drive (" + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + ").",
@@ -148,7 +149,8 @@ This isn't normally needed, but in special circumstances you might
 know the folder ID that you wish to access but not be able to get
 there through a path traversal.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "access_scopes",
 			Help: `Set scopes to be requested by rclone.
@@ -260,7 +262,8 @@ this flag there.
 
 At the time of writing this only works with OneDrive personal paid accounts.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:    "hash_type",
 			Default: "auto",

backend/opendrive/opendrive.go

@@ -42,9 +42,10 @@ func init() {
 		Description: "OpenDrive",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "username",
-			Help:     "Username.",
-			Required: true,
+			Name:      "username",
+			Help:      "Username.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:       "password",
 			Help:       "Password.",

backend/oracleobjectstorage/options.go

@@ -92,14 +92,16 @@ func newOptions() []fs.Option {
 			Help:  noAuthHelpText,
 		}},
 	}, {
-		Name:     "namespace",
-		Help:     "Object storage namespace",
-		Required: true,
+		Name:      "namespace",
+		Help:      "Object storage namespace",
+		Required:  true,
+		Sensitive: true,
 	}, {
-		Name:     "compartment",
-		Help:     "Object storage compartment OCID",
-		Provider: "!no_auth",
-		Required: true,
+		Name:      "compartment",
+		Help:      "Object storage compartment OCID",
+		Provider:  "!no_auth",
+		Required:  true,
+		Sensitive: true,
 	}, {
 		Name:     "region",
 		Help:     "Object storage Region",

backend/pcloud/pcloud.go

@@ -110,10 +110,11 @@ func init() {
 				encoder.EncodeBackSlash |
 				encoder.EncodeInvalidUtf8),
 		}, {
-			Name:     "root_folder_id",
-			Help:     "Fill in for rclone to use a non root folder as its starting point.",
-			Default:  "d0",
-			Advanced: true,
+			Name:      "root_folder_id",
+			Help:      "Fill in for rclone to use a non root folder as its starting point.",
+			Default:   "d0",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "hostname",
 			Help: `Hostname to connect to.
@@ -138,7 +139,8 @@ with rclone authorize.
 This is only required when you want to use the cleanup command. Due to a bug
 in the pcloud API the required API does not support OAuth authentication so
 we have to rely on user password authentication for it.`,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:       "password",
 			Help:       "Your pcloud password.",

backend/pikpak/pikpak.go

@@ -158,9 +158,10 @@ func init() {
 			return nil, fmt.Errorf("unknown state %q", config.State)
 		},
 		Options: append(pikpakOAuthOptions(), []fs.Option{{
-			Name:     "user",
-			Help:     "Pikpak username.",
-			Required: true,
+			Name:      "user",
+			Help:      "Pikpak username.",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			Name:       "pass",
 			Help:       "Pikpak password.",
@@ -173,7 +174,8 @@ Leave blank normally.
 
 Fill in for rclone to use a non root folder as its starting point.
 `,
-			Advanced: true,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "use_trash",
 			Default:  true,

backend/premiumizeme/premiumizeme.go

@@ -82,14 +82,15 @@ func init() {
 				OAuth2Config: oauthConfig,
 			})
 		},
-		Options: []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "api_key",
 			Help: `API Key.
 
 This is not normally used - use oauth instead.
 `,
-			Hide:    fs.OptionHideBoth,
-			Default: "",
+			Hide:      fs.OptionHideBoth,
+			Default:   "",
+			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -99,7 +100,7 @@ This is not normally used - use oauth instead.
 				encoder.EncodeBackSlash |
 				encoder.EncodeDoubleQuote |
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 

backend/protondrive/protondrive_test.go

@@ -0,0 +1,16 @@
+package protondrive_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/protondrive"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestProtonDrive:",
+		NilObject:  (*protondrive.Object)(nil),
+	})
+}

backend/putio/putio.go

@@ -67,7 +67,7 @@ func init() {
 				NoOffline:    true,
 			})
 		},
-		Options: []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
@@ -77,7 +77,7 @@ func init() {
 			Default: (encoder.Display |
 				encoder.EncodeBackSlash |
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 

backend/qingstor/qingstor.go

@@ -49,11 +49,13 @@ func init() {
 				Help:  "Get QingStor credentials from the environment (env vars or IAM).",
 			}},
 		}, {
-			Name: "access_key_id",
-			Help: "QingStor Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+			Name:      "access_key_id",
+			Help:      "QingStor Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+			Sensitive: true,
 		}, {
-			Name: "secret_access_key",
-			Help: "QingStor Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+			Name:      "secret_access_key",
+			Help:      "QingStor Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+			Sensitive: true,
 		}, {
 			Name: "endpoint",
 			Help: "Enter an endpoint URL to connection QingStor API.\n\nLeave blank will use the default value \"https://qingstor.com:443\".",

backend/s3/s3.go

@@ -66,7 +66,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi",
+		Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Leviia, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi",
 		NewFs:       NewFs,
 		CommandHelp: commandHelp,
 		Config: func(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) {
@@ -127,6 +127,9 @@ func init() {
 			}, {
 				Value: "LyveCloud",
 				Help:  "Seagate Lyve Cloud",
+			}, {
+				Value: "Leviia",
+				Help:  "Leviia Object Storage",
 			}, {
 				Value: "Liara",
 				Help:  "Liara Object Storage",
@@ -154,6 +157,9 @@ func init() {
 			}, {
 				Value: "Storj",
 				Help:  "Storj (S3 Compatible Gateway)",
+			}, {
+				Value: "Synology",
+				Help:  "Synology C2 Object Storage",
 			}, {
 				Value: "TencentCOS",
 				Help:  "Tencent Cloud Object Storage (COS)",
@@ -179,11 +185,13 @@ func init() {
 				Help:  "Get AWS credentials from the environment (env vars or IAM).",
 			}},
 		}, {
-			Name: "access_key_id",
-			Help: "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+			Name:      "access_key_id",
+			Help:      "AWS Access Key ID.\n\nLeave blank for anonymous access or runtime credentials.",
+			Sensitive: true,
 		}, {
-			Name: "secret_access_key",
-			Help: "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+			Name:      "secret_access_key",
+			Help:      "AWS Secret Access Key (password).\n\nLeave blank for anonymous access or runtime credentials.",
+			Sensitive: true,
 		}, {
 			// References:
 			// 1. https://docs.aws.amazon.com/general/latest/gr/rande.html
@@ -463,10 +471,30 @@ func init() {
 				Value: "sa-east-1",
 				Help:  "South America (São Paulo)",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region where your data stored.\n",
+			Provider: "Synology",
+			Examples: []fs.OptionExample{{
+				Value: "eu-001",
+				Help:  "Europe Region 1",
+			}, {
+				Value: "eu-002",
+				Help:  "Europe Region 2",
+			}, {
+				Value: "us-001",
+				Help:  "US Region 1",
+			}, {
+				Value: "us-002",
+				Help:  "US Region 2",
+			}, {
+				Value: "tw-001",
+				Help:  "Asia (Taiwan)",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive",
+			Provider: "!AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Qiniu,RackCorp,Scaleway,Storj,Synology,TencentCOS,HuaweiOBS,IDrive",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -815,6 +843,15 @@ func init() {
 				Value: "s3.sa-east-1.petabox.io",
 				Help:  "South America (São Paulo)",
 			}},
+		}, {
+			// Leviia endpoints: https://www.leviia.com/object-storage/
+			Name:     "endpoint",
+			Help:     "Endpoint for Leviia Object Storage API.",
+			Provider: "Leviia",
+			Examples: []fs.OptionExample{{
+				Value: "s3.leviia.com",
+				Help:  "The default endpoint\nLeviia",
+			}},
 		}, {
 			// Liara endpoints: https://liara.ir/landing/object-storage
 			Name:     "endpoint",
@@ -1000,6 +1037,26 @@ func init() {
 				Value: "gateway.storjshare.io",
 				Help:  "Global Hosted Gateway",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Synology C2 Object Storage API.",
+			Provider: "Synology",
+			Examples: []fs.OptionExample{{
+				Value: "eu-001.s3.synologyc2.net",
+				Help:  "EU Endpoint 1",
+			}, {
+				Value: "eu-002.s3.synologyc2.net",
+				Help:  "EU Endpoint 2",
+			}, {
+				Value: "us-001.s3.synologyc2.net",
+				Help:  "US Endpoint 1",
+			}, {
+				Value: "us-002.s3.synologyc2.net",
+				Help:  "US Endpoint 2",
+			}, {
+				Value: "tw-001.s3.synologyc2.net",
+				Help:  "TW Endpoint 1",
+			}},
 		}, {
 			// cos endpoints: https://intl.cloud.tencent.com/document/product/436/6224
 			Name:     "endpoint",
@@ -1156,7 +1213,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
-			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Scaleway,StackPath,Storj,RackCorp,Qiniu,Petabox",
+			Provider: "!AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Scaleway,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -1644,7 +1701,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Liara,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS,Petabox",
+			Provider: "!AWS,Alibaba,ArvanCloud,HuaweiOBS,ChinaMobile,Cloudflare,IBMCOS,IDrive,IONOS,Leviia,Liara,Qiniu,RackCorp,Scaleway,StackPath,Storj,TencentCOS,Petabox",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1659,7 +1716,7 @@ doesn't copy the ACL from the source but rather writes a fresh one.
 If the acl is an empty string then no X-Amz-Acl: header is added and
 the default (private) will be used.
 `,
-			Provider: "!Storj,Cloudflare",
+			Provider: "!Storj,Synology,Cloudflare",
 			Examples: []fs.OptionExample{{
 				Value:    "default",
 				Help:     "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -1775,6 +1832,7 @@ header is added and the default (private) will be used.
 				Value: "arn:aws:kms:us-east-1:*",
 				Help:  "arn:aws:kms:*",
 			}},
+			Sensitive: true,
 		}, {
 			Name: "sse_customer_key",
 			Help: `To use SSE-C you may provide the secret encryption key used to encrypt/decrypt your data.
@@ -1786,6 +1844,7 @@ Alternatively you can provide --sse-customer-key-base64.`,
 				Value: "",
 				Help:  "None",
 			}},
+			Sensitive: true,
 		}, {
 			Name: "sse_customer_key_base64",
 			Help: `If using SSE-C you must provide the secret encryption key encoded in base64 format to encrypt/decrypt your data.
@@ -1797,6 +1856,7 @@ Alternatively you can provide --sse-customer-key.`,
 				Value: "",
 				Help:  "None",
 			}},
+			Sensitive: true,
 		}, {
 			Name: "sse_customer_key_md5",
 			Help: `If using SSE-C you may provide the secret encryption key MD5 checksum (optional).
@@ -1809,6 +1869,7 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
 				Value: "",
 				Help:  "None",
 			}},
+			Sensitive: true,
 		}, {
 			Name:     "storage_class",
 			Help:     "The storage class to use when storing new objects in S3.",
@@ -2050,9 +2111,10 @@ If empty it will default to the environment variable "AWS_PROFILE" or
 `,
 			Advanced: true,
 		}, {
-			Name:     "session_token",
-			Help:     "An AWS session token.",
-			Advanced: true,
+			Name:      "session_token",
+			Help:      "An AWS session token.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name: "upload_concurrency",
 			Help: `Concurrency for multipart uploads.
@@ -2969,11 +3031,15 @@ func setQuirks(opt *Options) {
 		if opt.ChunkSize < 64*fs.Mebi {
 			opt.ChunkSize = 64 * fs.Mebi
 		}
+	case "Synology":
+		useMultipartEtag = false
 	case "TencentCOS":
 		listObjectsV2 = false    // untested
 		useMultipartEtag = false // untested
 	case "Wasabi":
 		// No quirks
+	case "Leviia":
+		// No quirks
 	case "Qiniu":
 		useMultipartEtag = false
 		urlEncodeListings = false
@@ -4355,17 +4421,17 @@ to normal storage.
 
 Usage Examples:
 
-    rclone backend restore s3:bucket/path/to/object [-o priority=PRIORITY] [-o lifetime=DAYS]
-    rclone backend restore s3:bucket/path/to/directory [-o priority=PRIORITY] [-o lifetime=DAYS]
-    rclone backend restore s3:bucket [-o priority=PRIORITY] [-o lifetime=DAYS]
+    rclone backend restore s3:bucket/path/to/object -o priority=PRIORITY -o lifetime=DAYS
+    rclone backend restore s3:bucket/path/to/directory -o priority=PRIORITY -o lifetime=DAYS
+    rclone backend restore s3:bucket -o priority=PRIORITY -o lifetime=DAYS
 
 This flag also obeys the filters. Test first with --interactive/-i or --dry-run flags
 
-    rclone --interactive backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+    rclone --interactive backend restore --include "*.txt" s3:bucket/path -o priority=Standard -o lifetime=1
 
 All the objects shown will be marked for restore, then
 
-    rclone backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+    rclone backend restore --include "*.txt" s3:bucket/path -o priority=Standard -o lifetime=1
 
 It returns a list of status dictionaries with Remote and Status
 keys. The Status will be OK if it was successful or an error message

backend/seafile/seafile.go

@@ -67,15 +67,18 @@ func init() {
 				Value: "https://cloud.seafile.com/",
 				Help:  "Connect to cloud.seafile.com.",
 			}},
+			Sensitive: true,
 		}, {
-			Name:     configUser,
-			Help:     "User name (usually email address).",
-			Required: true,
+			Name:      configUser,
+			Help:      "User name (usually email address).",
+			Required:  true,
+			Sensitive: true,
 		}, {
 			// Password is not required, it will be left blank for 2FA
 			Name:       configPassword,
 			Help:       "Password.",
 			IsPassword: true,
+			Sensitive:  true,
 		}, {
 			Name:    config2FA,
 			Help:    "Two-factor authentication ('true' if the account has 2FA enabled).",
@@ -87,6 +90,7 @@ func init() {
 			Name:       configLibraryKey,
 			Help:       "Library password (for encrypted libraries only).\n\nLeave blank if you pass it through the command line.",
 			IsPassword: true,
+			Sensitive:  true,
 		}, {
 			Name:     configCreateLibrary,
 			Help:     "Should rclone create a library if it doesn't exist.",
@@ -94,9 +98,10 @@ func init() {
 			Default:  false,
 		}, {
 			// Keep the authentication token after entering the 2FA code
-			Name: configAuthToken,
-			Help: "Authentication token.",
-			Hide: fs.OptionHideBoth,
+			Name:      configAuthToken,
+			Help:      "Authentication token.",
+			Hide:      fs.OptionHideBoth,
+			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,

backend/sftp/sftp.go

@@ -27,7 +27,6 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
-	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
@@ -59,13 +58,15 @@ func init() {
 		Description: "SSH/SFTP",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Name:     "host",
-			Help:     "SSH host to connect to.\n\nE.g. \"example.com\".",
-			Required: true,
+			Name:      "host",
+			Help:      "SSH host to connect to.\n\nE.g. \"example.com\".",
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:    "user",
-			Help:    "SSH username.",
-			Default: currentUser,
+			Name:      "user",
+			Help:      "SSH username.",
+			Default:   currentUser,
+			Sensitive: true,
 		}, {
 			Name:    "port",
 			Help:    "SSH port number.",
@@ -75,8 +76,9 @@ func init() {
 			Help:       "SSH password, leave blank to use ssh-agent.",
 			IsPassword: true,
 		}, {
-			Name: "key_pem",
-			Help: "Raw PEM-encoded private key.\n\nIf specified, will override key_file parameter.",
+			Name:      "key_pem",
+			Help:      "Raw PEM-encoded private key.\n\nIf specified, will override key_file parameter.",
+			Sensitive: true,
 		}, {
 			Name: "key_file",
 			Help: "Path to PEM-encoded private key file.\n\nLeave blank or set key-use-agent to use ssh-agent." + env.ShellExpandHelp,
@@ -87,6 +89,7 @@ func init() {
 Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
 in the new OpenSSH format can't be used.`,
 			IsPassword: true,
+			Sensitive:  true,
 		}, {
 			Name: "pubkey_file",
 			Help: `Optional path to public key file.
@@ -165,7 +168,19 @@ E.g. if shared folders can be found in directories representing volumes:
 
 E.g. if home directory can be found in a shared folder called "home":
 
-    rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory`,
+    rclone sync /home/local/directory remote:/home/directory --sftp-path-override /volume1/homes/USER/directory
+	
+To specify only the path to the SFTP remote's root, and allow rclone to add any relative subpaths automatically (including unwrapping/decrypting remotes as necessary), add the '@' character to the beginning of the path.
+
+E.g. the first example above could be rewritten as:
+
+	rclone sync /home/local/directory remote:/directory --sftp-path-override @/volume2
+	
+Note that when using this method with Synology "home" folders, the full "/homes/USER" path should be specified instead of "/home".
+
+E.g. the second example above should be rewritten as:
+
+	rclone sync /home/local/directory remote:/homes/USER/directory --sftp-path-override @/volume1`,
 			Advanced: true,
 		}, {
 			Name:     "set_modtime",
@@ -384,6 +399,47 @@ Example:
     ssh-ed25519 ssh-rsa ssh-dss
 `,
 			Advanced: true,
+		}, {
+			Name:    "ssh",
+			Default: fs.SpaceSepList{},
+			Help: `Path and arguments to external ssh binary.
+
+Normally rclone will use its internal ssh library to connect to the
+SFTP server. However it does not implement all possible ssh options so
+it may be desirable to use an external ssh binary.
+
+Rclone ignores all the internal config if you use this option and
+expects you to configure the ssh binary with the user/host/port and
+any other options you need.
+
+**Important** The ssh command must log in without asking for a
+password so needs to be configured with keys or certificates.
+
+Rclone will run the command supplied either with the additional
+arguments "-s sftp" to access the SFTP subsystem or with commands such
+as "md5sum /path/to/file" appended to read checksums.
+
+Any arguments with spaces in should be surrounded by "double quotes".
+
+An example setting might be:
+
+    ssh -o ServerAliveInterval=20 user@example.com
+
+Note that when using an external ssh binary rclone makes a new ssh
+connection for every hash it calculates.
+`,
+		}, {
+			Name:    "socks_proxy",
+			Default: "",
+			Help: `Socks 5 proxy host.
+	
+Supports the format user:pass@host:port, user@host:port, host:port.
+
+Example:
+
+	myUser:myPass@localhost:9005
+	`,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -423,6 +479,8 @@ type Options struct {
 	KeyExchange             fs.SpaceSepList `config:"key_exchange"`
 	MACs                    fs.SpaceSepList `config:"macs"`
 	HostKeyAlgorithms       fs.SpaceSepList `config:"host_key_algorithms"`
+	SSH                     fs.SpaceSepList `config:"ssh"`
+	SocksProxy              string          `config:"socks_proxy"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -459,41 +517,16 @@ type Object struct {
 	sha1sum *string     // Cached SHA1 checksum
 }
 
-// dial starts a client connection to the given SSH server. It is a
-// convenience function that connects to the given network address,
-// initiates the SSH handshake, and then sets up a Client.
-func (f *Fs) dial(ctx context.Context, network, addr string, sshConfig *ssh.ClientConfig) (*ssh.Client, error) {
-	dialer := fshttp.NewDialer(ctx)
-	conn, err := dialer.Dial(network, addr)
-	if err != nil {
-		return nil, err
-	}
-	c, chans, reqs, err := ssh.NewClientConn(conn, addr, sshConfig)
-	if err != nil {
-		return nil, err
-	}
-	fs.Debugf(f, "New connection %s->%s to %q", c.LocalAddr(), c.RemoteAddr(), c.ServerVersion())
-	return ssh.NewClient(c, chans, reqs), nil
-}
-
 // conn encapsulates an ssh client and corresponding sftp client
 type conn struct {
-	sshClient  *ssh.Client
+	sshClient  sshClient
 	sftpClient *sftp.Client
 	err        chan error
 }
 
 // Wait for connection to close
 func (c *conn) wait() {
-	c.err <- c.sshClient.Conn.Wait()
-}
-
-// Send a keepalive over the ssh connection
-func (c *conn) sendKeepAlive() {
-	_, _, err := c.sshClient.SendRequest("keepalive@openssh.com", true, nil)
-	if err != nil {
-		fs.Debugf(nil, "Failed to send keep alive: %v", err)
-	}
+	c.err <- c.sshClient.Wait()
 }
 
 // Send keepalives every interval over the ssh connection until done is closed
@@ -505,7 +538,7 @@ func (c *conn) sendKeepAlives(interval time.Duration) (done chan struct{}) {
 		for {
 			select {
 			case <-t.C:
-				c.sendKeepAlive()
+				c.sshClient.SendKeepAlive()
 			case <-done:
 				return
 			}
@@ -557,7 +590,11 @@ func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
 	c = &conn{
 		err: make(chan error, 1),
 	}
-	c.sshClient, err = f.dial(ctx, "tcp", f.opt.Host+":"+f.opt.Port, f.config)
+	if len(f.opt.SSH) == 0 {
+		c.sshClient, err = f.newSSHClientInternal(ctx, "tcp", f.opt.Host+":"+f.opt.Port, f.config)
+	} else {
+		c.sshClient, err = f.newSSHClientExternal()
+	}
 	if err != nil {
 		return nil, fmt.Errorf("couldn't connect SSH: %w", err)
 	}
@@ -571,7 +608,7 @@ func (f *Fs) sftpConnection(ctx context.Context) (c *conn, err error) {
 }
 
 // Set any environment variables on the ssh.Session
-func (f *Fs) setEnv(s *ssh.Session) error {
+func (f *Fs) setEnv(s sshSession) error {
 	for _, env := range f.opt.SetEnv {
 		equal := strings.IndexRune(env, '=')
 		if equal < 0 {
@@ -588,8 +625,8 @@ func (f *Fs) setEnv(s *ssh.Session) error {
 
 // Creates a new SFTP client on conn, using the specified subsystem
 // or sftp server, and zero or more option functions
-func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.Client, error) {
-	s, err := conn.NewSession()
+func (f *Fs) newSftpClient(client sshClient, opts ...sftp.ClientOption) (*sftp.Client, error) {
+	s, err := client.NewSession()
 	if err != nil {
 		return nil, err
 	}
@@ -662,6 +699,9 @@ func (f *Fs) getSftpConnection(ctx context.Context) (c *conn, err error) {
 // Getwd request
 func (f *Fs) putSftpConnection(pc **conn, err error) {
 	c := *pc
+	if !c.sshClient.CanReuse() {
+		return
+	}
 	*pc = nil
 	if err != nil {
 		// work out if this is an expected error
@@ -740,6 +780,10 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if err != nil {
 		return nil, err
 	}
+	if len(opt.SSH) != 0 && (opt.User != "" || opt.Host != "" || opt.Port != "") {
+		fs.Logf(name, "--sftp-ssh is in use - ignoring user/host/port from config - set in the parameters to --sftp-ssh (remove them from the config to silence this warning)")
+	}
+
 	if opt.User == "" {
 		opt.User = currentUser
 	}
@@ -1012,8 +1056,8 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 			fs.Debugf(f, "Failed to get shell session for shell type detection command: %v", err)
 		} else {
 			var stdout, stderr bytes.Buffer
-			session.Stdout = &stdout
-			session.Stderr = &stderr
+			session.SetStdout(&stdout)
+			session.SetStderr(&stderr)
 			shellCmd := "echo ${ShellId}%ComSpec%"
 			fs.Debugf(f, "Running shell type detection remote command: %s", shellCmd)
 			err = session.Run(shellCmd)
@@ -1423,8 +1467,8 @@ func (f *Fs) run(ctx context.Context, cmd string) ([]byte, error) {
 	}()
 
 	var stdout, stderr bytes.Buffer
-	session.Stdout = &stdout
-	session.Stderr = &stderr
+	session.SetStdout(&stdout)
+	session.SetStderr(&stderr)
 
 	fs.Debugf(f, "Running remote command: %s", cmd)
 	err = session.Run(cmd)
@@ -1731,6 +1775,9 @@ func (f *Fs) remotePath(remote string) string {
 func (f *Fs) remoteShellPath(remote string) string {
 	if f.opt.PathOverride != "" {
 		shellPath := path.Join(f.opt.PathOverride, remote)
+		if f.opt.PathOverride[0] == '@' {
+			shellPath = path.Join(strings.TrimPrefix(f.opt.PathOverride, "@"), f.absRoot, remote)
+		}
 		fs.Debugf(f, "Shell path redirected to %q with option path_override", shellPath)
 		return shellPath
 	}
@@ -1988,9 +2035,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return fmt.Errorf("Update: %w", err)
 	}
+	// Hang on to the connection for the whole upload so it doesn't get re-used while we are uploading
 	file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
-	o.fs.putSftpConnection(&c, err)
 	if err != nil {
+		o.fs.putSftpConnection(&c, err)
 		return fmt.Errorf("Update Create failed: %w", err)
 	}
 	// remove the file if upload failed
@@ -2010,14 +2058,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	_, err = file.ReadFrom(&sizeReader{Reader: in, size: src.Size()})
 	if err != nil {
+		o.fs.putSftpConnection(&c, err)
 		remove()
 		return fmt.Errorf("Update ReadFrom failed: %w", err)
 	}
 	err = file.Close()
 	if err != nil {
+		o.fs.putSftpConnection(&c, err)
 		remove()
 		return fmt.Errorf("Update Close failed: %w", err)
 	}
+	// Release connection only when upload has finished so we don't upload multiple files on the same connection
+	o.fs.putSftpConnection(&c, err)
 
 	// Set the mod time - this stats the object if o.fs.opt.SetModTime == true
 	err = o.SetModTime(ctx, src.ModTime(ctx))

backend/sftp/sftp_test.go

@@ -30,3 +30,13 @@ func TestIntegration2(t *testing.T) {
 		NilObject:  (*sftp.Object)(nil),
 	})
 }
+
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSFTPRcloneSSH:",
+		NilObject:  (*sftp.Object)(nil),
+	})
+}

backend/sftp/ssh.go

@@ -0,0 +1,73 @@
+//go:build !plan9
+// +build !plan9
+
+package sftp
+
+import "io"
+
+// Interfaces for ssh client and session implemented in ssh_internal.go and ssh_external.go
+
+// An interface for an ssh client to abstract over internal ssh library and external binary
+type sshClient interface {
+	// Wait blocks until the connection has shut down, and returns the
+	// error causing the shutdown.
+	Wait() error
+
+	// SendKeepAlive sends a keepalive message to keep the connection open
+	SendKeepAlive()
+
+	// Close the connection
+	Close() error
+
+	// NewSession opens a new sshSession for this sshClient. (A
+	// session is a remote execution of a program.)
+	NewSession() (sshSession, error)
+
+	// CanReuse indicates if this client can be reused
+	CanReuse() bool
+}
+
+// An interface for an ssh session to abstract over internal ssh library and external binary
+type sshSession interface {
+	// Setenv sets an environment variable that will be applied to any
+	// command executed by Shell or Run.
+	Setenv(name, value string) error
+
+	// Start runs cmd on the remote host. Typically, the remote
+	// server passes cmd to the shell for interpretation.
+	// A Session only accepts one call to Run, Start or Shell.
+	Start(cmd string) error
+
+	// StdinPipe returns a pipe that will be connected to the
+	// remote command's standard input when the command starts.
+	StdinPipe() (io.WriteCloser, error)
+
+	// StdoutPipe returns a pipe that will be connected to the
+	// remote command's standard output when the command starts.
+	// There is a fixed amount of buffering that is shared between
+	// stdout and stderr streams. If the StdoutPipe reader is
+	// not serviced fast enough it may eventually cause the
+	// remote command to block.
+	StdoutPipe() (io.Reader, error)
+
+	// RequestSubsystem requests the association of a subsystem
+	// with the session on the remote host. A subsystem is a
+	// predefined command that runs in the background when the ssh
+	// session is initiated
+	RequestSubsystem(subsystem string) error
+
+	// Run runs cmd on the remote host. Typically, the remote
+	// server passes cmd to the shell for interpretation.
+	// A Session only accepts one call to Run, Start, Shell, Output,
+	// or CombinedOutput.
+	Run(cmd string) error
+
+	// Close the session
+	Close() error
+
+	// Set the stdout
+	SetStdout(io.Writer)
+
+	// Set the stderr
+	SetStderr(io.Writer)
+}

backend/sftp/ssh_external.go

@@ -0,0 +1,223 @@
+//go:build !plan9
+// +build !plan9
+
+package sftp
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+
+	"github.com/rclone/rclone/fs"
+)
+
+// Implement the sshClient interface for external ssh programs
+type sshClientExternal struct {
+	f       *Fs
+	session *sshSessionExternal
+}
+
+func (f *Fs) newSSHClientExternal() (sshClient, error) {
+	return &sshClientExternal{f: f}, nil
+}
+
+// Wait for connection to close
+func (s *sshClientExternal) Wait() error {
+	if s.session == nil {
+		return nil
+	}
+	return s.session.Wait()
+}
+
+// Send a keepalive over the ssh connection
+func (s *sshClientExternal) SendKeepAlive() {
+	// Up to the user to configure -o ServerAliveInterval=20 on their ssh connections
+}
+
+// Close the connection
+func (s *sshClientExternal) Close() error {
+	if s.session == nil {
+		return nil
+	}
+	return s.session.Close()
+}
+
+// NewSession makes a new external SSH connection
+func (s *sshClientExternal) NewSession() (sshSession, error) {
+	session := s.f.newSshSessionExternal()
+	if s.session == nil {
+		fs.Debugf(s.f, "ssh external: creating additional session")
+	}
+	return session, nil
+}
+
+// CanReuse indicates if this client can be reused
+func (s *sshClientExternal) CanReuse() bool {
+	if s.session == nil {
+		return true
+	}
+	exited := s.session.exited()
+	canReuse := !exited && s.session.runningSFTP
+	// fs.Debugf(s.f, "ssh external: CanReuse %v, exited=%v runningSFTP=%v", canReuse, exited, s.session.runningSFTP)
+	return canReuse
+}
+
+// Check interfaces
+var _ sshClient = &sshClientExternal{}
+
+// implement the sshSession interface for external ssh binary
+type sshSessionExternal struct {
+	f           *Fs
+	cmd         *exec.Cmd
+	cancel      func()
+	startCalled bool
+	runningSFTP bool
+}
+
+func (f *Fs) newSshSessionExternal() *sshSessionExternal {
+	s := &sshSessionExternal{
+		f: f,
+	}
+
+	// Make a cancellation function for this to call in Close()
+	ctx, cancel := context.WithCancel(context.Background())
+	s.cancel = cancel
+
+	// Connect to a remote host and request the sftp subsystem via
+	// the 'ssh' command. This assumes that passwordless login is
+	// correctly configured.
+	ssh := append([]string(nil), s.f.opt.SSH...)
+	s.cmd = exec.CommandContext(ctx, ssh[0], ssh[1:]...)
+
+	// Allow the command a short time only to shut down
+	// FIXME enable when we get rid of go1.19
+	// s.cmd.WaitDelay = time.Second
+
+	return s
+}
+
+// Setenv sets an environment variable that will be applied to any
+// command executed by Shell or Run.
+func (s *sshSessionExternal) Setenv(name, value string) error {
+	return errors.New("ssh external: can't set environment variables")
+}
+
+const requestSubsystem = "***Subsystem***:"
+
+// Start runs cmd on the remote host. Typically, the remote
+// server passes cmd to the shell for interpretation.
+// A Session only accepts one call to Run, Start or Shell.
+func (s *sshSessionExternal) Start(cmd string) error {
+	if s.startCalled {
+		return errors.New("internal error: ssh external: command already running")
+	}
+	s.startCalled = true
+
+	// Adjust the args
+	if strings.HasPrefix(cmd, requestSubsystem) {
+		s.cmd.Args = append(s.cmd.Args, "-s", cmd[len(requestSubsystem):])
+		s.runningSFTP = true
+	} else {
+		s.cmd.Args = append(s.cmd.Args, cmd)
+		s.runningSFTP = false
+	}
+
+	fs.Debugf(s.f, "ssh external: running: %v", fs.SpaceSepList(s.cmd.Args))
+
+	// start the process
+	err := s.cmd.Start()
+	if err != nil {
+		return fmt.Errorf("ssh external: start process: %w", err)
+	}
+
+	return nil
+}
+
+// RequestSubsystem requests the association of a subsystem
+// with the session on the remote host. A subsystem is a
+// predefined command that runs in the background when the ssh
+// session is initiated
+func (s *sshSessionExternal) RequestSubsystem(subsystem string) error {
+	return s.Start(requestSubsystem + subsystem)
+}
+
+// StdinPipe returns a pipe that will be connected to the
+// remote command's standard input when the command starts.
+func (s *sshSessionExternal) StdinPipe() (io.WriteCloser, error) {
+	rd, err := s.cmd.StdinPipe()
+	if err != nil {
+		return nil, fmt.Errorf("ssh external: stdin pipe: %w", err)
+	}
+	return rd, nil
+}
+
+// StdoutPipe returns a pipe that will be connected to the
+// remote command's standard output when the command starts.
+// There is a fixed amount of buffering that is shared between
+// stdout and stderr streams. If the StdoutPipe reader is
+// not serviced fast enough it may eventually cause the
+// remote command to block.
+func (s *sshSessionExternal) StdoutPipe() (io.Reader, error) {
+	wr, err := s.cmd.StdoutPipe()
+	if err != nil {
+		return nil, fmt.Errorf("ssh external: stdout pipe: %w", err)
+	}
+	return wr, nil
+}
+
+// Return whether the command has finished or not
+func (s *sshSessionExternal) exited() bool {
+	return s.cmd.ProcessState != nil
+}
+
+// Wait for the command to exit
+func (s *sshSessionExternal) Wait() error {
+	if s.exited() {
+		return nil
+	}
+	err := s.cmd.Wait()
+	if err == nil {
+		fs.Debugf(s.f, "ssh external: command exited OK")
+	} else {
+		fs.Debugf(s.f, "ssh external: command exited with error: %v", err)
+	}
+	return err
+}
+
+// Run runs cmd on the remote host. Typically, the remote
+// server passes cmd to the shell for interpretation.
+// A Session only accepts one call to Run, Start, Shell, Output,
+// or CombinedOutput.
+func (s *sshSessionExternal) Run(cmd string) error {
+	err := s.Start(cmd)
+	if err != nil {
+		return err
+	}
+	return s.Wait()
+}
+
+// Close the external ssh
+func (s *sshSessionExternal) Close() error {
+	fs.Debugf(s.f, "ssh external: close")
+	// Cancel the context which kills the process
+	s.cancel()
+	// Wait for it to finish
+	_ = s.Wait()
+	return nil
+}
+
+// Set the stdout
+func (s *sshSessionExternal) SetStdout(wr io.Writer) {
+	s.cmd.Stdout = wr
+}
+
+// Set the stderr
+func (s *sshSessionExternal) SetStderr(wr io.Writer) {
+	s.cmd.Stderr = wr
+}
+
+// Check interfaces
+var _ sshSession = &sshSessionExternal{}

backend/sftp/ssh_internal.go

@@ -0,0 +1,101 @@
+//go:build !plan9
+// +build !plan9
+
+package sftp
+
+import (
+	"context"
+	"io"
+	"net"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/proxy"
+	"golang.org/x/crypto/ssh"
+)
+
+// Internal ssh connections with "golang.org/x/crypto/ssh"
+
+type sshClientInternal struct {
+	srv *ssh.Client
+}
+
+// newSSHClientInternal starts a client connection to the given SSH server. It is a
+// convenience function that connects to the given network address,
+// initiates the SSH handshake, and then sets up a Client.
+func (f *Fs) newSSHClientInternal(ctx context.Context, network, addr string, sshConfig *ssh.ClientConfig) (sshClient, error) {
+
+	baseDialer := fshttp.NewDialer(ctx)
+	var (
+		conn net.Conn
+		err  error
+	)
+	if f.opt.SocksProxy != "" {
+		conn, err = proxy.SOCKS5Dial(network, addr, f.opt.SocksProxy, baseDialer)
+	} else {
+		conn, err = baseDialer.Dial(network, addr)
+	}
+	if err != nil {
+		return nil, err
+	}
+	c, chans, reqs, err := ssh.NewClientConn(conn, addr, sshConfig)
+	if err != nil {
+		return nil, err
+	}
+	fs.Debugf(f, "New connection %s->%s to %q", c.LocalAddr(), c.RemoteAddr(), c.ServerVersion())
+	srv := ssh.NewClient(c, chans, reqs)
+	return sshClientInternal{srv}, nil
+}
+
+// Wait for connection to close
+func (s sshClientInternal) Wait() error {
+	return s.srv.Conn.Wait()
+}
+
+// Send a keepalive over the ssh connection
+func (s sshClientInternal) SendKeepAlive() {
+	_, _, err := s.srv.SendRequest("keepalive@openssh.com", true, nil)
+	if err != nil {
+		fs.Debugf(nil, "Failed to send keep alive: %v", err)
+	}
+}
+
+// Close the connection
+func (s sshClientInternal) Close() error {
+	return s.srv.Close()
+}
+
+// CanReuse indicates if this client can be reused
+func (s sshClientInternal) CanReuse() bool {
+	return true
+}
+
+// Check interfaces
+var _ sshClient = sshClientInternal{}
+
+// Thin wrapper for *ssh.Session to implement sshSession interface
+type sshSessionInternal struct {
+	*ssh.Session
+}
+
+// Set the stdout
+func (s sshSessionInternal) SetStdout(wr io.Writer) {
+	s.Session.Stdout = wr
+}
+
+// Set the stderr
+func (s sshSessionInternal) SetStderr(wr io.Writer) {
+	s.Session.Stderr = wr
+}
+
+// NewSession makes an sshSession from an sshClient
+func (s sshClientInternal) NewSession() (sshSession, error) {
+	session, err := s.srv.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	return sshSessionInternal{Session: session}, nil
+}
+
+// Check interfaces
+var _ sshSession = sshSessionInternal{}

backend/sharefile/sharefile.go

@@ -155,7 +155,7 @@ func init() {
 				CheckAuth:    checkAuth,
 			})
 		},
-		Options: []fs.Option{{
+		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "upload_cutoff",
 			Help:     "Cutoff for switching to multipart upload.",
 			Default:  defaultUploadCutoff,
@@ -182,6 +182,7 @@ standard values here or any folder ID (long hex number ID).`,
 				Value: "top",
 				Help:  "Access the home, favorites, and shared folders as well as the connectors.",
 			}},
+			Sensitive: true,
 		}, {
 			Name:    "chunk_size",
 			Default: defaultChunkSize,
@@ -216,7 +217,7 @@ be set manually to something like: https://XXX.sharefile.com
 				encoder.EncodeLeftSpace |
 				encoder.EncodeLeftPeriod |
 				encoder.EncodeInvalidUtf8),
-		}},
+		}}...),
 	})
 }
 

backend/sia/sia.go

@@ -45,7 +45,8 @@ func init() {
 
 Note that siad must run with --disable-api-security to open API port for other hosts (not recommended).
 Keep default if Sia daemon runs on localhost.`,
-			Default: "http://127.0.0.1:9980",
+			Default:   "http://127.0.0.1:9980",
+			Sensitive: true,
 		}, {
 			Name: "api_password",
 			Help: `Sia Daemon API Password.

backend/smb/smb.go

@@ -41,13 +41,15 @@ func init() {
 		NewFs:       NewFs,
 
 		Options: []fs.Option{{
-			Name:     "host",
-			Help:     "SMB server hostname to connect to.\n\nE.g. \"example.com\".",
-			Required: true,
+			Name:      "host",
+			Help:      "SMB server hostname to connect to.\n\nE.g. \"example.com\".",
+			Required:  true,
+			Sensitive: true,
 		}, {
-			Name:    "user",
-			Help:    "SMB username.",
-			Default: currentUser,
+			Name:      "user",
+			Help:      "SMB username.",
+			Default:   currentUser,
+			Sensitive: true,
 		}, {
 			Name:    "port",
 			Help:    "SMB port number.",
@@ -57,9 +59,10 @@ func init() {
 			Help:       "SMB password.",
 			IsPassword: true,
 		}, {
-			Name:    "domain",
-			Help:    "Domain name for NTLM authentication.",
-			Default: "WORKGROUP",
+			Name:      "domain",
+			Help:      "Domain name for NTLM authentication.",
+			Default:   "WORKGROUP",
+			Sensitive: true,
 		}, {
 			Name: "spn",
 			Help: `Service principal name.
@@ -71,6 +74,7 @@ authentication, and it often needs to be set for clusters. For example:
 
 Leave blank if not sure.
 `,
+			Sensitive: true,
 		}, {
 			Name:    "idle_timeout",
 			Default: fs.Duration(60 * time.Second),
@@ -447,7 +451,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error) {
 	share, dir := f.split("/")
 	if share == "" {
-		return nil, fs.ErrorListBucketRequired
+		// Just return empty info rather than an error if called on the root
+		return &fs.Usage{}, nil
 	}
 	dir = f.toSambaPath(dir)
 
@@ -470,6 +475,45 @@ func (f *Fs) About(ctx context.Context) (_ *fs.Usage, err error) {
 	return usage, nil
 }
 
+// OpenWriterAt opens with a handle for random access writes
+//
+// Pass in the remote desired and the size if known.
+//
+// It truncates any existing object
+func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
+	var err error
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	share, filename := o.split()
+	if share == "" || filename == "" {
+		return nil, fs.ErrorIsDir
+	}
+
+	err = o.fs.ensureDirectory(ctx, share, filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make parent directories: %w", err)
+	}
+
+	filename = o.fs.toSambaPath(filename)
+
+	o.fs.addSession() // Show session in use
+	defer o.fs.removeSession()
+
+	cn, err := o.fs.getConnection(ctx, share)
+	if err != nil {
+		return nil, err
+	}
+
+	fl, err := cn.smbShare.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open: %w", err)
+	}
+
+	return fl, nil
+}
+
 // Shutdown the backend, closing any background tasks and any
 // cached connections.
 func (f *Fs) Shutdown(ctx context.Context) error {

backend/storj/fs.go

@@ -98,9 +98,10 @@ func init() {
 				},
 				}},
 			{
-				Name:     "access_grant",
-				Help:     "Access grant.",
-				Provider: "existing",
+				Name:      "access_grant",
+				Help:      "Access grant.",
+				Provider:  "existing",
+				Sensitive: true,
 			},
 			{
 				Name:     "satellite_address",
@@ -120,14 +121,16 @@ func init() {
 				},
 			},
 			{
-				Name:     "api_key",
-				Help:     "API key.",
-				Provider: newProvider,
+				Name:      "api_key",
+				Help:      "API key.",
+				Provider:  newProvider,
+				Sensitive: true,
 			},
 			{
-				Name:     "passphrase",
-				Help:     "Encryption passphrase.\n\nTo access existing objects enter passphrase used for uploading.",
-				Provider: newProvider,
+				Name:      "passphrase",
+				Help:      "Encryption passphrase.\n\nTo access existing objects enter passphrase used for uploading.",
+				Provider:  newProvider,
+				Sensitive: true,
 			},
 		},
 	})

backend/sugarsync/sugarsync.go

@@ -132,42 +132,50 @@ func init() {
 			}
 			return nil, fmt.Errorf("unknown state %q", config.State)
 		}, Options: []fs.Option{{
-			Name: "app_id",
-			Help: "Sugarsync App ID.\n\nLeave blank to use rclone's.",
+			Name:      "app_id",
+			Help:      "Sugarsync App ID.\n\nLeave blank to use rclone's.",
+			Sensitive: true,
 		}, {
-			Name: "access_key_id",
-			Help: "Sugarsync Access Key ID.\n\nLeave blank to use rclone's.",
+			Name:      "access_key_id",
+			Help:      "Sugarsync Access Key ID.\n\nLeave blank to use rclone's.",
+			Sensitive: true,
 		}, {
-			Name: "private_access_key",
-			Help: "Sugarsync Private Access Key.\n\nLeave blank to use rclone's.",
+			Name:      "private_access_key",
+			Help:      "Sugarsync Private Access Key.\n\nLeave blank to use rclone's.",
+			Sensitive: true,
 		}, {
 			Name:    "hard_delete",
 			Help:    "Permanently delete files if true\notherwise put them in the deleted files.",
 			Default: false,
 		}, {
-			Name:     "refresh_token",
-			Help:     "Sugarsync refresh token.\n\nLeave blank normally, will be auto configured by rclone.",
-			Advanced: true,
+			Name:      "refresh_token",
+			Help:      "Sugarsync refresh token.\n\nLeave blank normally, will be auto configured by rclone.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "authorization",
-			Help:     "Sugarsync authorization.\n\nLeave blank normally, will be auto configured by rclone.",
-			Advanced: true,
+			Name:      "authorization",
+			Help:      "Sugarsync authorization.\n\nLeave blank normally, will be auto configured by rclone.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     "authorization_expiry",
 			Help:     "Sugarsync authorization expiry.\n\nLeave blank normally, will be auto configured by rclone.",
 			Advanced: true,
 		}, {
-			Name:     "user",
-			Help:     "Sugarsync user.\n\nLeave blank normally, will be auto configured by rclone.",
-			Advanced: true,
+			Name:      "user",
+			Help:      "Sugarsync user.\n\nLeave blank normally, will be auto configured by rclone.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "root_id",
-			Help:     "Sugarsync root id.\n\nLeave blank normally, will be auto configured by rclone.",
-			Advanced: true,
+			Name:      "root_id",
+			Help:      "Sugarsync root id.\n\nLeave blank normally, will be auto configured by rclone.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
-			Name:     "deleted_id",
-			Help:     "Sugarsync deleted folder id.\n\nLeave blank normally, will be auto configured by rclone.",
-			Advanced: true,
+			Name:      "deleted_id",
+			Help:      "Sugarsync deleted folder id.\n\nLeave blank normally, will be auto configured by rclone.",
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,

backend/swift/swift.go

@@ -116,11 +116,13 @@ func init() {
 				},
 			},
 		}, {
-			Name: "user",
-			Help: "User name to log in (OS_USERNAME).",
+			Name:      "user",
+			Help:      "User name to log in (OS_USERNAME).",
+			Sensitive: true,
 		}, {
-			Name: "key",
-			Help: "API key or password (OS_PASSWORD).",
+			Name:      "key",
+			Help:      "API key or password (OS_PASSWORD).",
+			Sensitive: true,
 		}, {
 			Name: "auth",
 			Help: "Authentication URL for server (OS_AUTH_URL).",
@@ -147,20 +149,25 @@ func init() {
 				Help:  "Blomp Cloud Storage",
 			}},
 		}, {
-			Name: "user_id",
-			Help: "User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).",
+			Name:      "user_id",
+			Help:      "User ID to log in - optional - most swift systems use user and leave this blank (v3 auth) (OS_USER_ID).",
+			Sensitive: true,
 		}, {
-			Name: "domain",
-			Help: "User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)",
+			Name:      "domain",
+			Help:      "User domain - optional (v3 auth) (OS_USER_DOMAIN_NAME)",
+			Sensitive: true,
 		}, {
-			Name: "tenant",
-			Help: "Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME).",
+			Name:      "tenant",
+			Help:      "Tenant name - optional for v1 auth, this or tenant_id required otherwise (OS_TENANT_NAME or OS_PROJECT_NAME).",
+			Sensitive: true,
 		}, {
-			Name: "tenant_id",
-			Help: "Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID).",
+			Name:      "tenant_id",
+			Help:      "Tenant ID - optional for v1 auth, this or tenant required otherwise (OS_TENANT_ID).",
+			Sensitive: true,
 		}, {
-			Name: "tenant_domain",
-			Help: "Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME).",
+			Name:      "tenant_domain",
+			Help:      "Tenant domain - optional (v3 auth) (OS_PROJECT_DOMAIN_NAME).",
+			Sensitive: true,
 		}, {
 			Name: "region",
 			Help: "Region name - optional (OS_REGION_NAME).",
@@ -168,17 +175,21 @@ func init() {
 			Name: "storage_url",
 			Help: "Storage URL - optional (OS_STORAGE_URL).",
 		}, {
-			Name: "auth_token",
-			Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN).",
+			Name:      "auth_token",
+			Help:      "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN).",
+			Sensitive: true,
 		}, {
-			Name: "application_credential_id",
-			Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID).",
+			Name:      "application_credential_id",
+			Help:      "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID).",
+			Sensitive: true,
 		}, {
-			Name: "application_credential_name",
-			Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME).",
+			Name:      "application_credential_name",
+			Help:      "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME).",
+			Sensitive: true,
 		}, {
-			Name: "application_credential_secret",
-			Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET).",
+			Name:      "application_credential_secret",
+			Help:      "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET).",
+			Sensitive: true,
 		}, {
 			Name:    "auth_version",
 			Help:    "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION).",

backend/uptobox/uptobox.go

@@ -43,8 +43,9 @@ func init() {
 		Description: "Uptobox",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
-			Help: "Your access token.\n\nGet it from https://uptobox.com/my_account.",
-			Name: "access_token",
+			Help:      "Your access token.\n\nGet it from https://uptobox.com/my_account.",
+			Name:      "access_token",
+			Sensitive: true,
 		}, {
 			Help:     "Set to make uploaded files private",
 			Name:     "private",

backend/webdav/chunking.go

@@ -14,7 +14,6 @@ import (
 	"io"
 	"net/http"
 	"path"
-	"strings"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/lib/readers"
@@ -41,10 +40,6 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-func (f *Fs) getChunksUploadURL() string {
-	return strings.Replace(f.endpointURL, "/dav/files/", "/dav/uploads/", 1)
-}
-
 func (o *Object) getChunksUploadDir() (string, error) {
 	hasher := md5.New()
 	_, err := hasher.Write([]byte(o.filePath()))
@@ -55,12 +50,16 @@ func (o *Object) getChunksUploadDir() (string, error) {
 	return uploadDir, nil
 }
 
-func (f *Fs) verifyChunkConfig() error {
-	if f.opt.ChunkSize != 0 && !validateNextCloudChunkedURL.MatchString(f.endpointURL) {
-		return errors.New("chunked upload with nextcloud must use /dav/files/USER endpoint not /webdav")
+func (f *Fs) getChunksUploadURL() (string, error) {
+	submatch := nextCloudURLRegex.FindStringSubmatch(f.endpointURL)
+	if submatch == nil {
+		return "", errors.New("the remote url looks incorrect. Note that nextcloud chunked uploads require you to use the /dav/files/USER endpoint instead of /webdav. Please check 'rclone config show remotename' to verify that the url field ends in /dav/files/USERNAME")
 	}
 
-	return nil
+	baseURL, user := submatch[1], submatch[2]
+	chunksUploadURL := fmt.Sprintf("%s/dav/uploads/%s/", baseURL, user)
+
+	return chunksUploadURL, nil
 }
 
 func (o *Object) shouldUseChunkedUpload(src fs.ObjectInfo) bool {

backend/webdav/webdav.go

@@ -96,15 +96,17 @@ func init() {
 				Help:  "Other site/service or software",
 			}},
 		}, {
-			Name: "user",
-			Help: "User name.\n\nIn case NTLM authentication is used, the username should be in the format 'Domain\\User'.",
+			Name:      "user",
+			Help:      "User name.\n\nIn case NTLM authentication is used, the username should be in the format 'Domain\\User'.",
+			Sensitive: true,
 		}, {
 			Name:       "pass",
 			Help:       "Password.",
 			IsPassword: true,
 		}, {
-			Name: "bearer_token",
-			Help: "Bearer token instead of user/pass (e.g. a Macaroon).",
+			Name:      "bearer_token",
+			Help:      "Bearer token instead of user/pass (e.g. a Macaroon).",
+			Sensitive: true,
 		}, {
 			Name:     "bearer_token_command",
 			Help:     "Command to run to get a bearer token.",
@@ -569,7 +571,8 @@ func (f *Fs) fetchAndSetBearerToken() error {
 	return nil
 }
 
-var validateNextCloudChunkedURL = regexp.MustCompile(`^.*/dav/files/[^/]+/?$`)
+// The WebDAV url can optionally be suffixed with a path. This suffix needs to be ignored for determining the temporary upload directory of chunks.
+var nextCloudURLRegex = regexp.MustCompile(`^(.*)/dav/files/([^/]+)`)
 
 // setQuirks adjusts the Fs for the vendor passed in
 func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
@@ -592,11 +595,18 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		f.propsetMtime = true
 		f.hasOCSHA1 = true
 		f.canChunk = true
-		if err := f.verifyChunkConfig(); err != nil {
-			return err
+
+		if f.opt.ChunkSize == 0 {
+			fs.Logf(nil, "Chunked uploads are disabled because nextcloud_chunk_size is set to 0")
+		} else {
+			chunksUploadURL, err := f.getChunksUploadURL()
+			if err != nil {
+				return err
+			}
+
+			f.chunksUploadURL = chunksUploadURL
+			fs.Logf(nil, "Chunks temporary upload directory: %s", f.chunksUploadURL)
 		}
-		f.chunksUploadURL = f.getChunksUploadURL()
-		fs.Logf(nil, "Chunks temporary upload directory: %s", f.chunksUploadURL)
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth

bin/make_manual.py

@@ -66,6 +66,7 @@ docs = [
     "pcloud.md",
     "pikpak.md",
     "premiumizeme.md",
+    "protondrive.md",
     "putio.md",
     "seafile.md",
     "sftp.md",
@@ -114,7 +115,7 @@ commands_order = [
 ignore_docs = [
     "downloads.md",
     "privacy.md",
-    "donate.md",
+    "sponsor.md",
 ]
 
 def read_doc(doc):

cmd/cmd.go

@@ -35,6 +35,7 @@ import (
 	fslog "github.com/rclone/rclone/fs/log"
 	"github.com/rclone/rclone/fs/rc/rcflags"
 	"github.com/rclone/rclone/fs/rc/rcserver"
+	fssync "github.com/rclone/rclone/fs/sync"
 	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/buildinfo"
 	"github.com/rclone/rclone/lib/exitcode"
@@ -501,6 +502,8 @@ func resolveExitCode(err error) {
 		os.Exit(exitcode.UncategorizedError)
 	case errors.Is(err, accounting.ErrorMaxTransferLimitReached):
 		os.Exit(exitcode.TransferExceeded)
+	case errors.Is(err, fssync.ErrorMaxDurationReached):
+		os.Exit(exitcode.DurationExceeded)
 	case fserrors.ShouldRetry(err):
 		os.Exit(exitcode.RetryError)
 	case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err):

cmd/config/config.go

@@ -26,6 +26,7 @@ func init() {
 	configCommand.AddCommand(configTouchCommand)
 	configCommand.AddCommand(configPathsCommand)
 	configCommand.AddCommand(configShowCommand)
+	configCommand.AddCommand(configRedactedCommand)
 	configCommand.AddCommand(configDumpCommand)
 	configCommand.AddCommand(configProvidersCommand)
 	configCommand.AddCommand(configCreateCommand)
@@ -118,6 +119,35 @@ var configShowCommand = &cobra.Command{
 	},
 }
 
+var configRedactedCommand = &cobra.Command{
+	Use:   "redacted [<remote>]",
+	Short: `Print redacted (decrypted) config file, or the redacted config for a single remote.`,
+	Long: `This prints a redacted copy of the config file, either the
+whole config file or for a given remote.
+
+The config file will be redacted by replacing all passwords and other
+sensitive info with XXX.
+
+This makes the config file suitable for posting online for support.
+
+It should be double checked before posting as the redaction may not be perfect.
+
+`,
+	Annotations: map[string]string{
+		"versionIntroduced": "v1.64",
+	},
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(0, 1, command, args)
+		if len(args) == 0 {
+			config.ShowRedactedConfig()
+		} else {
+			name := strings.TrimRight(args[0], ":")
+			config.ShowRedactedRemote(name)
+		}
+		fmt.Println("### Double check the config for sensitive info before posting publicly")
+	},
+}
+
 var configDumpCommand = &cobra.Command{
 	Use:   "dump",
 	Short: `Dump the config file as JSON.`,

cmd/mountlib/check_linux.go

@@ -4,22 +4,20 @@
 package mountlib
 
 import (
-	"errors"
 	"fmt"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/artyom/mtab"
+	"github.com/moby/sys/mountinfo"
 )
 
 const (
-	mtabPath     = "/proc/mounts"
 	pollInterval = 100 * time.Millisecond
 )
 
 // CheckMountEmpty checks if folder is not already a mountpoint.
-// On Linux we use the OS-specific /proc/mount API so the check won't access the path.
+// On Linux we use the OS-specific /proc/self/mountinfo API so the check won't access the path.
 // Directories marked as "mounted" by autofs are considered not mounted.
 func CheckMountEmpty(mountpoint string) error {
 	const msg = "directory already mounted, use --allow-non-empty to mount anyway: %s"
@@ -29,43 +27,48 @@ func CheckMountEmpty(mountpoint string) error {
 		return fmt.Errorf("cannot get absolute path: %s: %w", mountpoint, err)
 	}
 
-	entries, err := mtab.Entries(mtabPath)
+	infos, err := mountinfo.GetMounts(mountinfo.SingleEntryFilter(mountpointAbs))
 	if err != nil {
-		return fmt.Errorf("cannot read %s: %w", mtabPath, err)
+		return fmt.Errorf("cannot get mounts: %w", err)
 	}
+
 	foundAutofs := false
-	for _, entry := range entries {
-		if entry.Dir == mountpointAbs {
-			if entry.Type != "autofs" {
-				return fmt.Errorf(msg, mountpointAbs)
-			}
-			foundAutofs = true
+	for _, info := range infos {
+		if info.FSType != "autofs" {
+			return fmt.Errorf(msg, mountpointAbs)
 		}
+		foundAutofs = true
 	}
 	// It isn't safe to list an autofs in the middle of mounting
 	if foundAutofs {
 		return nil
 	}
+
 	return checkMountEmpty(mountpoint)
 }
 
 // CheckMountReady checks whether mountpoint is mounted by rclone.
 // Only mounts with type "rclone" or "fuse.rclone" count.
 func CheckMountReady(mountpoint string) error {
+	const msg = "mount not ready: %s"
+
 	mountpointAbs, err := filepath.Abs(mountpoint)
 	if err != nil {
 		return fmt.Errorf("cannot get absolute path: %s: %w", mountpoint, err)
 	}
-	entries, err := mtab.Entries(mtabPath)
+
+	infos, err := mountinfo.GetMounts(mountinfo.SingleEntryFilter(mountpointAbs))
 	if err != nil {
-		return fmt.Errorf("cannot read %s: %w", mtabPath, err)
+		return fmt.Errorf("cannot get mounts: %w", err)
 	}
-	for _, entry := range entries {
-		if entry.Dir == mountpointAbs && strings.Contains(entry.Type, "rclone") {
+
+	for _, info := range infos {
+		if strings.Contains(info.FSType, "rclone") {
 			return nil
 		}
 	}
-	return errors.New("mount not ready")
+
+	return fmt.Errorf(msg, mountpointAbs)
 }
 
 // WaitMountReady waits until mountpoint is mounted by rclone.

cmd/serve/webdav/webdav.go

@@ -6,8 +6,10 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"mime"
 	"net/http"
 	"os"
+	"path"
 	"strconv"
 	"strings"
 	"time"
@@ -580,12 +582,14 @@ func (fi FileInfo) ContentType(ctx context.Context) (contentType string, err err
 		fs.Errorf(fi, "Expecting vfs.Node, got %T", fi.FileInfo)
 		return "application/octet-stream", nil
 	}
-	entry := node.DirEntry()
+	entry := node.DirEntry() // can be nil
 	switch x := entry.(type) {
 	case fs.Object:
 		return fs.MimeType(ctx, x), nil
 	case fs.Directory:
 		return "inode/directory", nil
+	case nil:
+		return mime.TypeByExtension(path.Ext(node.Name())), nil
 	}
 	fs.Errorf(fi, "Expecting fs.Object or fs.Directory, got %T", entry)
 	return "application/octet-stream", nil

cmd/test/info/info.go

@@ -72,7 +72,7 @@ a bit of go code for each one.
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1e6, command, args)
 		if !checkNormalization && !checkControl && !checkLength && !checkStreaming && !all {
-			log.Fatalf("no tests selected - select a test or use -all")
+			log.Fatalf("no tests selected - select a test or use --all")
 		}
 		if all {
 			checkNormalization = true

docs/content/_index.md

@@ -21,7 +21,7 @@ notoc: true
 
 Rclone is a command-line program to manage files on cloud storage. It
 is a feature-rich alternative to cloud vendors' web storage
-interfaces. [Over 40 cloud storage products](#providers) support
+interfaces. [Over 70 cloud storage products](#providers) support
 rclone including S3 object stores, business & consumer file storage
 services, as well as standard transfer protocols.
 
@@ -136,6 +136,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
 {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
 {{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
+{{< provider name="Leviia Object Storage" home="https://www.leviia.com/object-storage" config="/s3/#leviia" >}}
 {{< provider name="Liara Object Storage" home="https://liara.ir/landing/object-storage" config="/s3/#liara-object-storage" >}}
 {{< provider name="Mail.ru Cloud" home="https://cloud.mail.ru/" config="/mailru/" >}}
 {{< provider name="Memset Memstore" home="https://www.memset.com/cloud/storage/" config="/swift/" >}}
@@ -170,6 +171,7 @@ WebDAV or S3, that work out of the box.)
 {{< provider name="SMB / CIFS" home="https://en.wikipedia.org/wiki/Server_Message_Block" config="/smb/" >}}
 {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
 {{< provider name="Storj" home="https://storj.io/" config="/storj/" >}}
+{{< provider name="Synology" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
 {{< provider name="SugarSync" home="https://sugarsync.com/" config="/sugarsync/" >}}
 {{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
 {{< provider name="Uptobox" home="https://uptobox.com" config="/uptobox/" >}}

docs/content/authors.md

@@ -741,3 +741,15 @@ put them back in again.` >}}
   * Sawada Tsunayoshi <34431649+TsunayoshiSawada@users.noreply.github.com>
   * Dean Attali <daattali@gmail.com>
   * Fjodor42 <molgaard@gmail.com>
+  * BakaWang <wa11579@hotmail.com>
+  * Mahad <56235065+Mahad-lab@users.noreply.github.com>
+  * Vladislav Vorobev <x.miere@gmail.com>
+  * darix <darix@users.noreply.github.com>
+  * Benjamin <36415086+bbenjamin-sys@users.noreply.github.com>
+  * Chun-Hung Tseng <henrybear327@users.noreply.github.com>
+  * Ricardo D'O. Albanus <rdalbanus@users.noreply.github.com>
+  * gabriel-suela <gscsuela@gmail.com>
+  * Tiago Boeing <contato@tiagoboeing.com>
+  * Edwin Mackenzie-Owen <edwin.mowen@gmail.com>
+  * Niklas Hambüchen <mail@nh2.me>
+  * yuudi <yuudi@users.noreply.github.com>

docs/content/b2.md

@@ -231,6 +231,19 @@ $ rclone -q --b2-versions ls b2:cleanup-test
         9 one.txt
 ```
 
+#### Versions naming caveat
+
+When using `--b2-versions` flag rclone is relying on the file name
+to work out whether the objects are versions or not. Versions' names
+are created by inserting timestamp between file name and its extension.
+```
+        9 file.txt
+        8 file-v2023-07-17-161032-000.txt
+       16 file-v2023-06-15-141003-000.txt
+```
+If there are real files present with the same names as versions, then
+behaviour of `--b2-versions` can be unpredictable.
+
 ### Data usage
 
 It is useful to know how many requests are sent to the server in different scenarios.

docs/content/changelog.md

@@ -5,6 +5,27 @@ description: "Rclone Changelog"
 
 # Changelog
 
+## v1.63.1 - 2023-07-17
+
+[See commits](https://github.com/rclone/rclone/compare/v1.63.0...v1.63.1)
+
+* Bug Fixes
+    * build: Fix macos builds for versions < 12 (Anagh Kumar Baranwal)
+    * dirtree: Fix performance with large directories of directories and `--fast-list` (Nick Craig-Wood)
+    * operations
+        * Fix deadlock when using `lsd`/`ls` with `--progress` (Nick Craig-Wood)
+        * Fix `.rclonelink` files not being converted back to symlinks (Nick Craig-Wood)
+    * doc fixes (Dean Attali, Mahad, Nick Craig-Wood, Sawada Tsunayoshi, Vladislav Vorobev)
+* Local
+    * Fix partial directory read for corrupted filesystem (Nick Craig-Wood)
+* Box
+    * Fix reconnect failing with HTTP 400 Bad Request (albertony)
+* Smb
+    * Fix "Statfs failed: bucket or container name is needed" when mounting (Nick Craig-Wood)
+* WebDAV
+    * Nextcloud: fix must use /dav/files/USER endpoint not /webdav error (Paul)
+    * Nextcloud chunking: add more guidance for the user to check the config (darix)
+
 ## v1.63.0 - 2023-06-30
 
 [See commits](https://github.com/rclone/rclone/compare/v1.62.0...v1.63.0)

docs/content/chunker.md

@@ -97,7 +97,7 @@ will put files in a directory called `name` in the current directory.
 
 When rclone starts a file upload, chunker checks the file size. If it
 doesn't exceed the configured chunk size, chunker will just pass the file
-to the wrapped remote. If a file is large, chunker will transparently cut
+to the wrapped remote (however, see caveat below). If a file is large, chunker will transparently cut
 data in pieces with temporary names and stream them one by one, on the fly.
 Each data chunk will contain the specified number of bytes, except for the
 last one which may have less data. If file size is unknown in advance
@@ -129,6 +129,14 @@ proceed with current command.
 You can set the `--chunker-fail-hard` flag to have commands abort with
 error message in such cases.
 
+**Caveat**: As it is now, chunker will always create a temporary file in the 
+backend and then rename it, even if the file is below the chunk threshold.
+This will result in unnecessary API calls and can severely restrict throughput
+when handling transfers primarily composed of small files on some backends (e.g. Box).
+A workaround to this issue is to use chunker only for files above the chunk threshold
+via `--min-size` and then perform a separate call without chunker on the remaining
+files. 
+
 
 #### Chunk names
 

docs/content/crypt.md

@@ -379,7 +379,7 @@ address this problem to a certain degree.
 For cloud storage systems with case sensitive file names (e.g. Google Drive),
 `base64` can be used to reduce file name length. 
 For cloud storage systems using UTF-16 to store file names internally
-(e.g. OneDrive, Dropbox), `base32768` can be used to drastically reduce
+(e.g. OneDrive, Dropbox, Box), `base32768` can be used to drastically reduce
 file name length. 
 
 An alternative, future rclone file name encryption mode may tolerate
@@ -600,7 +600,7 @@ Properties:
         - Encode using base64. Suitable for case sensitive remote.
     - "base32768"
         - Encode using base32768. Suitable if your remote counts UTF-16 or
-        - Unicode codepoint instead of UTF-8 byte length. (Eg. Onedrive, Dropbox)
+        - Unicode codepoint instead of UTF-8 byte length. (Eg. Onedrive, Dropbox, Box)
 
 #### --crypt-suffix
 

docs/content/docs.md

@@ -1454,14 +1454,14 @@ what will happen.
 
 ### --max-duration=TIME ###
 
-Rclone will stop scheduling new transfers when it has run for the
+Rclone will stop transferring when it has run for the
 duration specified.
-
 Defaults to off.
 
-When the limit is reached any existing transfers will complete.
+When the limit is reached all transfers will stop immediately.
+Use `--cutoff-mode` to modify this behaviour.
 
-Rclone won't exit with an error if the transfer limit is reached.
+Rclone will exit with exit code 10 if the duration limit is reached.
 
 ### --max-transfer=SIZE ###
 
@@ -1469,9 +1469,24 @@ Rclone will stop transferring when it has reached the size specified.
 Defaults to off.
 
 When the limit is reached all transfers will stop immediately.
+Use `--cutoff-mode` to modify this behaviour.
 
 Rclone will exit with exit code 8 if the transfer limit is reached.
 
+### --cutoff-mode=hard|soft|cautious ###
+
+This modifies the behavior of `--max-transfer` and `--max-duration`
+Defaults to `--cutoff-mode=hard`.
+
+Specifying `--cutoff-mode=hard` will stop transferring immediately
+when Rclone reaches the limit.
+
+Specifying `--cutoff-mode=soft` will stop starting new transfers
+when Rclone reaches the limit.
+
+Specifying `--cutoff-mode=cautious` will try to prevent Rclone
+from reaching the limit. Only applicable for `--max-transfer`
+
 ## -M, --metadata
 
 Setting this flag enables rclone to copy the metadata from the source
@@ -1484,20 +1499,6 @@ Add metadata `key` = `value` when uploading. This can be repeated as
 many times as required. See the [#metadata](metadata section) for more
 info.
 
-### --cutoff-mode=hard|soft|cautious ###
-
-This modifies the behavior of `--max-transfer`
-Defaults to `--cutoff-mode=hard`.
-
-Specifying `--cutoff-mode=hard` will stop transferring immediately
-when Rclone reaches the limit.
-
-Specifying `--cutoff-mode=soft` will stop starting new transfers
-when Rclone reaches the limit.
-
-Specifying `--cutoff-mode=cautious` will try to prevent Rclone
-from reaching the limit.
-
 ### --modify-window=TIME ###
 
 When checking whether a file has been modified, this is the maximum
@@ -2561,6 +2562,7 @@ it will log a high priority message if the retry was successful.
   * `7` - Fatal error (one that more retries won't fix, like account suspended) (Fatal errors)
   * `8` - Transfer exceeded - limit set by --max-transfer reached
   * `9` - Operation successful, but no files transferred
+  * `10` - Duration exceeded - limit set by --max-duration reached
 
 Environment Variables
 ---------------------

docs/content/donate.md

@@ -1,68 +0,0 @@
----
-title: "Donations"
-description: "Donations to the rclone project."
-type: page
----
-
-# {{< icon "fa fa-heart heart" >}} Donations to the rclone project
-
-Rclone is a free open-source project with thousands of contributions
-from volunteers all round the world and I would like to thank all of
-you for donating your time to the project.
-
-However, maintaining rclone is a lot of work - easily the equivalent
-of a **full time job** - for me. Nothing stands still in the world of
-cloud storage.  Rclone needs constant attention adapting to changes by
-cloud providers, adding new providers, adding new features, keeping
-the integration tests working, fixing bugs and many more things!
-
-I love doing the work and I'd like to spend more time doing it - your
-support helps make that possible.
-
-Thank you :-)
-
-{{< nick >}}
-
-PS I'm available for rclone and object storage related consultancy -
-[email me](mailto:nick@craig-wood.com) for more info.
-
-{{< monthly_donations >}}
-
-## Personal users
-
-If you are a personal user and you would like to support the project
-with sponsorship as a way of saying thank you that would be most
-appreciated. {{< icon "fa fa-heart heart" >}}
-
-## Business users
-
-If your business distributes rclone as part of its products (which the
-generous MIT licence allows) or uses it internally then it would make
-business sense to sponsor the rclone project to ensure that the
-project you rely on stays healthy and well maintained.
-
-If you run one of the cloud storage providers that rclone supports and
-rclone is driving revenue your way then you know it makes sense to
-sponsor the project. {{< icon "far fa-smile" >}}
-
-Note that if you choose the "GitHub Sponsors" option they will provide
-proper tax invoices appropriate for your country.
-
-## Monthly donations
-
-Monthly donations help keep rclone development sustainable in the long
-run so this is the preferred option. A small amount every month is
-much better than a one off donation as it allows planning for the
-future.
-
-{{< monthly_donations >}}
-
-## One off donations
-
-If you don't want to contribute monthly then of course we'd love a one
-off donation.
-
-{{< one_off_donations >}}
-
-If you require a receipt or wish to contribute in a different way then
-please [drop me an email](mailto:nick@craig-wood.com).

docs/content/drive.md

@@ -1522,7 +1522,7 @@ be the same account as the Google Drive you want to access)
 "Google Drive API".
 
 4. Click "Credentials" in the left-side panel (not "Create
-credentials", which opens the wizard), then "Create credentials"
+credentials", which opens the wizard).
 
 5. If you already configured an "Oauth Consent Screen", then skip
 to the next step; if not, click on "CONFIGURE CONSENT SCREEN" button 

docs/content/dropbox.md

@@ -494,7 +494,7 @@ to be the same account as the Dropbox you want to access)
 
 2. Choose an API => Usually this should be `Dropbox API`
 
-3. Choose the type of access you want to use => `Full Dropbox` or `App Folder`
+3. Choose the type of access you want to use => `Full Dropbox` or `App Folder`. If you want to use Team Folders, `Full Dropbox` is required ([see here](https://www.dropboxforum.com/t5/Dropbox-API-Support-Feedback/How-to-create-team-folder-inside-my-app-s-folder/m-p/601005/highlight/true#M27911)).
 
 4. Name your App. The app name is global, so you can't use `rclone` for example
 

docs/content/flags.md

@@ -86,7 +86,7 @@ These flags are available for every command.
       --max-delete int                              When synchronizing, limit the number of deletes (default -1)
       --max-delete-size SizeSuffix                  When synchronizing, limit the total size of deletes (default off)
       --max-depth int                               If set limits the recursion depth to this (default -1)
-      --max-duration Duration                       Maximum duration rclone will transfer data for (default 0s)
+      --max-duration Duration                       Maximum duration rclone will transfer data for (default off)
       --max-size SizeSuffix                         Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P (default off)
       --max-stats-groups int                        Maximum number of stats groups to keep in memory, on max oldest is discarded (default 1000)
       --max-transfer SizeSuffix                     Maximum size of data to transfer (default off)

docs/content/mailru.md

@@ -8,8 +8,6 @@ versionIntroduced: "v1.50"
 
 [Mail.ru Cloud](https://cloud.mail.ru/) is a cloud storage provided by a Russian internet company [Mail.Ru Group](https://mail.ru). The official desktop client is [Disk-O:](https://disk-o.cloud/en), available on Windows and Mac OS.
 
-Currently it is recommended to disable 2FA on Mail.ru accounts intended for rclone until it gets eventually implemented.
-
 ## Features highlights
 
 - Paths may be as deep as required, e.g. `remote:directory/subdirectory`

docs/content/mega.md

@@ -284,6 +284,10 @@ Properties:
 
 {{< rem autogenerated options stop >}}
 
+### Process `killed`
+
+On accounts with large files or something else, memory usage can significantly increase when executing list/sync instructions. When running on cloud providers (like AWS with EC2), check if the instance type has sufficient memory/CPU to execute the commands. Use the resource monitoring tools to inspect after sending the commands. Look [at this issue](https://forum.rclone.org/t/rclone-with-mega-appears-to-work-only-in-some-accounts/40233/4).
+
 ## Limitations
 
 This backend uses the [go-mega go library](https://github.com/t3rm1n4l/go-mega) which is an opensource

docs/content/protondrive.md

@@ -0,0 +1,138 @@
+---
+title: "Proton Drive"
+description: "Rclone docs for Proton Drive"
+versionIntroduced: "v1.64.0"
+---
+
+# {{< icon "fa fa-atom-simple" >}} Proton Drive
+
+[Proton Drive](https://proton.me/drive) is an end-to-end encrypted Swiss vault
+ for your files that protects your data.
+
+This is a rclone backend for Proton Drive which supports the file transfer
+features of Proton Drive using the same client-side encryption.
+
+Due to the fact that Proton Drive doesn't publish its API documentation, this 
+backend is implemented with the best effort by reading the open-sourced client 
+source code and observing the Proton Drive traffic on the browser.
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, e.g. `remote:directory/subdirectory`.
+
+## Configurations
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+[snip]
+XX / Proton Drive
+   \ "Proton Drive"
+[snip]
+Storage> protondrive
+User name
+user> you@protonmail.com
+Password.
+y) Yes type in my own password
+g) Generate random password
+n) No leave this optional password blank
+y/g/n> y
+Enter the password:
+password:
+Confirm the password:
+password:
+Option 2fa.
+2FA code (if the account requires one)
+Enter a value. Press Enter to leave empty.
+2fa> 123456
+Remote config
+--------------------
+[remote]
+type = protondrive
+user = you@protonmail.com
+pass = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+**NOTE:** The Proton Drive encryption keys need to have been already generated 
+after a regular login via the browser, otherwise attempting to use the 
+credentials in `rclone` will fail.
+
+Once configured you can then use `rclone` like this,
+
+List directories in top level of your Proton Drive
+
+    rclone lsd remote:
+
+List all the files in your Proton Drive
+
+    rclone ls remote:
+
+To copy a local directory to an Proton Drive directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time
+
+Proton Drive Bridge does not support modification times yet.
+
+### Restricted filename characters
+
+Invalid UTF-8 bytes will be [replaced](/overview/#invalid-utf8), also left and 
+right spaces will be removed ([code reference](https://github.com/ProtonMail/WebClients/blob/b4eba99d241af4fdae06ff7138bd651a40ef5d3c/applications/drive/src/app/store/_links/validation.ts#L51))
+
+### Duplicated files
+
+Proton Drive can not have two files with exactly the same name and path. If the 
+conflict occurs, depending on the advanced config, the file might or might not 
+be overwritten.
+
+### Caching
+
+The cache is currently built for the case when the rclone is the only instance 
+performing operations to the mount point. The event system, which is the proton
+API system that provides visibility of what has changed on the drive, is yet 
+to be implemented, so updates from other clients won’t be reflected in the 
+cache. Thus, if there are concurrent clients accessing the same mount point, 
+then we might have a problem with caching the stale data.
+
+## Limitations
+
+This backend uses the 
+[Proton-API-Bridge](https://github.com/henrybear327/Proton-API-Bridge), which 
+is based on [go-proton-api](https://github.com/henrybear327/go-proton-api), a 
+fork of the [official repo](https://github.com/ProtonMail/go-proton-api).
+
+There is no official API documentation available from Proton Drive. But, thanks 
+to Proton open sourcing [proton-go-api](https://github.com/ProtonMail/go-proton-api) 
+and the web, iOS, and Android client codebases, we don't need to completely 
+reverse engineer the APIs by observing the web client traffic! 
+
+[proton-go-api](https://github.com/ProtonMail/go-proton-api) provides the basic 
+building blocks of API calls and error handling, such as 429 exponential 
+back-off, but it is pretty much just a barebone interface to the Proton API. 
+For example, the encryption and decryption of the Proton Drive file are not 
+provided in this library. 
+
+The Proton-API-Bridge, attempts to bridge the gap, so rclone can be built on 
+top of this quickly. This codebase handles the intricate tasks before and after 
+calling Proton APIs, particularly the complex encryption scheme, allowing 
+developers to implement features for other software on top of this codebase. 
+There are likely quite a few errors in this library, as there isn't official 
+documentation available. 

docs/content/s3.md

@@ -22,6 +22,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
 {{< provider name="IDrive e2" home="https://www.idrive.com/e2/?refer=rclone" config="/s3/#idrive-e2" >}}
 {{< provider name="IONOS Cloud" home="https://cloud.ionos.com/storage/object-storage" config="/s3/#ionos" >}}
+ {{< provider name="Leviia Object Storage" home="https://www.leviia.com/object-storage/" config="/s3/#leviia" >}}
 {{< provider name="Liara Object Storage" home="https://liara.ir/landing/object-storage" config="/s3/#liara-cloud" >}}
 {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
 {{< provider name="Petabox" home="https://petabox.io/" config="/s3/#petabox" >}}
@@ -32,6 +33,7 @@ The S3 backend can be used with a number of different providers:
 {{< provider name="SeaweedFS" home="https://github.com/chrislusf/seaweedfs/" config="/s3/#seaweedfs" >}}
 {{< provider name="StackPath" home="https://www.stackpath.com/products/object-storage/" config="/s3/#stackpath" >}}
 {{< provider name="Storj" home="https://storj.io/" config="/s3/#storj" >}}
+{{< provider name="Synology C2 Object Storage" home="https://c2.synology.com/en-global/object-storage/overview" config="/s3/#synology-c2" >}}
 {{< provider name="Tencent Cloud Object Storage (COS)" home="https://intl.cloud.tencent.com/product/cos" config="/s3/#tencent-cos" >}}
 {{< provider name="Wasabi" home="https://wasabi.com/" config="/s3/#wasabi" end="true" >}}
 {{< /provider_list >}}
@@ -462,6 +464,19 @@ $ rclone -q --s3-versions ls s3:cleanup-test
         9 one.txt
 ```
 
+#### Versions naming caveat
+
+When using `--s3-versions` flag rclone is relying on the file name
+to work out whether the objects are versions or not. Versions' names
+are created by inserting timestamp between file name and its extension.
+```
+        9 file.txt
+        8 file-v2023-07-17-161032-000.txt
+       16 file-v2023-06-15-141003-000.txt
+```
+If there are real files present with the same names as versions, then
+behaviour of `--s3-versions` can be unpredictable.
+
 ### Cleanup
 
 If you run `rclone cleanup s3:bucket` then it will remove all pending
@@ -649,7 +664,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
 {{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
 ### Standard options
 
-Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
+Here are the Standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-provider
 
@@ -708,6 +723,8 @@ Properties:
         - StackPath Object Storage
     - "Storj"
         - Storj (S3 Compatible Gateway)
+    - "Synology"
+        - Synology C2 Object Storage
     - "TencentCOS"
         - Tencent Cloud Object Storage (COS)
     - "Wasabi"
@@ -1069,7 +1086,7 @@ Properties:
 
 - Config:      region
 - Env Var:     RCLONE_S3_REGION
-- Provider:    !AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Qiniu,RackCorp,Scaleway,Storj,TencentCOS,HuaweiOBS,IDrive
+- Provider:    !AWS,Alibaba,ArvanCloud,ChinaMobile,Cloudflare,IONOS,Petabox,Liara,Qiniu,RackCorp,Scaleway,Storj,Synology,TencentCOS,HuaweiOBS,IDrive
 - Type:        string
 - Required:    false
 - Examples:
@@ -1565,6 +1582,29 @@ Properties:
 
 #### --s3-endpoint
 
+Endpoint for Synology C2 Object Storage API.
+
+Properties:
+
+- Config:      endpoint
+- Env Var:     RCLONE_S3_ENDPOINT
+- Provider:    Synology
+- Type:        string
+- Required:    false
+- Examples:
+    - "eu-001.s3.synologyc2.net"
+        - Europe Region 1
+    - "eu-002.s3.synologyc2.net"
+        - Europe Region 2
+    - "us-001.s3.synologyc2.net"
+        - US Region 1
+    - "us-002.s3.synologyc2.net"
+        - US Region 2
+    - "tw-001.s3.synologyc2.net"
+        - Asia Region (Taiwan)
+
+#### --s3-endpoint
+
 Endpoint for Tencent COS API.
 
 Properties:
@@ -1702,7 +1742,7 @@ Properties:
 
 - Config:      endpoint
 - Env Var:     RCLONE_S3_ENDPOINT
-- Provider:    !AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Scaleway,StackPath,Storj,RackCorp,Qiniu,Petabox
+- Provider:    !AWS,ArvanCloud,IBMCOS,IDrive,IONOS,TencentCOS,HuaweiOBS,Alibaba,ChinaMobile,GCS,Liara,Scaleway,StackPath,Storj,Synology,RackCorp,Qiniu,Petabox
 - Type:        string
 - Required:    false
 - Examples:
@@ -2368,7 +2408,7 @@ Properties:
 
 ### Advanced options
 
-Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi).
+Here are the Advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, ArvanCloud, Ceph, China Mobile, Cloudflare, GCS, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi).
 
 #### --s3-bucket-acl
 
@@ -3421,7 +3461,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 ...
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
    \ (s3)
 ...
 Storage> s3
@@ -3605,7 +3645,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
- 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
    \ (s3)
 [snip]
 Storage> 5
@@ -3900,7 +3940,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -4006,7 +4046,7 @@ Option Storage.
 Type of storage to configure.
 Choose a number from below, or type in your own value.
 [snip]
-XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
+XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -4252,7 +4292,7 @@ Choose a number from below, or type in your own value
    \ (alias)
  4 / Amazon Drive
    \ (amazon cloud drive)
- 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS, Qiniu and Wasabi
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
    \ (s3)
 [snip]
 Storage> s3
@@ -5132,7 +5172,129 @@ e) Edit this remote
 d) Delete this remote
 y/e/d> y
 ```
+### Leviia Cloud Object Storage {#leviia}
 
+[Leviia Object Storage](https://www.leviia.com/object-storage/), backup and secure your data in a 100% French cloud, independent of GAFAM..
+
+To configure access to Leviia, follow the steps below:
+
+1. Run `rclone config` and select `n` for a new remote.
+
+```
+rclone config
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+n/s/q> n
+```
+
+2. Give the name of the configuration. For example, name it 'leviia'.
+
+```
+name> leviia
+```
+
+3. Select `s3` storage.
+
+```
+Choose a number from below, or type in your own value
+ 1 / 1Fichier
+   \ (fichier)
+ 2 / Akamai NetStorage
+   \ (netstorage)
+ 3 / Alias for an existing remote
+   \ (alias)
+ 4 / Amazon Drive
+   \ (amazon cloud drive)
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, Liara, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
+   \ (s3)
+[snip]
+Storage> s3
+```
+
+4. Select `Leviia` provider.
+```
+Choose a number from below, or type in your own value
+1 / Amazon Web Services (AWS) S3
+   \ "AWS"
+[snip]
+15 / Leviia Object Storage
+   \ (Leviia)
+[snip]
+provider> Leviia
+```
+
+5. Enter your SecretId and SecretKey of Leviia.
+
+```
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+env_auth> 1
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+access_key_id> ZnIx.xxxxxxxxxxxxxxx
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+secret_access_key> xxxxxxxxxxx
+```
+
+6. Select endpoint for Leviia.
+
+```
+   / The default endpoint
+ 1 | Leviia.
+   \ (s3.leviia.com)
+[snip]
+endpoint> 1
+```
+7. Choose acl.
+
+```
+Note that this ACL is applied when server-side copying objects as S3
+doesn't copy the ACL from the source but rather writes a fresh one.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+   / Owner gets FULL_CONTROL.
+ 1 | No one else has access rights (default).
+   \ (private)
+   / Owner gets FULL_CONTROL.
+ 2 | The AllUsers group gets READ access.
+   \ (public-read)
+[snip]
+acl> 1
+Edit advanced config? (y/n)
+y) Yes
+n) No (default)
+y/n> n
+Remote config
+--------------------
+[leviia]
+- type: s3
+- provider: Leviia
+- access_key_id: ZnIx.xxxxxxx
+- secret_access_key: xxxxxxxx
+- endpoint: s3.leviia.com
+- acl: private
+--------------------
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+Current remotes:
+
+Name                 Type
+====                 ====
+leviia                s3
+```
 ### Liara {#liara-cloud}
 
 Here is an example of making a [Liara Object Storage](https://liara.ir/landing/object-storage)
@@ -5749,3 +5911,137 @@ use policy `mfs` (most free space) as a member of an rclone union
 remote.
 
 See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+
+
+
+### Synology C2 Object Storage {#synology-c2}
+
+[Synology C2 Object Storage](https://c2.synology.com/en-global/object-storage/overview) provides a secure, S3-compatible, and cost-effective cloud storage solution without API request, download fees, and deletion penalty.
+
+The S3 compatible gateway is configured using `rclone config` with a
+type of `s3` and with a provider name of `Synology`. Here is an example
+run of the configurator.
+
+First run:
+
+```
+rclone config
+```
+
+This will guide you through an interactive setup process.
+
+```
+No remotes found, make a new one?
+n) New remote
+s) Set configuration password
+q) Quit config
+
+n/s/q> n
+
+Enter name for new remote.1
+name> syno
+
+Type of storage to configure.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+
+ 5 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, GCS, ArvanCloud, DigitalOcean, Dreamhost, Huawei OBS, IBM COS, IDrive e2, IONOS Cloud, Liara, Lyve Cloud, Minio, Netease, Petabox, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Synology, Tencent COS, Qiniu and Wasabi
+   \ "s3"
+
+Storage> s3
+
+Choose your S3 provider.
+Enter a string value. Press Enter for the default ("").
+Choose a number from below, or type in your own value
+ 24 / Synology C2 Object Storage
+   \ (Synology)
+
+provider> Synology
+
+Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
+Only applies if access_key_id and secret_access_key is blank.
+Enter a boolean value (true or false). Press Enter for the default ("false").
+Choose a number from below, or type in your own value
+ 1 / Enter AWS credentials in the next step
+   \ "false"
+ 2 / Get AWS credentials from the environment (env vars or IAM)
+   \ "true"
+
+env_auth> 1
+
+AWS Access Key ID.
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+
+access_key_id> accesskeyid
+
+AWS Secret Access Key (password)
+Leave blank for anonymous access or runtime credentials.
+Enter a string value. Press Enter for the default ("").
+
+secret_access_key> secretaccesskey
+
+Region where your data stored.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / Europe Region 1
+   \ (eu-001)
+ 2 / Europe Region 2
+   \ (eu-002)
+ 3 / US Region 1
+   \ (us-001)
+ 4 / US Region 2
+   \ (us-002)
+ 5 / Asia (Taiwan)
+   \ (tw-001)
+
+region > 1
+
+Option endpoint.
+Endpoint for Synology C2 Object Storage API.
+Choose a number from below, or type in your own value.
+Press Enter to leave empty.
+ 1 / EU Endpoint 1
+   \ (eu-001.s3.synologyc2.net)
+ 2 / US Endpoint 1
+   \ (us-001.s3.synologyc2.net)
+ 3 / TW Endpoint 1
+   \ (tw-001.s3.synologyc2.net)
+
+endpoint> 1
+
+Option location_constraint.
+Location constraint - must be set to match the Region.
+Leave blank if not sure. Used when creating buckets only.
+Enter a value. Press Enter to leave empty.
+location_constraint>
+
+Edit advanced config? (y/n)
+y) Yes
+n) No
+y/n> y
+
+Option no_check_bucket.
+If set, don't attempt to check the bucket exists or create it.
+This can be useful when trying to minimise the number of transactions
+rclone does if you know the bucket exists already.
+It can also be needed if the user you are using does not have bucket
+creation permissions. Before v1.52.0 this would have passed silently
+due to a bug.
+Enter a boolean value (true or false). Press Enter for the default (true).
+
+no_check_bucket> true
+
+Configuration complete.
+Options:
+- type: s3
+- provider: Synology
+- region: eu-001
+- endpoint: eu-001.s3.synologyc2.net
+- no_check_bucket: true
+Keep this "syno" remote?
+y) Yes this is OK (default)
+e) Edit this remote
+d) Delete this remote
+
+y/e/d> y

fs/config/ui.go

@@ -302,19 +302,41 @@ func mustFindByName(name string) *fs.RegInfo {
 	return fs.MustFind(fsType)
 }
 
+// findByName finds the RegInfo for the remote name passed in or
+// returns an error
+func findByName(name string) (*fs.RegInfo, error) {
+	fsType := FileGet(name, "type")
+	if fsType == "" {
+		return nil, fmt.Errorf("couldn't find type of fs for %q", name)
+	}
+	return fs.Find(fsType)
+}
+
 // printRemoteOptions prints the options of the remote
-func printRemoteOptions(name string, prefix string, sep string) {
-	fs := mustFindByName(name)
+func printRemoteOptions(name string, prefix string, sep string, redacted bool) {
+	fsInfo, err := findByName(name)
+	if err != nil {
+		fmt.Printf("# %v\n", err)
+		fsInfo = nil
+	}
 	for _, key := range LoadedData().GetKeyList(name) {
 		isPassword := false
-		for _, option := range fs.Options {
-			if option.Name == key && option.IsPassword {
-				isPassword = true
-				break
+		isSensitive := false
+		if fsInfo != nil {
+			for _, option := range fsInfo.Options {
+				if option.Name == key {
+					if option.IsPassword {
+						isPassword = true
+					} else if option.Sensitive {
+						isSensitive = true
+					}
+				}
 			}
 		}
 		value := FileGet(name, key)
-		if isPassword && value != "" {
+		if redacted && (isSensitive || isPassword) && value != "" {
+			fmt.Printf("%s%s%sXXX\n", prefix, key, sep)
+		} else if isPassword && value != "" {
 			fmt.Printf("%s%s%s*** ENCRYPTED ***\n", prefix, key, sep)
 		} else {
 			fmt.Printf("%s%s%s%s\n", prefix, key, sep, value)
@@ -324,13 +346,19 @@ func printRemoteOptions(name string, prefix string, sep string) {
 
 // listRemoteOptions lists the options of the remote
 func listRemoteOptions(name string) {
-	printRemoteOptions(name, "- ", ": ")
+	printRemoteOptions(name, "- ", ": ", false)
 }
 
 // ShowRemote shows the contents of the remote in config file format
 func ShowRemote(name string) {
 	fmt.Printf("[%s]\n", name)
-	printRemoteOptions(name, "", " = ")
+	printRemoteOptions(name, "", " = ", false)
+}
+
+// ShowRedactedRemote shows the contents of the remote in config file format
+func ShowRedactedRemote(name string) {
+	fmt.Printf("[%s]\n", name)
+	printRemoteOptions(name, "", " = ", true)
 }
 
 // OkRemote prints the contents of the remote and ask if it is OK
@@ -634,6 +662,22 @@ func ShowConfig() {
 	fmt.Printf("%s", str)
 }
 
+// ShowRedactedConfig prints the redacted (unencrypted) config options
+func ShowRedactedConfig() {
+	remotes := LoadedData().GetSectionList()
+	if len(remotes) == 0 {
+		fmt.Println("; empty config")
+		return
+	}
+	sort.Strings(remotes)
+	for i, remote := range remotes {
+		if i != 0 {
+			fmt.Println()
+		}
+		ShowRedactedRemote(remote)
+	}
+}
+
 // EditConfig edits the config file interactively
 func EditConfig(ctx context.Context) (err error) {
 	for {

fs/fs_test.go

@@ -243,6 +243,7 @@ func TestOptionMarshalJSON(t *testing.T) {
 "NoPrefix": false,
 "Advanced": true,
 "Exclusive": false,
+"Sensitive": false,
 "DefaultStr": "false",
 "ValueStr": "true",
 "Type": "bool"

fs/fshttp/fshttpdump/fshttpdump.go

@@ -0,0 +1,97 @@
+package fshttpdump
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httputil"
+	"sync"
+
+	"github.com/rclone/rclone/fs"
+)
+
+const (
+	separatorReq  = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
+	separatorResp = "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
+)
+
+var (
+	logMutex sync.Mutex
+)
+
+// cleanAuth gets rid of one authBuf header within the first 4k
+func cleanAuth(buf, authBuf []byte) []byte {
+	// Find how much buffer to check
+	n := 4096
+	if len(buf) < n {
+		n = len(buf)
+	}
+	// See if there is an Authorization: header
+	i := bytes.Index(buf[:n], authBuf)
+	if i < 0 {
+		return buf
+	}
+	i += len(authBuf)
+	// Overwrite the next 4 chars with 'X'
+	for j := 0; i < len(buf) && j < 4; j++ {
+		if buf[i] == '\n' {
+			break
+		}
+		buf[i] = 'X'
+		i++
+	}
+	// Snip out to the next '\n'
+	j := bytes.IndexByte(buf[i:], '\n')
+	if j < 0 {
+		return buf[:i]
+	}
+	n = copy(buf[i:], buf[i+j:])
+	return buf[:i+n]
+}
+
+var authBufs = [][]byte{
+	[]byte("Authorization: "),
+	[]byte("X-Auth-Token: "),
+}
+
+// cleanAuths gets rid of all the possible Auth headers
+func cleanAuths(buf []byte) []byte {
+	for _, authBuf := range authBufs {
+		buf = cleanAuth(buf, authBuf)
+	}
+	return buf
+}
+
+func DumpRequest(req *http.Request, dump fs.DumpFlags, client bool) {
+	if dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
+		dumper := httputil.DumpRequestOut
+		if !client {
+			dumper = httputil.DumpRequest
+		}
+		buf, _ := dumper(req, dump&(fs.DumpBodies|fs.DumpRequests) != 0)
+		if dump&fs.DumpAuth == 0 {
+			buf = cleanAuths(buf)
+		}
+		logMutex.Lock()
+		fs.Debugf(nil, "%s", separatorReq)
+		fs.Debugf(nil, "%s (req %p)", "HTTP REQUEST", req)
+		fs.Debugf(nil, "%s", string(buf))
+		fs.Debugf(nil, "%s", separatorReq)
+		logMutex.Unlock()
+	}
+}
+
+func DumpResponse(resp *http.Response, req *http.Request, err error, dump fs.DumpFlags) {
+	if dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
+		logMutex.Lock()
+		fs.Debugf(nil, "%s", separatorResp)
+		fs.Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
+		if err != nil {
+			fs.Debugf(nil, "Error: %v", err)
+		} else {
+			buf, _ := httputil.DumpResponse(resp, dump&(fs.DumpBodies|fs.DumpResponses) != 0)
+			fs.Debugf(nil, "%s", string(buf))
+		}
+		fs.Debugf(nil, "%s", separatorResp)
+		logMutex.Unlock()
+	}
+}

fs/fshttp/fshttpdump/fshttpdump_test.go

@@ -1,4 +1,4 @@
-package fshttp
+package fshttpdump
 
 import (
 	"testing"

fs/fshttp/http.go

@@ -2,7 +2,6 @@
 package fshttp
 
 import (
-	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
@@ -10,27 +9,21 @@ import (
 	"net"
 	"net/http"
 	"net/http/cookiejar"
-	"net/http/httputil"
 	"os"
 	"sync"
 	"time"
 
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/fshttp/fshttpdump"
 	"github.com/rclone/rclone/lib/structs"
 	"golang.org/x/net/publicsuffix"
 )
 
-const (
-	separatorReq  = ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
-	separatorResp = "<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
-)
-
 var (
 	transport    http.RoundTripper
 	noTransport  = new(sync.Once)
 	cookieJar, _ = cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
-	logMutex     sync.Mutex
 )
 
 // ResetTransport resets the existing transport, allowing it to take new settings.
@@ -204,49 +197,6 @@ func checkServerTime(req *http.Request, resp *http.Response) {
 	checkedHostMu.Unlock()
 }
 
-// cleanAuth gets rid of one authBuf header within the first 4k
-func cleanAuth(buf, authBuf []byte) []byte {
-	// Find how much buffer to check
-	n := 4096
-	if len(buf) < n {
-		n = len(buf)
-	}
-	// See if there is an Authorization: header
-	i := bytes.Index(buf[:n], authBuf)
-	if i < 0 {
-		return buf
-	}
-	i += len(authBuf)
-	// Overwrite the next 4 chars with 'X'
-	for j := 0; i < len(buf) && j < 4; j++ {
-		if buf[i] == '\n' {
-			break
-		}
-		buf[i] = 'X'
-		i++
-	}
-	// Snip out to the next '\n'
-	j := bytes.IndexByte(buf[i:], '\n')
-	if j < 0 {
-		return buf[:i]
-	}
-	n = copy(buf[i:], buf[i+j:])
-	return buf[:i+n]
-}
-
-var authBufs = [][]byte{
-	[]byte("Authorization: "),
-	[]byte("X-Auth-Token: "),
-}
-
-// cleanAuths gets rid of all the possible Auth headers
-func cleanAuths(buf []byte) []byte {
-	for _, authBuf := range authBufs {
-		buf = cleanAuth(buf, authBuf)
-	}
-	return buf
-}
-
 // RoundTrip implements the RoundTripper interface.
 func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
 	// Limit transactions per second if required
@@ -262,34 +212,11 @@ func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error
 		t.filterRequest(req)
 	}
 	// Logf request
-	if t.dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
-		buf, _ := httputil.DumpRequestOut(req, t.dump&(fs.DumpBodies|fs.DumpRequests) != 0)
-		if t.dump&fs.DumpAuth == 0 {
-			buf = cleanAuths(buf)
-		}
-		logMutex.Lock()
-		fs.Debugf(nil, "%s", separatorReq)
-		fs.Debugf(nil, "%s (req %p)", "HTTP REQUEST", req)
-		fs.Debugf(nil, "%s", string(buf))
-		fs.Debugf(nil, "%s", separatorReq)
-		logMutex.Unlock()
-	}
+	fshttpdump.DumpRequest(req, t.dump, true)
 	// Do round trip
 	resp, err = t.Transport.RoundTrip(req)
 	// Logf response
-	if t.dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpAuth|fs.DumpRequests|fs.DumpResponses) != 0 {
-		logMutex.Lock()
-		fs.Debugf(nil, "%s", separatorResp)
-		fs.Debugf(nil, "%s (req %p)", "HTTP RESPONSE", req)
-		if err != nil {
-			fs.Debugf(nil, "Error: %v", err)
-		} else {
-			buf, _ := httputil.DumpResponse(resp, t.dump&(fs.DumpBodies|fs.DumpResponses) != 0)
-			fs.Debugf(nil, "%s", string(buf))
-		}
-		fs.Debugf(nil, "%s", separatorResp)
-		logMutex.Unlock()
-	}
+	fshttpdump.DumpResponse(resp, req, err, t.dump)
 	// Update metrics
 	t.metrics.onResponse(req, resp)
 

fs/operations/operations.go

@@ -358,7 +358,7 @@ func Copy(ctx context.Context, f fs.Fs, dst fs.Object, remote string, src fs.Obj
 		inplace       = true
 		remotePartial = remote
 	)
-	if !ci.Inplace && f.Features().Move != nil && f.Features().PartialUploads {
+	if !ci.Inplace && f.Features().Move != nil && f.Features().PartialUploads && !strings.HasSuffix(remote, ".rclonelink") {
 		// Avoid making the leaf name longer if it's already lengthy to avoid
 		// trouble with file name length limits.
 		suffix := "." + random.String(8) + ".partial"
@@ -826,17 +826,19 @@ func Same(fdst, fsrc fs.Info) bool {
 	return SameConfig(fdst, fsrc) && strings.Trim(fdst.Root(), "/") == strings.Trim(fsrc.Root(), "/")
 }
 
-// fixRoot returns the Root with a trailing / if not empty. It is
-// aware of case insensitive filesystems.
-func fixRoot(f fs.Info) string {
-	s := strings.Trim(filepath.ToSlash(f.Root()), "/")
+// fixRoot returns the Root with a trailing / if not empty.
+//
+// It returns a case folded version for case insensitive file systems
+func fixRoot(f fs.Info) (s string, folded string) {
+	s = strings.Trim(filepath.ToSlash(f.Root()), "/")
 	if s != "" {
 		s += "/"
 	}
+	folded = s
 	if f.Features().CaseInsensitive {
-		s = strings.ToLower(s)
+		folded = strings.ToLower(s)
 	}
-	return s
+	return s, folded
 }
 
 // OverlappingFilterCheck returns true if fdst and fsrc point to the same
@@ -845,37 +847,28 @@ func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool {
 	if !SameConfig(fdst, fsrc) {
 		return false
 	}
-	fdstRoot := fixRoot(fdst)
-	fsrcRoot := fixRoot(fsrc)
-	if strings.HasPrefix(fdstRoot, fsrcRoot) {
+	fdstRoot, fdstRootFolded := fixRoot(fdst)
+	fsrcRoot, fsrcRootFolded := fixRoot(fsrc)
+	if fdstRootFolded == fsrcRootFolded {
+		return true
+	} else if strings.HasPrefix(fdstRootFolded, fsrcRootFolded) {
 		fdstRelative := fdstRoot[len(fsrcRoot):]
-		return filterCheckR(ctx, fdstRelative, 0, fsrc)
+		return filterCheck(ctx, fsrc, fdstRelative)
+	} else if strings.HasPrefix(fsrcRootFolded, fdstRootFolded) {
+		fsrcRelative := fsrcRoot[len(fdstRoot):]
+		return filterCheck(ctx, fdst, fsrcRelative)
 	}
-	return strings.HasPrefix(fsrcRoot, fdstRoot)
+	return false
 }
 
-// filterCheckR checks if fdst would be included in the sync
-func filterCheckR(ctx context.Context, fdstRelative string, pos int, fsrc fs.Fs) bool {
-	include := true
+// filterCheck checks if dir is included in f
+func filterCheck(ctx context.Context, f fs.Fs, dir string) bool {
 	fi := filter.GetConfig(ctx)
-	includeDirectory := fi.IncludeDirectory(ctx, fsrc)
-	dirs := strings.SplitAfterN(fdstRelative, "/", pos+2)
-	newPath := ""
-	for i := 0; i <= pos; i++ {
-		newPath += dirs[i]
-	}
-	if !strings.HasSuffix(newPath, "/") {
-		newPath += "/"
-	}
-	if strings.HasPrefix(fdstRelative, newPath) {
-		include, _ = includeDirectory(newPath)
-		if include {
-			if newPath == fdstRelative {
-				return true
-			}
-			pos++
-			include = filterCheckR(ctx, fdstRelative, pos, fsrc)
-		}
+	includeDirectory := fi.IncludeDirectory(ctx, f)
+	include, err := includeDirectory(dir)
+	if err != nil {
+		fs.Errorf(f, "Failed to discover whether directory is included: %v", err)
+		return true
 	}
 	return include
 }
@@ -886,9 +879,9 @@ func SameDir(fdst, fsrc fs.Info) bool {
 	if !SameConfig(fdst, fsrc) {
 		return false
 	}
-	fdstRoot := fixRoot(fdst)
-	fsrcRoot := fixRoot(fsrc)
-	return fdstRoot == fsrcRoot
+	_, fdstRootFolded := fixRoot(fdst)
+	_, fsrcRootFolded := fixRoot(fsrc)
+	return fdstRootFolded == fsrcRootFolded
 }
 
 // Retry runs fn up to maxTries times if it returns a retriable error

fs/operations/operations_test.go

@@ -1418,11 +1418,13 @@ func TestOverlappingFilterCheckWithFilter(t *testing.T) {
 	ctx := context.Background()
 	fi, err := filter.NewFilter(nil)
 	require.NoError(t, err)
-	require.NoError(t, fi.Add(false, "*/exclude/"))
-	fi.Opt.ExcludeFile = []string{".ignore"}
+	require.NoError(t, fi.Add(false, "/exclude/"))
+	require.NoError(t, fi.Add(false, "/Exclude2/"))
+	require.NoError(t, fi.Add(true, "*"))
 	ctx = filter.ReplaceConfig(ctx, fi)
 
 	src := &testFs{testFsInfo{name: "name", root: "root"}}
+	src.features.CaseInsensitive = true
 	slash := string(os.PathSeparator) // native path separator
 	for _, test := range []struct {
 		name     string
@@ -1430,25 +1432,32 @@ func TestOverlappingFilterCheckWithFilter(t *testing.T) {
 		expected bool
 	}{
 		{"name", "root", true},
+		{"name", "ROOT", true}, // case insensitive is set
 		{"name", "/root", true},
 		{"name", "root/", true},
 		{"name", "root" + slash, true},
 		{"name", "root/exclude", false},
+		{"name", "root/Exclude2", false},
+		{"name", "root/include", true},
 		{"name", "root/exclude/", false},
+		{"name", "root/Exclude2/", false},
+		{"name", "root/exclude/sub", false},
+		{"name", "root/Exclude2/sub", false},
 		{"name", "/root/exclude/", false},
 		{"name", "root" + slash + "exclude", false},
 		{"name", "root" + slash + "exclude" + slash, false},
-		{"name", "root/.ignore", false},
-		{"name", "root" + slash + ".ignore", false},
 		{"namey", "root/include", false},
 		{"namey", "root/include/", false},
 		{"namey", "root" + slash + "include", false},
 		{"namey", "root" + slash + "include" + slash, false},
 	} {
 		dst := &testFs{testFsInfo{name: test.name, root: test.root}}
+		dst.features.CaseInsensitive = true
 		what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
 		actual := operations.OverlappingFilterCheck(ctx, dst, src)
 		assert.Equal(t, test.expected, actual, what)
+		actual = operations.OverlappingFilterCheck(ctx, src, dst)
+		assert.Equal(t, test.expected, actual, what)
 	}
 }
 

fs/rc/internal.go

@@ -389,10 +389,7 @@ func rcSetSoftMemoryLimit(ctx context.Context, in Params) (out Params, err error
 	if err != nil {
 		return nil, err
 	}
-	oldMemLimit, err := debug.SetMemoryLimit(memLimit)
-	if err != nil {
-		return nil, err
-	}
+	oldMemLimit := debug.SetMemoryLimit(memLimit)
 	out = Params{
 		"existing-mem-limit": oldMemLimit,
 	}

fs/rc/jobs/job.go

@@ -10,6 +10,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/cache"
@@ -120,8 +121,9 @@ type Jobs struct {
 }
 
 var (
-	running = newJobs()
-	jobID   = int64(0)
+	running   = newJobs()
+	jobID     = int64(0)
+	executeID = uuid.New().String()
 )
 
 // newJobs makes a new Jobs structure
@@ -403,7 +405,8 @@ func init() {
 
 Results:
 
-- jobids - array of integer job ids.
+- executeId - string id of rclone executing (change after restart)
+- jobids - array of integer job ids (starting at 1 on each restart)
 `,
 	})
 }
@@ -412,6 +415,7 @@ Results:
 func rcJobList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
 	out = make(rc.Params)
 	out["jobids"] = running.IDs()
+	out["executeId"] = executeID
 	return out, nil
 }
 

fs/rc/jobs/job_test.go

@@ -374,10 +374,24 @@ func TestRcJobList(t *testing.T) {
 	call := rc.Calls.Get("job/list")
 	assert.NotNil(t, call)
 	in := rc.Params{}
-	out, err := call.Fn(context.Background(), in)
+	out1, err := call.Fn(context.Background(), in)
 	require.NoError(t, err)
-	require.NotNil(t, out)
-	assert.Equal(t, rc.Params{"jobids": []int64{1}}, out)
+	require.NotNil(t, out1)
+	assert.Equal(t, []int64{1}, out1["jobids"], "should have job listed")
+
+	_, _, err = NewJob(ctx, longFn, rc.Params{"_async": true})
+	assert.NoError(t, err)
+
+	call = rc.Calls.Get("job/list")
+	assert.NotNil(t, call)
+	in = rc.Params{}
+	out2, err := call.Fn(context.Background(), in)
+	require.NoError(t, err)
+	require.NotNil(t, out2)
+	assert.Equal(t, 2, len(out2["jobids"].([]int64)), "should have all jobs listed")
+
+	require.NotNil(t, out1["executeId"], "should have executeId")
+	assert.Equal(t, out1["executeId"], out2["executeId"], "executeId should be the same")
 }
 
 func TestRcAsyncJobStop(t *testing.T) {

fs/rc/rc.go

@@ -18,22 +18,21 @@ import (
 
 // Options contains options for the remote control server
 type Options struct {
-	HTTP                     libhttp.Config
-	Auth                     libhttp.AuthConfig
-	Template                 libhttp.TemplateConfig
-	Enabled                  bool   // set to enable the server
-	Serve                    bool   // set to serve files from remotes
-	Files                    string // set to enable serving files locally
-	NoAuth                   bool   // set to disable auth checks on AuthRequired methods
-	WebUI                    bool   // set to launch the web ui
-	WebGUIUpdate             bool   // set to check new update
-	WebGUIForceUpdate        bool   // set to force download new update
-	WebGUINoOpenBrowser      bool   // set to disable auto opening browser
-	WebGUIFetchURL           string // set the default url for fetching webgui
-	AccessControlAllowOrigin string // set the access control for CORS configuration
-	EnableMetrics            bool   // set to disable prometheus metrics on /metrics
-	JobExpireDuration        time.Duration
-	JobExpireInterval        time.Duration
+	HTTP                libhttp.Config
+	Auth                libhttp.AuthConfig
+	Template            libhttp.TemplateConfig
+	Enabled             bool   // set to enable the server
+	Serve               bool   // set to serve files from remotes
+	Files               string // set to enable serving files locally
+	NoAuth              bool   // set to disable auth checks on AuthRequired methods
+	WebUI               bool   // set to launch the web ui
+	WebGUIUpdate        bool   // set to check new update
+	WebGUIForceUpdate   bool   // set to force download new update
+	WebGUINoOpenBrowser bool   // set to disable auto opening browser
+	WebGUIFetchURL      string // set the default url for fetching webgui
+	EnableMetrics       bool   // set to disable prometheus metrics on /metrics
+	JobExpireDuration   time.Duration
+	JobExpireInterval   time.Duration
 }
 
 // DefaultOpt is the default values used for Options

fs/rc/rcflags/rcflags.go

@@ -27,7 +27,6 @@ func AddFlags(flagSet *pflag.FlagSet) {
 	flags.BoolVarP(flagSet, &Opt.WebGUIForceUpdate, "rc-web-gui-force-update", "", false, "Force update to latest version of web gui")
 	flags.BoolVarP(flagSet, &Opt.WebGUINoOpenBrowser, "rc-web-gui-no-open-browser", "", false, "Don't open the browser automatically")
 	flags.StringVarP(flagSet, &Opt.WebGUIFetchURL, "rc-web-fetch-url", "", "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest", "URL to fetch the releases for webgui")
-	flags.StringVarP(flagSet, &Opt.AccessControlAllowOrigin, "rc-allow-origin", "", "", "Set the allowed origin for CORS")
 	flags.BoolVarP(flagSet, &Opt.EnableMetrics, "rc-enable-metrics", "", false, "Enable prometheus metrics on /metrics")
 	flags.DurationVarP(flagSet, &Opt.JobExpireDuration, "rc-job-expire-duration", "", Opt.JobExpireDuration, "Expire finished async jobs older than this value")
 	flags.DurationVarP(flagSet, &Opt.JobExpireInterval, "rc-job-expire-interval", "", Opt.JobExpireInterval, "Interval to check for expired async jobs")

fs/rc/rcserver/rcserver.go

@@ -15,7 +15,6 @@ import (
 	"regexp"
 	"sort"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/go-chi/chi/v5/middleware"
@@ -38,7 +37,6 @@ import (
 )
 
 var promHandler http.Handler
-var onlyOnceWarningAllowOrigin sync.Once
 
 func init() {
 	rcloneCollector := accounting.NewRcloneCollector(context.Background())
@@ -214,28 +212,6 @@ func writeError(path string, in rc.Params, w http.ResponseWriter, err error, sta
 func (s *Server) handler(w http.ResponseWriter, r *http.Request) {
 	path := strings.TrimLeft(r.URL.Path, "/")
 
-	allowOrigin := rcflags.Opt.AccessControlAllowOrigin
-	if allowOrigin != "" {
-		onlyOnceWarningAllowOrigin.Do(func() {
-			if allowOrigin == "*" {
-				fs.Logf(nil, "Warning: Allow origin set to *. This can cause serious security problems.")
-			}
-		})
-		w.Header().Add("Access-Control-Allow-Origin", allowOrigin)
-	} else {
-		urls := s.server.URLs()
-		if len(urls) == 1 {
-			w.Header().Add("Access-Control-Allow-Origin", urls[0])
-		} else {
-			fs.Errorf(nil, "Warning, need exactly 1 URL for Access-Control-Allow-Origin, got %d %q", len(urls), urls)
-		}
-	}
-
-	// echo back access control headers client needs
-	//reqAccessHeaders := r.Header.Get("Access-Control-Request-Headers")
-	w.Header().Add("Access-Control-Request-Method", "POST, OPTIONS, GET, HEAD")
-	w.Header().Add("Access-Control-Allow-Headers", "authorization, Content-Type")
-
 	switch r.Method {
 	case "POST":
 		s.handlePost(w, r, path)

fs/rc/rcserver/rcserver_test.go

@@ -552,32 +552,6 @@ Unknown command
 	testServer(t, tests, &opt)
 }
 
-func TestMethods(t *testing.T) {
-	tests := []testRun{{
-		Name:     "options",
-		URL:      "",
-		Method:   "OPTIONS",
-		Status:   http.StatusOK,
-		Expected: "",
-		Headers: map[string]string{
-			"Access-Control-Allow-Origin":   "testURL",
-			"Access-Control-Request-Method": "POST, OPTIONS, GET, HEAD",
-			"Access-Control-Allow-Headers":  "authorization, Content-Type",
-		},
-	}, {
-		Name:   "bad",
-		URL:    "",
-		Method: "POTATO",
-		Status: http.StatusMethodNotAllowed,
-		Expected: `Method Not Allowed
-`,
-	}}
-	opt := newTestOpt()
-	opt.Serve = true
-	opt.Files = testFs
-	testServer(t, tests, &opt)
-}
-
 func TestMetrics(t *testing.T) {
 	stats := accounting.GlobalStats()
 	tests := makeMetricsTestCases(stats)

fs/registry.go

@@ -154,6 +154,7 @@ type Option struct {
 	NoPrefix   bool             // set if the option for this should not use the backend prefix
 	Advanced   bool             // set if this is an advanced config option
 	Exclusive  bool             // set if the answer can only be one of the examples (empty string allowed unless Required or Default is set)
+	Sensitive  bool             // set if this option should be redacted when using rclone config redacted
 }
 
 // BaseOption is an alias for Option used internally

fs/sync/sync.go

@@ -20,6 +20,14 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 )
 
+// ErrorMaxDurationReached defines error when transfer duration is reached
+// Used for checking on exit and matching to correct exit code.
+var ErrorMaxDurationReached = errors.New("max transfer duration reached as set by --max-duration")
+
+// ErrorMaxDurationReachedFatal is returned from when the max
+// duration limit is reached.
+var ErrorMaxDurationReachedFatal = fserrors.FatalError(ErrorMaxDurationReached)
+
 type syncCopyMove struct {
 	// parameters
 	fdst               fs.Fs
@@ -845,10 +853,6 @@ func (s *syncCopyMove) tryRename(src fs.Object) bool {
 	return true
 }
 
-// errorMaxDurationReached defines error when transfer duration is reached
-// Used for checking on exit and matching to correct exit code.
-var errorMaxDurationReached = fserrors.FatalError(errors.New("max transfer duration reached as set by --max-duration"))
-
 // Syncs fsrc into fdst
 //
 // If Delete is true then it deletes any files in fdst that aren't in fsrc
@@ -945,8 +949,8 @@ func (s *syncCopyMove) run() error {
 
 	// If the duration was exceeded then add a Fatal Error so we don't retry
 	if !s.maxDurationEndTime.IsZero() && time.Since(s.maxDurationEndTime) > 0 {
-		fs.Errorf(s.fdst, "%v", errorMaxDurationReached)
-		s.processError(errorMaxDurationReached)
+		fs.Errorf(s.fdst, "%v", ErrorMaxDurationReachedFatal)
+		s.processError(ErrorMaxDurationReachedFatal)
 	}
 
 	// Print nothing to transfer message if there were no transfers and no errors

fs/sync/sync_test.go

@@ -996,7 +996,7 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
 	accounting.GlobalStats().ResetCounters()
 	startTime := time.Now()
 	err := Sync(ctx, r.Fremote, r.Flocal, false)
-	require.True(t, errors.Is(err, errorMaxDurationReached))
+	require.True(t, errors.Is(err, ErrorMaxDurationReached))
 
 	if cutoffMode == fs.CutoffModeHard {
 		r.CheckRemoteItems(t, file1)
@@ -1423,7 +1423,7 @@ func TestSyncOverlapWithFilter(t *testing.T) {
 	require.NoError(t, fi.Add(false, "/rclone-sync-test/"))
 	require.NoError(t, fi.Add(false, "*/layer2/"))
 	fi.Opt.ExcludeFile = []string{".ignore"}
-	ctx = filter.ReplaceConfig(ctx, fi)
+	filterCtx := filter.ReplaceConfig(ctx, fi)
 
 	subRemoteName := r.FremoteName + "/rclone-sync-test"
 	FremoteSync, err := fs.NewFs(ctx, subRemoteName)
@@ -1453,19 +1453,27 @@ func TestSyncOverlapWithFilter(t *testing.T) {
 	}
 
 	accounting.GlobalStats().ResetCounters()
-	checkNoErr(Sync(ctx, FremoteSync, r.Fremote, false))
+	checkNoErr(Sync(filterCtx, FremoteSync, r.Fremote, false))
+	checkErr(Sync(ctx, FremoteSync, r.Fremote, false))
+	checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync, false))
 	checkErr(Sync(ctx, r.Fremote, FremoteSync, false))
+	checkErr(Sync(filterCtx, r.Fremote, r.Fremote, false))
 	checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
+	checkErr(Sync(filterCtx, FremoteSync, FremoteSync, false))
 	checkErr(Sync(ctx, FremoteSync, FremoteSync, false))
 
-	checkNoErr(Sync(ctx, FremoteSync2, r.Fremote, false))
+	checkNoErr(Sync(filterCtx, FremoteSync2, r.Fremote, false))
+	checkErr(Sync(ctx, FremoteSync2, r.Fremote, false))
+	checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync2, false))
 	checkErr(Sync(ctx, r.Fremote, FremoteSync2, false))
-	checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
+	checkErr(Sync(filterCtx, FremoteSync2, FremoteSync2, false))
 	checkErr(Sync(ctx, FremoteSync2, FremoteSync2, false))
 
-	checkNoErr(Sync(ctx, FremoteSync3, r.Fremote, false))
+	checkNoErr(Sync(filterCtx, FremoteSync3, r.Fremote, false))
+	checkErr(Sync(ctx, FremoteSync3, r.Fremote, false))
+	checkNoErr(Sync(filterCtx, r.Fremote, FremoteSync3, false))
 	checkErr(Sync(ctx, r.Fremote, FremoteSync3, false))
-	checkErr(Sync(ctx, r.Fremote, r.Fremote, false))
+	checkErr(Sync(filterCtx, FremoteSync3, FremoteSync3, false))
 	checkErr(Sync(ctx, FremoteSync3, FremoteSync3, false))
 }
 

fs/versioncheck.go

@@ -1,8 +1,8 @@
-//go:build !go1.18
-// +build !go1.18
+//go:build !go1.19
+// +build !go1.19
 
 package fs
 
-// Upgrade to Go version 1.18 to compile rclone - latest stable go
+// Upgrade to Go version 1.19 to compile rclone - latest stable go
 // compiler recommended.
-func init() { Go_version_1_18_required_for_compilation() }
+func init() { Go_version_1_19_required_for_compilation() }

fstest/fstest.go

@@ -281,8 +281,8 @@ func CheckListingWithRoot(t *testing.T, f fs.Fs, dir string, items []Item, expec
 		if err != nil && err != fs.ErrorDirNotFound {
 			t.Fatalf("Error listing: %v", err)
 		}
-
 		gotListing = makeListingFromObjects(objs)
+
 		listingOK = wantListing == gotListing
 		if listingOK && (expectedDirs == nil || len(dirs) == len(expectedDirs)) {
 			// Put an extra sleep in if we did any retries just to make sure it really

fstest/test_all/config.yaml

@@ -238,6 +238,9 @@ backends:
  - backend:  "sftp"
    remote:   "TestSFTPRclone:"
    fastlist: false
+ - backend:  "sftp"
+   remote:   "TestSFTPRcloneSSH:"
+   fastlist: false
  - backend:  "sftp"
    remote:   "TestSFTPRsyncNet:"
    fastlist: false
@@ -363,6 +366,9 @@ backends:
  - backend:  "premiumizeme"
    remote:   "TestPremiumizeMe:"
    fastlist: false
+ - backend:  "protondrive"
+   remote:   "TestProtonDrive:"
+   fastlist: false
  - backend:  "putio"
    remote:   "TestPutio:"
    fastlist: false

fstest/test_all/test_all.go

@@ -73,7 +73,7 @@ func main() {
 	configfile.Install()
 
 	// Seed the random number generator
-	rand.Seed(time.Now().UTC().UnixNano())
+	randInstance := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
 
 	// Filter selection
 	if *testRemotes != "" {
@@ -103,7 +103,7 @@ func main() {
 
 	// Runs we will do for this test in random order
 	runs := conf.MakeRuns()
-	rand.Shuffle(len(runs), runs.Swap)
+	randInstance.Shuffle(len(runs), runs.Swap)
 
 	// Create Report
 	report := NewReport()

fstest/testserver/init.d/TestSFTPRcloneSSH

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# No password to make working with ssh binary easy
+
+NAME=rclone-serve-sftp-ssh
+IP=127.0.0.1
+PORT=28623
+
+start() {
+    run rclone serve sftp --addr ${IP}:${PORT} ${DATADIR}
+    
+    echo type=sftp
+    echo ssh=ssh -o StrictHostKeyChecking=no -p ${PORT} user@${IP}
+    echo _connect=${IP}:${PORT}
+}
+
+. $(dirname "$0")/rclone-serve.bash

go.mod

@@ -1,61 +1,63 @@
 module github.com/rclone/rclone
 
-go 1.18
+go 1.19
 
 require (
 	bazil.org/fuse v0.0.0-20221209211307-2abb8038c751
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0
-	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
 	github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358
 	github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd
 	github.com/Unknwon/goconfig v1.0.0
 	github.com/a8m/tree v0.0.0-20230208161321-36ae24ddad15
 	github.com/aalpar/deheap v0.0.0-20210914013432-0cc84d79dec3
 	github.com/abbot/go-http-auth v0.4.0
-	github.com/anacrolix/dms v1.5.0
-	github.com/anacrolix/log v0.13.1
-	github.com/artyom/mtab v1.0.0
+	github.com/anacrolix/dms v1.6.0
+	github.com/anacrolix/log v0.14.1
 	github.com/atotto/clipboard v0.1.4
-	github.com/aws/aws-sdk-go v1.44.246
+	github.com/aws/aws-sdk-go v1.44.311
 	github.com/buengese/sgzip v0.1.1
-	github.com/colinmarc/hdfs/v2 v2.3.0
+	github.com/colinmarc/hdfs/v2 v2.4.0
 	github.com/coreos/go-semver v0.3.1
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/dop251/scsu v0.0.0-20220106150536-84ac88021d00
 	github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
 	github.com/gabriel-vasile/mimetype v1.4.2
 	github.com/gdamore/tcell/v2 v2.6.0
-	github.com/go-chi/chi/v5 v5.0.8
+	github.com/go-chi/chi/v5 v5.0.10
 	github.com/google/uuid v1.3.0
-	github.com/hanwen/go-fuse/v2 v2.2.1-0.20230410213758-80c1c8221982
+	github.com/hanwen/go-fuse/v2 v2.3.0
+	github.com/henrybear327/Proton-API-Bridge v0.0.0-20230725204219-611ec433f83c
+	github.com/henrybear327/go-proton-api v0.0.0-20230725203741-316f1b3227a9
 	github.com/hirochachacha/go-smb2 v1.1.0
 	github.com/iguanesolutions/go-systemd/v5 v5.1.1
 	github.com/jcmturner/gokrb5/v8 v8.4.4
 	github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004
-	github.com/klauspost/compress v1.16.5
+	github.com/klauspost/compress v1.16.7
 	github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6
 	github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6
 	github.com/mattn/go-colorable v0.1.13
-	github.com/mattn/go-runewidth v0.0.14
+	github.com/mattn/go-runewidth v0.0.15
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/moby/sys/mountinfo v0.6.2
 	github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1
-	github.com/ncw/swift/v2 v2.0.1
-	github.com/oracle/oci-go-sdk/v65 v65.34.0
+	github.com/ncw/swift/v2 v2.0.2
+	github.com/oracle/oci-go-sdk/v65 v65.45.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6
 	github.com/pmezard/go-difflib v1.0.0
-	github.com/prometheus/client_golang v1.14.0
+	github.com/prometheus/client_golang v1.15.1
 	github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8
 	github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d
 	github.com/rfjakob/eme v1.1.2
 	github.com/rivo/uniseg v0.4.4
-	github.com/shirou/gopsutil/v3 v3.23.5
-	github.com/sirupsen/logrus v1.9.0
+	github.com/shirou/gopsutil/v3 v3.23.6
+	github.com/sirupsen/logrus v1.9.3
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/spf13/cobra v1.7.0
 	github.com/spf13/pflag v1.0.5
-	github.com/stretchr/testify v1.8.3
+	github.com/stretchr/testify v1.8.4
 	github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca
 	github.com/winfsp/cgofuse v1.5.1-0.20221118130120-84c0898ad2e0
 	github.com/xanzy/ssh-agent v0.3.3
@@ -63,37 +65,53 @@ require (
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0
 	go.etcd.io/bbolt v1.3.7
 	goftp.io/server v1.0.0-rc1
-	golang.org/x/crypto v0.7.0
-	golang.org/x/net v0.8.0
-	golang.org/x/oauth2 v0.6.0
-	golang.org/x/sync v0.1.0
-	golang.org/x/sys v0.8.0
-	golang.org/x/text v0.8.0
+	golang.org/x/crypto v0.11.0
+	golang.org/x/net v0.12.0
+	golang.org/x/oauth2 v0.10.0
+	golang.org/x/sync v0.3.0
+	golang.org/x/sys v0.10.0
+	golang.org/x/text v0.11.0
 	golang.org/x/time v0.3.0
-	google.golang.org/api v0.115.0
+	google.golang.org/api v0.134.0
 	gopkg.in/yaml.v2 v2.4.0
-	storj.io/uplink v1.10.0
+	storj.io/uplink v1.11.0
 )
 
 require (
-	cloud.google.com/go/compute v1.19.0 // indirect
+	cloud.google.com/go/compute v1.20.1 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
+	github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
+	github.com/ProtonMail/gluon v0.17.0 // indirect
+	github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
+	github.com/ProtonMail/go-srp v0.0.7 // indirect
+	github.com/ProtonMail/gopenpgp/v2 v2.7.2 // indirect
+	github.com/PuerkitoBio/goquery v1.8.1 // indirect
+	github.com/anacrolix/generics v0.0.0-20230113004304-d6428d516633 // indirect
+	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/bradenaw/juniper v0.13.1 // indirect
 	github.com/calebcase/tmpfile v1.0.3 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cloudflare/circl v1.3.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/cronokirby/saferith v0.33.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emersion/go-message v0.16.0 // indirect
+	github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
+	github.com/emersion/go-vcard v0.0.0-20230626131229-38c18b295bbd // indirect
+	github.com/flynn/noise v1.0.0 // indirect
 	github.com/gdamore/encoding v1.0.0 // indirect
 	github.com/geoffgarside/ber v1.1.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/go-resty/resty/v2 v2.7.0 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
-	github.com/googleapis/gax-go/v2 v2.8.0 // indirect
+	github.com/google/s2a-go v0.1.4 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
@@ -106,23 +124,26 @@ require (
 	github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
-	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
+	github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/kr/fs v0.1.0 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/prometheus/common v0.42.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/relvacode/iso8601 v1.3.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shoenig/go-m1cpu v0.1.6 // indirect
 	github.com/sony/gobreaker v0.5.0 // indirect
-	github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
+	github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
@@ -130,23 +151,24 @@ require (
 	github.com/zeebo/blake3 v0.2.3 // indirect
 	github.com/zeebo/errs v1.3.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230331144136-dcfb400f0633 // indirect
-	google.golang.org/grpc v1.54.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230720185612-659f7aaaa771 // indirect
+	google.golang.org/grpc v1.56.2 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
-	storj.io/drpc v0.0.32 // indirect
+	storj.io/common v0.0.0-20230602145716-d6ea82d58b3d // indirect
+	storj.io/drpc v0.0.33 // indirect
+	storj.io/picobuf v0.0.1 // indirect
 )
 
 require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8
+	github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/pkg/xattr v0.4.9
-	golang.org/x/mobile v0.0.0-20230301163155-e0f57694e12c
-	golang.org/x/term v0.7.0
+	golang.org/x/mobile v0.0.0-20230531173138-3c911d8e3eda
+	golang.org/x/term v0.10.0
 )