Version v1.68.0

The upload routine no longer returns a url to download the object.

This fixes the problem by fetching it if necessary when we attempt to
Open the object.

.gitattributes

@@ -1,3 +1,7 @@
+# Go writes go.mod and go.sum with lf even on windows
+go.mod text eol=lf
+go.sum text eol=lf
+
 # Ignore generated files in GitHub language statistics and diffs
 /MANUAL.* linguist-generated=true
 /rclone.1 linguist-generated=true

.github/workflows/build.yml

@@ -27,12 +27,12 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.19', 'go1.20']
+        job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.21.0-rc.3'
+            go: '>=1.23.0-rc.1'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -43,14 +43,14 @@ jobs:
 
           - job_name: linux_386
             os: ubuntu-latest
-            go: '1.21.0-rc.3'
+            go: '>=1.23.0-rc.1'
             goarch: 386
             gotags: cmount
             quicktest: true
 
           - job_name: mac_amd64
-            os: macos-11
-            go: '1.21.0-rc.3'
+            os: macos-latest
+            go: '>=1.23.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -58,15 +58,15 @@ jobs:
             deploy: true
 
           - job_name: mac_arm64
-            os: macos-11
-            go: '1.21.0-rc.3'
+            os: macos-latest
+            go: '>=1.23.0-rc.1'
             gotags: 'cmount'
             build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
             deploy: true
 
           - job_name: windows
             os: windows-latest
-            go: '1.21.0-rc.3'
+            go: '>=1.23.0-rc.1'
             gotags: cmount
             cgo: '0'
             build_flags: '-include "^windows/"'
@@ -76,20 +76,20 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.21.0-rc.3'
+            go: '>=1.23.0-rc.1'
             build_flags: '-exclude "^(windows/|darwin/|linux/)"'
             compile_all: true
             deploy: true
 
-          - job_name: go1.19
+          - job_name: go1.21
             os: ubuntu-latest
-            go: '1.19'
+            go: '1.21'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.20
+          - job_name: go1.22
             os: ubuntu-latest
-            go: '1.20'
+            go: '1.22'
             quicktest: true
             racequicktest: true
 
@@ -99,12 +99,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
           check-latest: true
@@ -124,7 +124,7 @@ jobs:
           sudo modprobe fuse
           sudo chmod 666 /dev/fuse
           sudo chown root:$USER /etc/fuse.conf
-          sudo apt-get install fuse3 libfuse-dev rpm pkg-config
+          sudo apt-get install fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
         if: matrix.os == 'ubuntu-latest'
 
       - name: Install Libraries on macOS
@@ -137,7 +137,8 @@ jobs:
           brew untap --force homebrew/cask
           brew update
           brew install --cask macfuse
-        if: matrix.os == 'macos-11'
+          brew install git-annex git-annex-remote-rclone
+        if: matrix.os == 'macos-latest'
 
       - name: Install Libraries on Windows
         shell: powershell
@@ -167,14 +168,6 @@ jobs:
           printf "\n\nSystem environment:\n\n"
           env
 
-      - name: Go module cache
-        uses: actions/cache@v3
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
       - name: Build rclone
         shell: bash
         run: |
@@ -216,7 +209,6 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
-          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
           make ci_beta
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
@@ -231,21 +223,71 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      - name: Get runner parameters
+        id: get-runner-parameters
+        shell: bash
+        run: |
+          echo "year-week=$(/bin/date -u "+%Y%V")" >> $GITHUB_OUTPUT
+          echo "runner-os-version=$ImageOS" >> $GITHUB_OUTPUT
+
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Code quality test
-        uses: golangci/golangci-lint-action@v3
-        with:
-          # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version
-          version: latest
-
-      # Run govulncheck on the latest go version, the one we build binaries with
       - name: Install Go
-        uses: actions/setup-go@v4
+        id: setup-go
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.21.0-rc.3'
+          go-version: '>=1.23.0-rc.1'
           check-latest: true
+          cache: false
+
+      - name: Cache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            ~/.cache/golangci-lint
+          key: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-${{ hashFiles('go.sum') }}
+          restore-keys: golangci-lint-${{ steps.get-runner-parameters.outputs.runner-os-version }}-go${{ steps.setup-go.outputs.go-version }}-${{ steps.get-runner-parameters.outputs.year-week }}-
+
+      - name: Code quality test (Linux)
+        uses: golangci/golangci-lint-action@v6
+        with:
+          version: latest
+          skip-cache: true
+
+      - name: Code quality test (Windows)
+        uses: golangci/golangci-lint-action@v6
+        env:
+          GOOS: "windows"
+        with:
+          version: latest
+          skip-cache: true
+
+      - name: Code quality test (macOS)
+        uses: golangci/golangci-lint-action@v6
+        env:
+          GOOS: "darwin"
+        with:
+          version: latest
+          skip-cache: true
+
+      - name: Code quality test (FreeBSD)
+        uses: golangci/golangci-lint-action@v6
+        env:
+          GOOS: "freebsd"
+        with:
+          version: latest
+          skip-cache: true
+
+      - name: Code quality test (OpenBSD)
+        uses: golangci/golangci-lint-action@v6
+        env:
+          GOOS: "openbsd"
+        with:
+          version: latest
+          skip-cache: true
 
       - name: Install govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -261,23 +303,15 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       # Upgrade together with NDK version
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: '1.21.0-rc.3'
-
-      - name: Go module cache
-        uses: actions/cache@v3
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: '>=1.23.0-rc.1'
 
       - name: Set global environment variables
         shell: bash

.github/workflows/build_publish_beta_docker_image.yml

@@ -10,26 +10,35 @@ jobs:
         runs-on: ubuntu-latest
         name: Build image job
         steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Login to Docker Hub
-              uses: docker/login-action@v2
+              uses: docker/login-action@v3
               with:
                 username: ${{ secrets.DOCKERHUB_USERNAME }}
                 password: ${{ secrets.DOCKERHUB_TOKEN }}
             - name: Extract metadata (tags, labels) for Docker
               id: meta
-              uses: docker/metadata-action@v4
+              uses: docker/metadata-action@v5
               with:
                 images: ghcr.io/${{ github.repository }}
             - name: Set up QEMU
-              uses: docker/setup-qemu-action@v2
+              uses: docker/setup-qemu-action@v3
             - name: Set up Docker Buildx
-              uses: docker/setup-buildx-action@v2
+              uses: docker/setup-buildx-action@v3
             - name: Login to GitHub Container Registry
-              uses: docker/login-action@v2
+              uses: docker/login-action@v3
               with:
                 registry: ghcr.io
                 # This is the user that triggered the Workflow. In this case, it will
@@ -42,9 +51,12 @@ jobs:
                 # See https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret
                 # for more detailed information.
                 password: ${{ secrets.GITHUB_TOKEN }}
-
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .
             - name: Build and publish image
-              uses: docker/build-push-action@v4
+              uses: docker/build-push-action@v6
               with:
                 file: Dockerfile
                 context: .
@@ -54,8 +66,12 @@ jobs:
                   rclone/rclone:beta
                 labels: ${{ steps.meta.outputs.labels }}
                 platforms: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
-                cache-from: type=gha
-                cache-to: type=gha,mode=max
+                cache-from: type=gha, scope=${{ github.workflow }}
+                cache-to: type=gha, mode=max, scope=${{ github.workflow }}
                 provenance: false
               # Eventually cache will need to be cleared if builds more frequent than once a week
               # https://github.com/docker/build-push-action/issues/252
+            - name: Show disk usage
+              shell: bash
+              run: |
+                df -h .

.github/workflows/build_publish_release_docker_image.yml

@@ -10,8 +10,17 @@ jobs:
         runs-on: ubuntu-latest
         name: Build image job
         steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Get actual patch version
@@ -39,8 +48,17 @@ jobs:
         runs-on: ubuntu-latest
         name: Build docker plugin job
         steps:
+            - name: Free some space
+              shell: bash
+              run: |
+                df -h .
+                # Remove android SDK
+                sudo rm -rf /usr/local/lib/android || true
+                # Remove .net runtime
+                sudo rm -rf /usr/share/dotnet || true
+                df -h .
             - name: Checkout master
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
               with:
                 fetch-depth: 0
             - name: Build and publish docker plugin

.github/workflows/notify.yml

@@ -0,0 +1,15 @@
+name: Notify users based on issue labels
+
+on:
+  issues:
+      types: [labeled]
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+    steps:
+        - uses: jenschelkopf/issue-label-notification-action@1.3
+          with:
+             token: ${{ secrets.NOTIFY_ACTION_TOKEN }}
+             recipients: |
+                  Support Contract=@rclone/support

.github/workflows/winget.yml

@@ -1,14 +1,14 @@
-name: Publish to Winget
-on:
-  release:
-    types: [released]
-
-jobs:
-  publish:
-    runs-on: windows-latest # Action can only run on Windows
-    steps:
-      - uses: vedantmgoyal2009/winget-releaser@v2
-        with:
-          identifier: Rclone.Rclone
-          installers-regex: '-windows-\w+\.zip$'
-          token: ${{ secrets.WINGET_TOKEN }}
+name: Publish to Winget
+on:
+  release:
+    types: [released]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: vedantmgoyal2009/winget-releaser@v2
+        with:
+          identifier: Rclone.Rclone
+          installers-regex: '-windows-\w+\.zip$'
+          token: ${{ secrets.WINGET_TOKEN }}

.gitignore

@@ -3,15 +3,20 @@ _junk/
 rclone
 rclone.exe
 build
-docs/public
+/docs/public/
+/docs/.hugo_build.lock
+/docs/static/img/logos/
 rclone.iml
 .idea
 .history
+.vscode
 *.test
-*.log
 *.iml
 fuzz-build.zip
 *.orig
 *.rej
 Thumbs.db
 __pycache__
+.DS_Store
+resource_windows_*.syso
+.devcontainer

.golangci.yml

@@ -13,6 +13,7 @@ linters:
     - stylecheck
     - unused
     - misspell
+    - gocritic
     #- prealloc
     #- maligned
   disable-all: true
@@ -33,24 +34,111 @@ issues:
       - staticcheck
       text: 'SA1019: "github.com/rclone/rclone/cmd/serve/httplib" is deprecated'
 
+  # don't disable the revive messages about comments on exported functions
+  include:
+    - EXC0012
+    - EXC0013
+    - EXC0014
+    - EXC0015
+
 run:
   # timeout for analysis, e.g. 30s, 5m, default is 1m
   timeout: 10m
 
 linters-settings:
   revive:
+    # setting rules seems to disable all the rules, so re-enable them here
     rules:
-      - name: unreachable-code
-        disabled: true
-      - name: unused-parameter
-        disabled: true
+      - name: blank-imports
+        disabled: false
+      - name: context-as-argument
+        disabled: false
+      - name: context-keys-type
+        disabled: false
+      - name: dot-imports
+        disabled: false
       - name: empty-block
         disabled: true
+      - name: error-naming
+        disabled: false
+      - name: error-return
+        disabled: false
+      - name: error-strings
+        disabled: false
+      - name: errorf
+        disabled: false
+      - name: exported
+        disabled: false
+      - name: increment-decrement
+        disabled: true
+      - name: indent-error-flow
+        disabled: false
+      - name: package-comments
+        disabled: false
+      - name: range
+        disabled: false
+      - name: receiver-naming
+        disabled: false
       - name: redefines-builtin-id
         disabled: true
       - name: superfluous-else
         disabled: true
+      - name: time-naming
+        disabled: false
+      - name: unexported-return
+        disabled: false
+      - name: unreachable-code
+        disabled: true
+      - name: unused-parameter
+        disabled: true
+      - name: var-declaration
+        disabled: false
+      - name: var-naming
+        disabled: false
   stylecheck:
     # Only enable the checks performed by the staticcheck stand-alone tool,
     # as documented here: https://staticcheck.io/docs/configuration/options/#checks
     checks: ["all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022", "-ST1023"]
+  gocritic:
+    # Enable all default checks with some exceptions and some additions (commented).
+    # Cannot use both enabled-checks and disabled-checks, so must specify all to be used.
+    disable-all: true
+    enabled-checks:
+      #- appendAssign # Enabled by default
+      - argOrder
+      - assignOp
+      - badCall
+      - badCond
+      #- captLocal # Enabled by default
+      - caseOrder
+      - codegenComment
+      #- commentFormatting # Enabled by default
+      - defaultCaseOrder
+      - deprecatedComment
+      - dupArg
+      - dupBranchBody
+      - dupCase
+      - dupSubExpr
+      - elseif
+      #- exitAfterDefer # Enabled by default
+      - flagDeref
+      - flagName
+      #- ifElseChain # Enabled by default
+      - mapKey
+      - newDeref
+      - offBy1
+      - regexpMust
+      - ruleguard # Not enabled by default
+      #- singleCaseSwitch # Enabled by default
+      - sloppyLen
+      - sloppyTypeAssert
+      - switchTrue
+      - typeSwitchVar
+      - underef
+      - unlambda
+      - unslice
+      - valSwap
+      - wrapperFunc
+    settings:
+      ruleguard:
+        rules: "${configDir}/bin/rules.go"

CONTRIBUTING.md

@@ -1,8 +1,8 @@
-# Contributing to rclone #
+# Contributing to rclone
 
 This is a short guide on how to contribute things to rclone.
 
-## Reporting a bug ##
+## Reporting a bug
 
 If you've just got a question or aren't sure if you've found a bug
 then please use the [rclone forum](https://forum.rclone.org/) instead
@@ -12,13 +12,13 @@ When filing an issue, please include the following information if
 possible as well as a description of the problem.  Make sure you test
 with the [latest beta of rclone](https://beta.rclone.org/):
 
-  * Rclone version (e.g. output from `rclone version`)
-  * Which OS you are using and how many bits (e.g. Windows 10, 64 bit)
-  * The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
-  * A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
-    * if the log contains secrets then edit the file with a text editor first to obscure them
+- Rclone version (e.g. output from `rclone version`)
+- Which OS you are using and how many bits (e.g. Windows 10, 64 bit)
+- The command you were trying to run (e.g. `rclone copy /tmp remote:tmp`)
+- A log of the command with the `-vv` flag (e.g. output from `rclone -vv copy /tmp remote:tmp`)
+    - if the log contains secrets then edit the file with a text editor first to obscure them
 
-## Submitting a new feature or bug fix ##
+## Submitting a new feature or bug fix
 
 If you find a bug that you'd like to fix, or a new feature that you'd
 like to implement then please submit a pull request via GitHub.
@@ -73,9 +73,9 @@ This is typically enough if you made a simple bug fix, otherwise please read the
 
 Make sure you
 
-  * Add [unit tests](#testing) for a new feature.
-  * Add [documentation](#writing-documentation) for a new feature.
-  * [Commit your changes](#committing-your-changes) using the [message guideline](#commit-messages).
+- Add [unit tests](#testing) for a new feature.
+- Add [documentation](#writing-documentation) for a new feature.
+- [Commit your changes](#committing-your-changes) using the [commit message guidelines](#commit-messages).
 
 When you are done with that push your changes to GitHub:
 
@@ -88,9 +88,9 @@ Your changes will then get reviewed and you might get asked to fix some stuff. I
 
 You may sometimes be asked to [base your changes on the latest master](#basing-your-changes-on-the-latest-master) or [squash your commits](#squashing-your-commits).
 
-## Using Git and GitHub ##
+## Using Git and GitHub
 
-### Committing your changes ###
+### Committing your changes
 
 Follow the guideline for [commit messages](#commit-messages) and then:
 
@@ -107,7 +107,7 @@ You can modify the message or changes in the latest commit using:
 
 If you amend to commits that have been pushed to GitHub, then you will have to [replace your previously pushed commits](#replacing-your-previously-pushed-commits).
 
-### Replacing your previously pushed commits ###
+### Replacing your previously pushed commits
 
 Note that you are about to rewrite the GitHub history of your branch. It is good practice to involve your collaborators before modifying commits that have been pushed to GitHub.
 
@@ -115,7 +115,7 @@ Your previously pushed commits are replaced by:
 
     git push --force origin my-new-feature 
 
-### Basing your changes on the latest master ###
+### Basing your changes on the latest master
 
 To base your changes on the latest version of the [rclone master](https://github.com/rclone/rclone/tree/master) (upstream):
 
@@ -149,13 +149,21 @@ If you squash commits that have been pushed to GitHub, then you will have to [re
 
 Tip: You may like to use `git rebase -i master` if you are experienced or have a more complex situation.
 
-### GitHub Continuous Integration ###
+### GitHub Continuous Integration
 
 rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
 
-## Testing ##
+## Testing
 
-### Quick testing ###
+### Code quality tests
+
+If you install [golangci-lint](https://github.com/golangci/golangci-lint) then you can run the same tests as get run in the CI which can be very helpful.
+
+You can run them with `make check` or with `golangci-lint run ./...`.
+
+Using these tests ensures that the rclone codebase all uses the same coding standards. These tests also check for easy mistakes to make (like forgetting to check an error return).
+
+### Quick testing
 
 rclone's tests are run from the go testing framework, so at the top
 level you can run this to run all the tests.
@@ -168,7 +176,7 @@ You can also use `make`, if supported by your platform
 
 The quicktest is [automatically run by GitHub](#github-continuous-integration) when you push your branch to GitHub.
 
-### Backend testing ###
+### Backend testing
 
 rclone contains a mixture of unit tests and integration tests.
 Because it is difficult (and in some respects pointless) to test cloud
@@ -201,9 +209,9 @@ altogether with an HTML report and test retries then from the
 project root:
 
     go install github.com/rclone/rclone/fstest/test_all
-    test_all -backend drive
+    test_all -backends drive
 
-### Full integration testing ###
+### Full integration testing
 
 If you want to run all the integration tests against all the remotes,
 then change into the project root and run
@@ -218,55 +226,56 @@ The commands may require some extra go packages which you can install with
 The full integration tests are run daily on the integration test server. You can
 find the results at https://pub.rclone.org/integration-tests/
 
-## Code Organisation ##
+## Code Organisation
 
 Rclone code is organised into a small number of top level directories
 with modules beneath.
 
-  * backend - the rclone backends for interfacing to cloud providers -
-    * all - import this to load all the cloud providers
-    * ...providers
-  * bin - scripts for use while building or maintaining rclone
-  * cmd - the rclone commands
-    * all - import this to load all the commands
-    * ...commands
-  * cmdtest - end-to-end tests of commands, flags, environment variables,...
-  * docs - the documentation and website
-    * content - adjust these docs only - everything else is autogenerated
-      * command - these are auto-generated - edit the corresponding .go file
-  * fs - main rclone definitions - minimal amount of code
-    * accounting - bandwidth limiting and statistics
-    * asyncreader - an io.Reader which reads ahead
-    * config - manage the config file and flags
-    * driveletter - detect if a name is a drive letter
-    * filter - implements include/exclude filtering
-    * fserrors - rclone specific error handling
-    * fshttp - http handling for rclone
-    * fspath - path handling for rclone
-    * hash - defines rclone's hash types and functions
-    * list - list a remote
-    * log - logging facilities
-    * march - iterates directories in lock step
-    * object - in memory Fs objects
-    * operations - primitives for sync, e.g. Copy, Move
-    * sync - sync directories
-    * walk - walk a directory
-  * fstest - provides integration test framework
-    * fstests - integration tests for the backends
-    * mockdir - mocks an fs.Directory
-    * mockobject - mocks an fs.Object
-    * test_all - Runs integration tests for everything
-  * graphics - the images used in the website, etc.
-  * lib - libraries used by the backend
-    * atexit - register functions to run when rclone exits
-    * dircache - directory ID to name caching
-    * oauthutil - helpers for using oauth
-    * pacer - retries with backoff and paces operations
-    * readers - a selection of useful io.Readers
-    * rest - a thin abstraction over net/http for REST
-  * vfs - Virtual FileSystem layer for implementing rclone mount and similar
+- backend - the rclone backends for interfacing to cloud providers -
+    - all - import this to load all the cloud providers
+    - ...providers
+- bin - scripts for use while building or maintaining rclone
+- cmd - the rclone commands
+    - all - import this to load all the commands
+    - ...commands
+- cmdtest - end-to-end tests of commands, flags, environment variables,...
+- docs - the documentation and website
+    - content - adjust these docs only - everything else is autogenerated
+    - command - these are auto-generated - edit the corresponding .go file
+- fs - main rclone definitions - minimal amount of code
+    - accounting - bandwidth limiting and statistics
+    - asyncreader - an io.Reader which reads ahead
+    - config - manage the config file and flags
+    - driveletter - detect if a name is a drive letter
+    - filter - implements include/exclude filtering
+    - fserrors - rclone specific error handling
+    - fshttp - http handling for rclone
+    - fspath - path handling for rclone
+    - hash - defines rclone's hash types and functions
+    - list - list a remote
+    - log - logging facilities
+    - march - iterates directories in lock step
+    - object - in memory Fs objects
+    - operations - primitives for sync, e.g. Copy, Move
+    - sync - sync directories
+    - walk - walk a directory
+- fstest - provides integration test framework
+    - fstests - integration tests for the backends
+    - mockdir - mocks an fs.Directory
+    - mockobject - mocks an fs.Object
+    - test_all - Runs integration tests for everything
+- graphics - the images used in the website, etc.
+- lib - libraries used by the backend
+    - atexit - register functions to run when rclone exits
+    - dircache - directory ID to name caching
+    - oauthutil - helpers for using oauth
+    - pacer - retries with backoff and paces operations
+    - readers - a selection of useful io.Readers
+    - rest - a thin abstraction over net/http for REST
+- librclone - in memory interface to rclone's API for embedding rclone
+- vfs - Virtual FileSystem layer for implementing rclone mount and similar
 
-## Writing Documentation ##
+## Writing Documentation
 
 If you are adding a new feature then please update the documentation.
 
@@ -277,22 +286,22 @@ alphabetical order.
 If you add a new backend option/flag, then it should be documented in
 the source file in the `Help:` field.
 
-  * Start with the most important information about the option,
+- Start with the most important information about the option,
     as a single sentence on a single line.
-    * This text will be used for the command-line flag help.
-    * It will be combined with other information, such as any default value,
+    - This text will be used for the command-line flag help.
+    - It will be combined with other information, such as any default value,
       and the result will look odd if not written as a single sentence.
-    * It should end with a period/full stop character, which will be shown
+    - It should end with a period/full stop character, which will be shown
       in docs but automatically removed when producing the flag help.
-    * Try to keep it below 80 characters, to reduce text wrapping in the terminal.
-  * More details can be added in a new paragraph, after an empty line (`"\n\n"`).
-    * Like with docs generated from Markdown, a single line break is ignored
+    - Try to keep it below 80 characters, to reduce text wrapping in the terminal.
+- More details can be added in a new paragraph, after an empty line (`"\n\n"`).
+    - Like with docs generated from Markdown, a single line break is ignored
       and two line breaks creates a new paragraph.
-    * This text will be shown to the user in `rclone config`
+    - This text will be shown to the user in `rclone config`
       and in the docs (where it will be added by `make backenddocs`,
       normally run some time before next release).
-  * To create options of enumeration type use the `Examples:` field.
-    * Each example value have their own `Help:` field, but they are treated
+- To create options of enumeration type use the `Examples:` field.
+    - Each example value have their own `Help:` field, but they are treated
       a bit different than the main option help text. They will be shown
       as an unordered list, therefore a single line break is enough to
       create a new list item. Also, for enumeration texts like name of
@@ -312,12 +321,12 @@ combined unmodified with other information (such as any default value).
 Note that you can use [GitHub's online editor](https://help.github.com/en/github/managing-files-in-a-repository/editing-files-in-another-users-repository)
 for small changes in the docs which makes it very easy.
 
-## Making a release ##
+## Making a release
 
 There are separate instructions for making a release in the RELEASE.md
 file.
 
-## Commit messages ##
+## Commit messages
 
 Please make the first line of your commit message a summary of the
 change that a user (not a developer) of rclone would like to read, and
@@ -358,7 +367,7 @@ error fixing the hang.
 Fixes #1498
 ```
 
-## Adding a dependency ##
+## Adding a dependency
 
 rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
@@ -370,7 +379,7 @@ To add a dependency `github.com/ncw/new_dependency` see the
 instructions below.  These will fetch the dependency and add it to
 `go.mod` and `go.sum`.
 
-    GO111MODULE=on go get github.com/ncw/new_dependency
+    go get github.com/ncw/new_dependency
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
@@ -378,15 +387,15 @@ go docs linked above), but don't unless you really need to.
 Please check in the changes generated by `go mod` including `go.mod`
 and `go.sum` in the same commit as your other changes.
 
-## Updating a dependency ##
+## Updating a dependency
 
 If you need to update a dependency then run
 
-    GO111MODULE=on go get -u golang.org/x/crypto
+    go get golang.org/x/crypto
 
 Check in a single commit as above.
 
-## Updating all the dependencies ##
+## Updating all the dependencies
 
 In order to update all the dependencies then run `make update`.  This
 just uses the go modules to update all the modules to their latest
@@ -395,7 +404,7 @@ stable release. Check in the changes in a single commit as above.
 This should be done early in the release cycle to pick up new versions
 of packages in time for them to get some testing.
 
-## Updating a backend ##
+## Updating a backend
 
 If you update a backend then please run the unit tests and the
 integration tests for that backend.
@@ -410,82 +419,133 @@ integration tests.
 
 The next section goes into more detail about the tests.
 
-## Writing a new backend ##
+## Writing a new backend
 
 Choose a name.  The docs here will use `remote` as an example.
 
 Note that in rclone terminology a file system backend is called a
 remote or an fs.
 
-Research
+### Research
 
-  * Look at the interfaces defined in `fs/types.go`
-  * Study one or more of the existing remotes
+- Look at the interfaces defined in `fs/types.go`
+- Study one or more of the existing remotes
 
-Getting going
+### Getting going
 
-  * Create `backend/remote/remote.go` (copy this from a similar remote)
-    * box is a good one to start from if you have a directory-based remote
-    * b2 is a good one to start from if you have a bucket-based remote
-  * Add your remote to the imports in `backend/all/all.go`
-  * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
-  * Try to implement as many optional methods as possible as it makes the remote more usable.
-  * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
-    * `rclone purge -v TestRemote:rclone-info`
-    * `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
-    * `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
-    * open `remote.csv` in a spreadsheet and examine
+- Create `backend/remote/remote.go` (copy this from a similar remote)
+    - box is a good one to start from if you have a directory-based remote (and shows how to use the directory cache)
+    - b2 is a good one to start from if you have a bucket-based remote
+- Add your remote to the imports in `backend/all/all.go`
+- HTTP based remotes are easiest to maintain if they use rclone's [lib/rest](https://pkg.go.dev/github.com/rclone/rclone/lib/rest) module, but if there is a really good Go SDK from the provider then use that instead.
+- Try to implement as many optional methods as possible as it makes the remote more usable.
+- Use [lib/encoder](https://pkg.go.dev/github.com/rclone/rclone/lib/encoder) to make sure we can encode any path name and `rclone info` to help determine the encodings needed
+    - `rclone purge -v TestRemote:rclone-info`
+    - `rclone test info --all --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
+    - `go run cmd/test/info/internal/build_csv/main.go -o remote.csv remote.json`
+    - open `remote.csv` in a spreadsheet and examine
 
-Unit tests
+### Guidelines for a speedy merge
 
-  * Create a config entry called `TestRemote` for the unit tests to use
-  * Create a `backend/remote/remote_test.go` - copy and adjust your example remote
-  * Make sure all tests pass with `go test -v`
+- **Do** use [lib/rest](https://pkg.go.dev/github.com/rclone/rclone/lib/rest) if you are implementing a REST like backend and parsing XML/JSON in the backend.
+- **Do** use rclone's Client or Transport from [fs/fshttp](https://pkg.go.dev/github.com/rclone/rclone/fs/fshttp) if your backend is HTTP based - this adds features like `--dump bodies`, `--tpslimit`, `--user-agent` without you having to code anything!
+- **Do** follow your example backend exactly - use the same code order, function names, layout, structure. **Don't** move stuff around and **Don't** delete the comments.
+- **Do not** split your backend up into `fs.go` and `object.go` (there are a few backends like that - don't follow them!)
+- **Do** put your API type definitions in a separate file - by preference `api/types.go`
+- **Remember** we have >50 backends to maintain so keeping them as similar as possible to each other is a high priority!
 
-Integration tests
+### Unit tests
 
-  * Add your backend to `fstest/test_all/config.yaml`
-      * Once you've done that then you can use the integration test framework from the project root:
-      * go install ./...
-      * test_all -backends remote
+- Create a config entry called `TestRemote` for the unit tests to use
+- Create a `backend/remote/remote_test.go` - copy and adjust your example remote
+- Make sure all tests pass with `go test -v`
+
+### Integration tests
+
+- Add your backend to `fstest/test_all/config.yaml`
+    - Once you've done that then you can use the integration test framework from the project root:
+    - go install ./...
+    - test_all -backends remote
 
 Or if you want to run the integration tests manually:
 
-  * Make sure integration tests pass with
-      * `cd fs/operations`
-      * `go test -v -remote TestRemote:`
-      * `cd fs/sync`
-      * `go test -v -remote TestRemote:`
-  * If your remote defines `ListR` check with this also
-      * `go test -v -remote TestRemote: -fast-list`
+- Make sure integration tests pass with
+    - `cd fs/operations`
+    - `go test -v -remote TestRemote:`
+    - `cd fs/sync`
+    - `go test -v -remote TestRemote:`
+- If your remote defines `ListR` check with this also
+    - `go test -v -remote TestRemote: -fast-list`
 
 See the [testing](#testing) section for more information on integration tests.
 
-Add your fs to the docs - you'll need to pick an icon for it from
+### Backend documentation
+
+Add your backend to the docs - you'll need to pick an icon for it from
 [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in
 alphabetical order of full name of remote (e.g. `drive` is ordered as
 `Google Drive`) but with the local file system last.
 
-  * `README.md` - main GitHub page
-  * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
-    * make sure this has the `autogenerated options` comments in (see your reference backend docs)
-    * update them with `make backenddocs` - revert any changes in other backends
-  * `docs/content/overview.md` - overview docs
-  * `docs/content/docs.md` - list of remotes in config section
-  * `docs/content/_index.md` - front page of rclone.org
-  * `docs/layouts/chrome/navbar.html` - add it to the website navigation
-  * `bin/make_manual.py` - add the page to the `docs` constant
+- `README.md` - main GitHub page
+- `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
+    - make sure this has the `autogenerated options` comments in (see your reference backend docs)
+    - update them in your backend with `bin/make_backend_docs.py remote`
+- `docs/content/overview.md` - overview docs
+- `docs/content/docs.md` - list of remotes in config section
+- `docs/content/_index.md` - front page of rclone.org
+- `docs/layouts/chrome/navbar.html` - add it to the website navigation
+- `bin/make_manual.py` - add the page to the `docs` constant
 
 Once you've written the docs, run `make serve` and check they look OK
 in the web browser and the links (internal and external) all work.
 
-## Writing a plugin ##
+## Adding a new s3 provider
+
+It is quite easy to add a new S3 provider to rclone.
+
+You'll need to modify the following files
+
+- `backend/s3/s3.go`
+    - Add the provider to `providerOption` at the top of the file
+    - Add endpoints and other config for your provider gated on the provider in `fs.RegInfo`.
+    - Exclude your provider from generic config questions (eg `region` and `endpoint).
+    - Add the provider to the `setQuirks` function - see the documentation there.
+- `docs/content/s3.md`
+    - Add the provider at the top of the page.
+    - Add a section about the provider linked from there.
+    - Add a transcript of a trial `rclone config` session
+        - Edit the transcript to remove things which might change in subsequent versions
+    - **Do not** alter or add to the autogenerated parts of `s3.md`
+    - **Do not** run `make backenddocs` or `bin/make_backend_docs.py s3`
+- `README.md` - this is the home page in github
+    - Add the provider and a link to the section you wrote in `docs/contents/s3.md`
+- `docs/content/_index.md` - this is the home page of rclone.org
+    - Add the provider and a link to the section you wrote in `docs/contents/s3.md`
+
+When adding the provider, endpoints, quirks, docs etc keep them in
+alphabetical order by `Provider` name, but with `AWS` first and
+`Other` last.
+
+Once you've written the docs, run `make serve` and check they look OK
+in the web browser and the links (internal and external) all work.
+
+Once you've written the code, test `rclone config` works to your
+satisfaction, and check the integration tests work `go test -v -remote
+NewS3Provider:`. You may need to adjust the quirks to get them to
+pass. Some providers just can't pass the tests with control characters
+in the names so if these fail and the provider doesn't support
+`urlEncodeListings` in the quirks then ignore them. Note that the
+`SetTier` test may also fail on non AWS providers.
+
+For an example of adding an s3 provider see [eb3082a1](https://github.com/rclone/rclone/commit/eb3082a1ebdb76d5625f14cedec3f5154a5e7b10).
+
+## Writing a plugin
 
 New features (backends, commands) can also be added "out-of-tree", through Go plugins.
 Changes will be kept in a dynamically loaded file instead of being compiled into the main binary.
 This is useful if you can't merge your changes upstream or don't want to maintain a fork of rclone.
 
-Usage
+### Usage
 
  - Naming
    - Plugins names must have the pattern `librcloneplugin_KIND_NAME.so`.
@@ -500,7 +560,7 @@ Usage
    - Plugins must be compiled against the exact version of rclone to work.
      (The rclone used during building the plugin must be the same as the source of rclone)
 
-Building
+### Building
 
 To turn your existing additions into a Go plugin, move them to an external repository
 and change the top-level package name to `main`.

Dockerfile

@@ -1,8 +1,9 @@
-FROM golang AS builder
+FROM golang:alpine AS builder
 
 COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
+RUN apk add --no-cache make bash gawk git
 RUN \
   CGO_ENABLED=0 \
   make

MAINTAINERS.md

@@ -19,6 +19,10 @@ Current active maintainers of rclone are:
 | wiserain         | @wiserain         | pikpak backend               |
 | albertony        | @albertony        |                              |
 | Chun-Hung Tseng  | @henrybear327     | Proton Drive Backend         |
+| Hideo Aoyama     | @boukendesho      | snap packaging               |
+| nielash          | @nielash          | bisync                       |
+| Dan McArdle      | @dmcardle         | gitannex                     |
+| Sam Harrison     | @childish-sambino | filescom                     |
 
 **This is a work in progress Draft**
 

Makefile

@@ -30,29 +30,37 @@ ifdef RELEASE_TAG
 	TAG := $(RELEASE_TAG)
 endif
 GO_VERSION := $(shell go version)
+GO_OS := $(shell go env GOOS)
 ifdef BETA_SUBDIR
 	BETA_SUBDIR := /$(BETA_SUBDIR)
 endif
 BETA_PATH := $(BRANCH_PATH)$(TAG)$(BETA_SUBDIR)
 BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
-BETA_UPLOAD_ROOT := memstore:beta-rclone-org
+BETA_UPLOAD_ROOT := beta.rclone.org:
 BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
 LINTTAGS=--build-tags "$(GOTAGS)"
 endif
+LDFLAGS=--ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)"
 
 .PHONY: rclone test_all vars version
 
 rclone:
-	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS)
+ifeq ($(GO_OS),windows)
+	go run bin/resource_windows.go -version $(TAG) -syso resource_windows_`go env GOARCH`.syso
+endif
+	go build -v $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS)
+ifeq ($(GO_OS),windows)
+	rm resource_windows_`go env GOARCH`.syso
+endif
 	mkdir -p `go env GOPATH`/bin/
 	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
 	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
 
 test_all:
-	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
+	go install $(LDFLAGS) $(BUILDTAGS) $(BUILD_ARGS) github.com/rclone/rclone/fstest/test_all
 
 vars:
 	@echo SHELL="'$(SHELL)'"
@@ -66,6 +74,10 @@ btest:
 	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
 	@echo "Copied markdown of beta release to clip board"
 
+btesth:
+	@echo "<a href="$(BETA_URL)">$(TAG)</a> on branch <a href="https://github.com/rclone/rclone/tree/$(BRANCH)">$(BRANCH)</a> (uploaded in 15-30 mins)" | xclip -r -sel clip -t text/html
+	@echo "Copied beta release in HTML to clip board"
+
 version:
 	@echo '$(TAG)'
 
@@ -76,13 +88,13 @@ test:	rclone test_all
 
 # Quick test
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) ./...
 
 racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -cpu=2 -race ./...
 
 compiletest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -run XXX ./...
+	RCLONE_CONFIG="/notfound" go test $(LDFLAGS) $(BUILDTAGS) -run XXX ./...
 
 # Do source code quality checks
 check:	rclone
@@ -92,16 +104,12 @@ check:	rclone
 
 # Get the build dependencies
 build_dep:
-	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
+	go run bin/get-github-release.go -use-api -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
 # Get the release dependencies we only install on linux
 release_dep_linux:
 	go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
 
-# Get the release dependencies we only install on Windows
-release_dep_windows:
-	GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
-
 # Update dependencies
 showupdates:
 	@echo "*** Direct dependencies that could be updated ***"
@@ -146,7 +154,7 @@ rcdocs: rclone
 
 install: rclone
 	install -d ${DESTDIR}/usr/bin
-	install -t ${DESTDIR}/usr/bin ${GOPATH}/bin/rclone
+	install ${GOPATH}/bin/rclone ${DESTDIR}/usr/bin
 
 clean:
 	go clean ./...
@@ -160,7 +168,7 @@ website:
 	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
-	rclone -v sync docs/public memstore:www-rclone-org
+	rclone -v sync docs/public www.rclone.org:
 
 upload_test_website:	website
 	rclone -P sync docs/public test-rclone-org:
@@ -187,8 +195,8 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -P copy build/ memstore:downloads-rclone-org/$(TAG)
-	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "memstore:downloads-rclone-org/$(TAG)/$$i" "memstore:downloads-rclone-org/$$j"'
+	rclone -P copy build/ downloads.rclone.org:/$(TAG)
+	rclone lsf build --files-only --include '*.{zip,deb,rpm}' --include version.txt | xargs -i bash -c 'i={}; j="$$i"; [[ $$i =~ (.*)(-v[0-9\.]+-)(.*) ]] && j=$${BASH_REMATCH[1]}-current-$${BASH_REMATCH[3]}; rclone copyto -v "downloads.rclone.org:/$(TAG)/$$i" "downloads.rclone.org:/$$j"'
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -198,7 +206,7 @@ cross:	doc
 
 beta:
 	go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone -v copy build/ memstore:pub-rclone-org/$(TAG)
+	rclone -v copy build/ pub.rclone.org:/$(TAG)
 	@echo Beta release ready at https://pub.rclone.org/$(TAG)/
 
 log_since_last_release:
@@ -211,18 +219,18 @@ ci_upload:
 	sudo chown -R $$USER build
 	find build -type l -delete
 	gzip -r9v build
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
+	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+	./rclone --no-check-dest --config bin/ci.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
 ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
 ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
+	rclone --no-check-dest --config bin/ci.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)$(BETA_SUBDIR)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
@@ -231,7 +239,7 @@ fetch_binaries:
 	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server -v -w --disableFastRender
+	cd docs && hugo server --logLevel info -w --disableFastRender
 
 tag:	retag doc
 	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new

README.md

@@ -1,3 +1,21 @@
+<div align="center">
+<sup>Special thanks to our sponsor:</sup>
+<br>
+<br>
+<a href="https://www.warp.dev/?utm_source=github&utm_medium=referral&utm_campaign=rclone_20231103">
+  <div>
+    <img src="https://rclone.org/img/logos/warp-github.svg" width="300" alt="Warp">
+  </div>
+  <b>Warp is a modern, Rust-based terminal with AI built in so you and your team can build great software, faster.</b>
+  <div>
+    <sup>Visit warp.dev to learn more.</sup>
+  </div>
+</a>
+<br>
+<hr>
+</div>
+<br>
+
 [<img src="https://rclone.org/img/logo_on_light__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-light-mode-only)
 [<img src="https://rclone.org/img/logo_on_dark__horizontal_color.svg" width="50%" alt="rclone logo">](https://rclone.org/#gh-dark-mode-only)
 
@@ -23,7 +41,6 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
   * Akamai Netstorage [:page_facing_up:](https://rclone.org/netstorage/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
-  * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * ArvanCloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
@@ -38,14 +55,18 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * Enterprise File Fabric [:page_facing_up:](https://rclone.org/filefabric/)
   * Fastmail Files [:page_facing_up:](https://rclone.org/webdav/#fastmail-files)
+  * Files.com [:page_facing_up:](https://rclone.org/filescom/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
+  * GoFile [:page_facing_up:](https://rclone.org/gofile/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
   * HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
+  * Hetzner Storage Box [:page_facing_up:](https://rclone.org/sftp/#hetzner-storage-box)
   * HiDrive [:page_facing_up:](https://rclone.org/hidrive/)
   * HTTP [:page_facing_up:](https://rclone.org/http/)
   * Huawei Cloud Object Storage Service(OBS) [:page_facing_up:](https://rclone.org/s3/#huawei-obs)
+  * ImageKit [:page_facing_up:](https://rclone.org/imagekit/)
   * Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
@@ -53,11 +74,15 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
   * Leviia Object Storage [:page_facing_up:](https://rclone.org/s3/#leviia)
   * Liara Object Storage [:page_facing_up:](https://rclone.org/s3/#liara-object-storage)
+  * Linkbox [:page_facing_up:](https://rclone.org/linkbox)
+  * Linode Object Storage [:page_facing_up:](https://rclone.org/s3/#linode)
+  * Magalu Object Storage [:page_facing_up:](https://rclone.org/s3/#magalu)
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
   * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
+  * Microsoft Azure Files Storage [:page_facing_up:](https://rclone.org/azurefiles/)
   * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
   * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
@@ -71,12 +96,16 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * Petabox [:page_facing_up:](https://rclone.org/s3/#petabox)
   * PikPak [:page_facing_up:](https://rclone.org/pikpak/)
+  * Pixeldrain [:page_facing_up:](https://rclone.org/pixeldrain/)
   * premiumize.me [:page_facing_up:](https://rclone.org/premiumizeme/)
   * put.io [:page_facing_up:](https://rclone.org/putio/)
+  * Proton Drive [:page_facing_up:](https://rclone.org/protondrive/)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
+  * Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
+  * rsync.net [:page_facing_up:](https://rclone.org/sftp/#rsync-net)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SeaweedFS [:page_facing_up:](https://rclone.org/s3/#seaweedfs)
@@ -87,6 +116,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
   * Synology C2 Object Storage [:page_facing_up:](https://rclone.org/s3/#synology-c2)
   * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
+  * Uloz.to [:page_facing_up:](https://rclone.org/ulozto/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
@@ -115,6 +145,7 @@ These backends adapt or modify other storage providers
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
+  * [Bisync](https://rclone.org/bisync/) (two way) to keep two directories in sync bidirectionally
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
   * Can sync to and from network, e.g. two different cloud accounts
   * Optional large file chunking ([Chunker](https://rclone.org/chunker/))

RELEASE.md

@@ -37,16 +37,45 @@ This file describes how to make the various kinds of releases
 
 ## Update dependencies
 
-Early in the next release cycle update the dependencies
+Early in the next release cycle update the dependencies.
 
   * Review any pinned packages in go.mod and remove if possible
-  * make updatedirect
-  * make
-  * git commit -a -v
-  * make update
-  * make
+  * `make updatedirect`
+  * `make GOTAGS=cmount`
+  * `make compiletest`
+  * Fix anything which doesn't compile at this point and commit changes here
+  * `git commit -a -v -m "build: update all dependencies"`
+
+If the `make updatedirect` upgrades the version of go in the `go.mod`
+then go to manual mode. `go1.20` here is the lowest supported version
+in the `go.mod`.
+
+```
+go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all > /tmp/potential-upgrades
+go get -d $(cat /tmp/potential-upgrades)
+go mod tidy -go=1.20 -compat=1.20
+```
+
+If the `go mod tidy` fails use the output from it to remove the
+package which can't be upgraded from `/tmp/potential-upgrades` when
+done
+
+```
+git co go.mod go.sum
+```
+
+And try again.
+
+Optionally upgrade the direct and indirect dependencies. This is very
+likely to fail if the manual method was used abve - in that case
+ignore it as it is too time consuming to fix.
+
+  * `make update`
+  * `make GOTAGS=cmount`
+  * `make compiletest`
   * roll back any updates which didn't compile
-  * git commit -a -v --amend
+  * `git commit -a -v --amend`
+  * **NB** watch out for this changing the default go version in `go.mod`
 
 Note that `make update` updates all direct and indirect dependencies
 and there can occasionally be forwards compatibility problems with
@@ -54,6 +83,9 @@ doing that so it may be necessary to roll back dependencies to the
 version specified by `make updatedirect` in order to get rclone to
 build.
 
+Once it compiles locally, push it on a test branch and commit fixes
+until the tests pass.
+
 ## Tidy beta
 
 At some point after the release run
@@ -90,34 +122,52 @@ Now
   * git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
   * git push
 
+## Sponsor logos
+
+If updating the website note that the sponsor logos have been moved out of the main repository.
+
+You will need to checkout `/docs/static/img/logos` from https://github.com/rclone/third-party-logos
+which is a private repo containing artwork from sponsors.
+
+## Update the website between releases
+
+Create an update website branch based off the last release
+
+    git co -b update-website
+
+If the branch already exists, double check there are no commits that need saving.
+
+Now reset the branch to the last release
+
+    git reset --hard v1.64.0
+
+Create the changes, check them in, test with `make serve` then
+
+    make upload_test_website
+
+Check out https://test.rclone.org and when happy
+
+    make upload_website
+
+Cherry pick any changes back to master and the stable branch if it is active.
+
 ## Making a manual build of docker
 
-The rclone docker image should autobuild on via GitHub actions.  If it doesn't
-or needs to be updated then rebuild like this.
+To do a basic build of rclone's docker image to debug builds locally:
 
-See: https://github.com/ilteoood/docker_buildx/issues/19
-See: https://github.com/ilteoood/docker_buildx/blob/master/scripts/install_buildx.sh
+```
+docker buildx build --load -t rclone/rclone:testing --progress=plain .
+docker run --rm rclone/rclone:testing version
+```
+
+To test the multipatform build
+
+```
+docker buildx build -t rclone/rclone:testing --progress=plain --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6 .
+```
+
+To make a full build then set the tags correctly and add `--push`
 
 ```
-git co v1.54.1
-docker pull golang
-export DOCKER_CLI_EXPERIMENTAL=enabled
-docker buildx create --name actions_builder --use
-docker run --rm --privileged docker/binfmt:820fdd95a9972a5308930a2bdfb8573dd4447ad3
-docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-SUPPORTED_PLATFORMS=$(docker buildx inspect --bootstrap | grep 'Platforms:*.*' | cut -d : -f2,3)
-echo "Supported platforms: $SUPPORTED_PLATFORMS"
 docker buildx build --platform linux/amd64,linux/386,linux/arm64,linux/arm/v7 -t rclone/rclone:1.54.1 -t rclone/rclone:1.54 -t rclone/rclone:1 -t rclone/rclone:latest --push .
-docker buildx stop actions_builder
-```
-
-### Old build for linux/amd64 only
-
-```
-docker pull golang
-docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
-docker push rclone/rclone:1.52.0
-docker push rclone/rclone:1.52
-docker push rclone/rclone:1
-docker push rclone/rclone:latest
 ```

VERSION

@@ -1 +1 @@
-v1.64.0
+v1.68.0

backend/alias/alias_internal_test.go

@@ -23,8 +23,8 @@ func prepare(t *testing.T, root string) {
 	configfile.Install()
 
 	// Configure the remote
-	config.FileSet(remoteName, "type", "alias")
-	config.FileSet(remoteName, "remote", root)
+	config.FileSetValue(remoteName, "type", "alias")
+	config.FileSetValue(remoteName, "remote", root)
 }
 
 func TestNewFS(t *testing.T) {
@@ -81,10 +81,12 @@ func TestNewFS(t *testing.T) {
 		for i, gotEntry := range gotEntries {
 			what := fmt.Sprintf("%s, entry=%d", what, i)
 			wantEntry := test.entries[i]
+			_, isDir := gotEntry.(fs.Directory)
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
-			_, isDir := gotEntry.(fs.Directory)
+			if !isDir {
+				require.Equal(t, wantEntry.size, gotEntry.Size(), what)
+			}
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}
 	}

backend/all/all.go

@@ -4,8 +4,8 @@ package all
 import (
 	// Active file systems
 	_ "github.com/rclone/rclone/backend/alias"
-	_ "github.com/rclone/rclone/backend/amazonclouddrive"
 	_ "github.com/rclone/rclone/backend/azureblob"
+	_ "github.com/rclone/rclone/backend/azurefiles"
 	_ "github.com/rclone/rclone/backend/b2"
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
@@ -17,16 +17,20 @@ import (
 	_ "github.com/rclone/rclone/backend/dropbox"
 	_ "github.com/rclone/rclone/backend/fichier"
 	_ "github.com/rclone/rclone/backend/filefabric"
+	_ "github.com/rclone/rclone/backend/filescom"
 	_ "github.com/rclone/rclone/backend/ftp"
+	_ "github.com/rclone/rclone/backend/gofile"
 	_ "github.com/rclone/rclone/backend/googlecloudstorage"
 	_ "github.com/rclone/rclone/backend/googlephotos"
 	_ "github.com/rclone/rclone/backend/hasher"
 	_ "github.com/rclone/rclone/backend/hdfs"
 	_ "github.com/rclone/rclone/backend/hidrive"
 	_ "github.com/rclone/rclone/backend/http"
+	_ "github.com/rclone/rclone/backend/imagekit"
 	_ "github.com/rclone/rclone/backend/internetarchive"
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
+	_ "github.com/rclone/rclone/backend/linkbox"
 	_ "github.com/rclone/rclone/backend/local"
 	_ "github.com/rclone/rclone/backend/mailru"
 	_ "github.com/rclone/rclone/backend/mega"
@@ -37,10 +41,12 @@ import (
 	_ "github.com/rclone/rclone/backend/oracleobjectstorage"
 	_ "github.com/rclone/rclone/backend/pcloud"
 	_ "github.com/rclone/rclone/backend/pikpak"
+	_ "github.com/rclone/rclone/backend/pixeldrain"
 	_ "github.com/rclone/rclone/backend/premiumizeme"
 	_ "github.com/rclone/rclone/backend/protondrive"
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
+	_ "github.com/rclone/rclone/backend/quatrix"
 	_ "github.com/rclone/rclone/backend/s3"
 	_ "github.com/rclone/rclone/backend/seafile"
 	_ "github.com/rclone/rclone/backend/sftp"
@@ -50,6 +56,7 @@ import (
 	_ "github.com/rclone/rclone/backend/storj"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
+	_ "github.com/rclone/rclone/backend/ulozto"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/uptobox"
 	_ "github.com/rclone/rclone/backend/webdav"

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -1,21 +0,0 @@
-// Test AmazonCloudDrive filesystem interface
-
-//go:build acd
-// +build acd
-
-package amazonclouddrive_test
-
-import (
-	"testing"
-
-	"github.com/rclone/rclone/backend/amazonclouddrive"
-	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fstest/fstests"
-)
-
-// TestIntegration runs integration tests against the remote
-func TestIntegration(t *testing.T) {
-	fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
-	fstests.RemoteName = "TestAmazonCloudDrive:"
-	fstests.Run(t)
-}

backend/azureblob/azureblob.go

@@ -1,14 +1,13 @@
 //go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
 
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob
 
 import (
-	"bytes"
 	"context"
 	"crypto/md5"
 	"encoding/base64"
+	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
 	"errors"
@@ -18,6 +17,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -33,7 +33,6 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/chunksize"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -46,10 +45,8 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/env"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/pool"
-	"github.com/rclone/rclone/lib/readers"
-	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -70,12 +67,16 @@ const (
 	emulatorAccount      = "devstoreaccount1"
 	emulatorAccountKey   = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 	emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
-	memoryPoolFlushTime  = fs.Duration(time.Minute) // flush the cached buffers after this long
-	memoryPoolUseMmap    = false
 )
 
 var (
 	errCantUpdateArchiveTierBlobs = fserrors.NoRetryError(errors.New("can't update archive tier blob without --azureblob-archive-tier-delete"))
+
+	// Take this when changing or reading metadata.
+	//
+	// It acts as global metadata lock so we don't bloat Object
+	// with an extra lock that will only very rarely be contended.
+	metadataMu sync.Mutex
 )
 
 // Register with Fs
@@ -294,10 +295,10 @@ avoid the time out.`,
 			Advanced: true,
 		}, {
 			Name: "access_tier",
-			Help: `Access tier of blob: hot, cool or archive.
+			Help: `Access tier of blob: hot, cool, cold or archive.
 
-Archived blobs can be restored by setting access tier to hot or
-cool. Leave blank if you intend to use default access tier, which is
+Archived blobs can be restored by setting access tier to hot, cool or
+cold. Leave blank if you intend to use default access tier, which is
 set at account level
 
 If there is no "access tier" specified, rclone doesn't apply any tier.
@@ -305,7 +306,7 @@ rclone performs "Set Tier" operation on blobs while uploading, if objects
 are not modified, specifying "access tier" to new one will have no effect.
 If blobs are in "archive tier" at remote, trying to perform data transfer
 operations from remote will not be allowed. User should first restore by
-tiering blob to "Hot" or "Cool".`,
+tiering blob to "Hot", "Cool" or "Cold".`,
 			Advanced: true,
 		}, {
 			Name:    "archive_tier_delete",
@@ -337,17 +338,16 @@ to start uploading.`,
 			Advanced: true,
 		}, {
 			Name:     "memory_pool_flush_time",
-			Default:  memoryPoolFlushTime,
+			Default:  fs.Duration(time.Minute),
 			Advanced: true,
-			Help: `How often internal memory buffer pools will be flushed.
-
-Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
-This option controls how often unused buffers will be removed from the pool.`,
+			Hide:     fs.OptionHideBoth,
+			Help:     `How often internal memory buffer pools will be flushed. (no longer used)`,
 		}, {
 			Name:     "memory_pool_use_mmap",
-			Default:  memoryPoolUseMmap,
+			Default:  false,
 			Advanced: true,
-			Help:     `Whether to use mmap buffers in internal memory pool.`,
+			Hide:     fs.OptionHideBoth,
+			Help:     `Whether to use mmap buffers in internal memory pool. (no longer used)`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -401,6 +401,24 @@ rclone does if you know the container exists already.
 			Help:     `If set, do not do HEAD before GET when getting objects.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "delete_snapshots",
+			Help: `Set to specify how to deal with snapshots on blob deletion.`,
+			Examples: []fs.OptionExample{
+				{
+					Value: "",
+					Help:  "By default, the delete operation fails if a blob has snapshots",
+				}, {
+					Value: string(blob.DeleteSnapshotsOptionTypeInclude),
+					Help:  "Specify 'include' to remove the root blob and all its snapshots",
+				}, {
+					Value: string(blob.DeleteSnapshotsOptionTypeOnly),
+					Help:  "Specify 'only' to remove only the snapshots but keep the root blob.",
+				},
+			},
+			Default:   "",
+			Exclusive: true,
+			Advanced:  true,
 		}},
 	})
 }
@@ -432,13 +450,12 @@ type Options struct {
 	ArchiveTierDelete          bool                 `config:"archive_tier_delete"`
 	UseEmulator                bool                 `config:"use_emulator"`
 	DisableCheckSum            bool                 `config:"disable_checksum"`
-	MemoryPoolFlushTime        fs.Duration          `config:"memory_pool_flush_time"`
-	MemoryPoolUseMmap          bool                 `config:"memory_pool_use_mmap"`
 	Enc                        encoder.MultiEncoder `config:"encoding"`
 	PublicAccess               string               `config:"public_access"`
 	DirectoryMarkers           bool                 `config:"directory_markers"`
 	NoCheckContainer           bool                 `config:"no_check_container"`
 	NoHeadObject               bool                 `config:"no_head_object"`
+	DeleteSnapshots            string               `config:"delete_snapshots"`
 }
 
 // Fs represents a remote azure server
@@ -457,8 +474,6 @@ type Fs struct {
 	cache         *bucket.Cache                // cache for container creation status
 	pacer         *fs.Pacer                    // To pace and retry the API calls
 	uploadToken   *pacer.TokenDispenser        // control concurrency
-	pool          *pool.Pool                   // memory pool
-	poolSize      int64                        // size of pages in memory pool
 	publicAccess  container.PublicAccessType   // Container Public Access Level
 }
 
@@ -471,7 +486,7 @@ type Object struct {
 	size       int64             // Size of the object
 	mimeType   string            // Content-Type of the object
 	accessTier blob.AccessTier   // Blob Access Tier
-	meta       map[string]string // blob metadata
+	meta       map[string]string // blob metadata - take metadataMu when accessing
 }
 
 // ------------------------------------------------------------
@@ -524,6 +539,7 @@ func (o *Object) split() (container, containerPath string) {
 func validateAccessTier(tier string) bool {
 	return strings.EqualFold(tier, string(blob.AccessTierHot)) ||
 		strings.EqualFold(tier, string(blob.AccessTierCool)) ||
+		strings.EqualFold(tier, string(blob.AccessTierCold)) ||
 		strings.EqualFold(tier, string(blob.AccessTierArchive))
 }
 
@@ -653,8 +669,8 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	if opt.AccessTier == "" {
 		opt.AccessTier = string(defaultAccessTier)
 	} else if !validateAccessTier(opt.AccessTier) {
-		return nil, fmt.Errorf("supported access tiers are %s, %s and %s",
-			string(blob.AccessTierHot), string(blob.AccessTierCool), string(blob.AccessTierArchive))
+		return nil, fmt.Errorf("supported access tiers are %s, %s, %s and %s",
+			string(blob.AccessTierHot), string(blob.AccessTierCool), string(blob.AccessTierCold), string(blob.AccessTierArchive))
 	}
 
 	if !validatePublicAccess((opt.PublicAccess)) {
@@ -671,13 +687,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
 		cache:       bucket.NewCache(),
 		cntSVCcache: make(map[string]*container.Client, 1),
-		pool: pool.New(
-			time.Duration(opt.MemoryPoolFlushTime),
-			int(opt.ChunkSize),
-			ci.Transfers,
-			opt.MemoryPoolUseMmap,
-		),
-		poolSize: int64(opt.ChunkSize),
 	}
 	f.publicAccess = container.PublicAccessType(opt.PublicAccess)
 	f.setRoot(root)
@@ -702,10 +711,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ClientOptions: policyClientOptions,
 	}
 
-	// Here we auth by setting one of cred, sharedKeyCred or f.svc
+	// Here we auth by setting one of cred, sharedKeyCred, f.svc or anonymous
 	var (
 		cred          azcore.TokenCredential
 		sharedKeyCred *service.SharedKeyCredential
+		anonymous     = false
 	)
 	switch {
 	case opt.EnvAuth:
@@ -865,6 +875,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		if err != nil {
 			return nil, fmt.Errorf("failed to acquire MSI token: %w", err)
 		}
+	case opt.Account != "":
+		// Anonymous access
+		anonymous = true
 	default:
 		return nil, errors.New("no authentication method configured")
 	}
@@ -894,6 +907,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 			if err != nil {
 				return nil, fmt.Errorf("create client failed: %w", err)
 			}
+		} else if anonymous {
+			// Anonymous public access
+			f.svc, err = service.NewClientWithNoCredential(opt.Endpoint, &clientOpt)
+			if err != nil {
+				return nil, fmt.Errorf("create public client failed: %w", err)
+			}
 		}
 	}
 	if f.svc == nil {
@@ -972,6 +991,9 @@ func (f *Fs) getBlockBlobSVC(container, containerPath string) *blockblob.Client
 
 // updateMetadataWithModTime adds the modTime passed in to o.meta.
 func (o *Object) updateMetadataWithModTime(modTime time.Time) {
+	metadataMu.Lock()
+	defer metadataMu.Unlock()
+
 	// Make sure o.meta is not nil
 	if o.meta == nil {
 		o.meta = make(map[string]string, 1)
@@ -1076,7 +1098,7 @@ func (f *Fs) list(ctx context.Context, containerName, directory, prefix string,
 			isDirectory := isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote)
 			if isDirectory {
 				// Don't insert the root directory
-				if remote == directory {
+				if remote == f.opt.Enc.ToStandardPath(directory) {
 					continue
 				}
 				// process directory markers as directories
@@ -1503,7 +1525,7 @@ func (f *Fs) deleteContainer(ctx context.Context, containerName string) error {
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	container, directory := f.split(dir)
 	// Remove directory marker file
-	if f.opt.DirectoryMarkers && container != "" && dir != "" {
+	if f.opt.DirectoryMarkers && container != "" && directory != "" {
 		o := &Object{
 			fs:     f,
 			remote: dir + "/",
@@ -1537,7 +1559,10 @@ func (f *Fs) Hashes() hash.Set {
 // Purge deletes all the files and directories including the old versions.
 func (f *Fs) Purge(ctx context.Context, dir string) error {
 	container, directory := f.split(dir)
-	if container == "" || directory != "" {
+	if container == "" {
+		return errors.New("can't purge from root")
+	}
+	if directory != "" {
 		// Delegate to caller if not root of a container
 		return fs.ErrorCantPurge
 	}
@@ -1594,19 +1619,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
-func (f *Fs) getMemoryPool(size int64) *pool.Pool {
-	if size == int64(f.opt.ChunkSize) {
-		return f.pool
-	}
-
-	return pool.New(
-		time.Duration(f.opt.MemoryPoolFlushTime),
-		int(size),
-		f.ci.Transfers,
-		f.opt.MemoryPoolUseMmap,
-	)
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1650,6 +1662,9 @@ func (o *Object) Size() int64 {
 
 // Set o.metadata from metadata
 func (o *Object) setMetadata(metadata map[string]*string) {
+	metadataMu.Lock()
+	defer metadataMu.Unlock()
+
 	if len(metadata) > 0 {
 		// Lower case the metadata
 		o.meta = make(map[string]string, len(metadata))
@@ -1674,6 +1689,9 @@ func (o *Object) setMetadata(metadata map[string]*string) {
 
 // Get metadata from o.meta
 func (o *Object) getMetadata() (metadata map[string]*string) {
+	metadataMu.Lock()
+	defer metadataMu.Unlock()
+
 	if len(o.meta) == 0 {
 		return nil
 	}
@@ -1885,12 +1903,7 @@ func (o *Object) ModTime(ctx context.Context) (result time.Time) {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	// Make sure o.meta is not nil
-	if o.meta == nil {
-		o.meta = make(map[string]string, 1)
-	}
-	// Set modTimeKey in it
-	o.meta[modTimeKey] = modTime.Format(timeFormatOut)
+	o.updateMetadataWithModTime(modTime)
 
 	blb := o.getBlobSVC()
 	opt := blob.SetMetadataOptions{}
@@ -1916,7 +1929,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	var offset int64
 	var count int64
 	if o.AccessTier() == blob.AccessTierArchive {
-		return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot or cool first")
+		return nil, fmt.Errorf("blob in archive tier, you need to set tier to hot, cool, cold first")
 	}
 	fs.FixRangeOption(options, o.size)
 	for _, option := range options {
@@ -1982,34 +1995,42 @@ func (rs *readSeekCloser) Close() error {
 	return nil
 }
 
-// increment the slice passed in as LSB binary
-func increment(xs []byte) {
-	for i, digit := range xs {
-		newDigit := digit + 1
-		xs[i] = newDigit
-		if newDigit >= digit {
-			// exit if no carry
-			break
-		}
-	}
+// record chunk number and id for Close
+type azBlock struct {
+	chunkNumber uint64
+	id          string
 }
 
-var warnStreamUpload sync.Once
+// Implements the fs.ChunkWriter interface
+type azChunkWriter struct {
+	chunkSize int64
+	size      int64
+	f         *Fs
+	ui        uploadInfo
+	blocksMu  sync.Mutex // protects the below
+	blocks    []azBlock  // list of blocks for finalize
+	o         *Object
+}
 
-// uploadMultipart uploads a file using multipart upload
+// OpenChunkWriter returns the chunk size and a ChunkWriter
 //
-// Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
-func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, blb *blockblob.Client, httpHeaders *blob.HTTPHeaders) (err error) {
-	// Calculate correct partSize
-	partSize := o.fs.opt.ChunkSize
-	totalParts := -1
-
-	// make concurrency machinery
-	concurrency := o.fs.opt.UploadConcurrency
-	if concurrency < 1 {
-		concurrency = 1
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+	// Temporary Object under construction
+	o := &Object{
+		fs:     f,
+		remote: remote,
 	}
-	tokens := pacer.NewTokenDispenser(concurrency)
+	ui, err := o.prepareUpload(ctx, src, options)
+	if err != nil {
+		return info, nil, fmt.Errorf("failed to prepare upload: %w", err)
+	}
+
+	// Calculate correct partSize
+	partSize := f.opt.ChunkSize
+	totalParts := -1
+	size := src.Size()
 
 	// Note that the max size of file is 4.75 TB (100 MB X 50,000
 	// blocks) and this is bigger than the max uncommitted block
@@ -2023,13 +2044,13 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 	// 195GB which seems like a not too unreasonable limit.
 	if size == -1 {
 		warnStreamUpload.Do(func() {
-			fs.Logf(o, "Streaming uploads using chunk size %v will have maximum file size of %v",
-				o.fs.opt.ChunkSize, partSize*fs.SizeSuffix(blockblob.MaxBlocks))
+			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				f.opt.ChunkSize, partSize*fs.SizeSuffix(blockblob.MaxBlocks))
 		})
 	} else {
-		partSize = chunksize.Calculator(o, size, blockblob.MaxBlocks, o.fs.opt.ChunkSize)
+		partSize = chunksize.Calculator(remote, size, blockblob.MaxBlocks, f.opt.ChunkSize)
 		if partSize > fs.SizeSuffix(blockblob.MaxStageBlockBytes) {
-			return fmt.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), fs.SizeSuffix(blockblob.MaxBlocks), fs.SizeSuffix(blockblob.MaxStageBlockBytes))
+			return info, nil, fmt.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), fs.SizeSuffix(blockblob.MaxBlocks), fs.SizeSuffix(blockblob.MaxStageBlockBytes))
 		}
 		totalParts = int(fs.SizeSuffix(size) / partSize)
 		if fs.SizeSuffix(size)%partSize != 0 {
@@ -2039,173 +2060,275 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, partSize)
 
-	// unwrap the accounting from the input, we use wrap to put it
-	// back on after the buffering
-	in, wrap := accounting.UnWrap(in)
-
-	// FIXME it would be nice to delete uncommitted blocks
-	// See: https://github.com/rclone/rclone/issues/5583
-	//
-	// However there doesn't seem to be an easy way of doing this other than
-	// by deleting the target.
-	//
-	// This means that a failed upload deletes the target which isn't ideal.
-	//
-	// Uploading a zero length blob and deleting it will remove the
-	// uncommitted blocks I think.
-	//
-	// Could check to see if a file exists already and if it
-	// doesn't then create a 0 length file and delete it to flush
-	// the uncommitted blocks.
-	//
-	// This is what azcopy does
-	// https://github.com/MicrosoftDocs/azure-docs/issues/36347#issuecomment-541457962
-	// defer atexit.OnError(&err, func() {
-	// 	fs.Debugf(o, "Cancelling multipart upload")
-	//      // Code goes here!
-	// })()
-
-	// Upload the chunks
-	var (
-		g, gCtx       = errgroup.WithContext(ctx)
-		remaining     = fs.SizeSuffix(size)                 // remaining size in file for logging only, -1 if size < 0
-		position      = fs.SizeSuffix(0)                    // position in file
-		memPool       = o.fs.getMemoryPool(int64(partSize)) // pool to get memory from
-		finished      = false                               // set when we have read EOF
-		blocks        []string                              // list of blocks for finalize
-		binaryBlockID = make([]byte, 8)                     // block counter as LSB first 8 bytes
-	)
-	for part := 0; !finished; part++ {
-		// Get a block of memory from the pool and a token which limits concurrency
-		tokens.Get()
-		buf := memPool.Get()
-
-		free := func() {
-			memPool.Put(buf) // return the buf
-			tokens.Put()     // return the token
-		}
-
-		// Fail fast, in case an errgroup managed function returns an error
-		// gCtx is cancelled. There is no point in uploading all the other parts.
-		if gCtx.Err() != nil {
-			free()
-			break
-		}
-
-		// Read the chunk
-		n, err := readers.ReadFill(in, buf) // this can never return 0, nil
-		if err == io.EOF {
-			if n == 0 { // end if no data
-				free()
-				break
-			}
-			finished = true
-		} else if err != nil {
-			free()
-			return fmt.Errorf("multipart upload failed to read source: %w", err)
-		}
-		buf = buf[:n]
-
-		// increment the blockID and save the blocks for finalize
-		increment(binaryBlockID)
-		blockID := base64.StdEncoding.EncodeToString(binaryBlockID)
-		blocks = append(blocks, blockID)
-
-		// Transfer the chunk
-		fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %d", part+1, totalParts, position, fs.SizeSuffix(size), len(buf))
-		g.Go(func() (err error) {
-			defer free()
-
-			// Upload the block, with MD5 for check
-			md5sum := md5.Sum(buf)
-			transactionalMD5 := md5sum[:]
-			err = o.fs.pacer.Call(func() (bool, error) {
-				bufferReader := bytes.NewReader(buf)
-				wrappedReader := wrap(bufferReader)
-				rs := readSeekCloser{wrappedReader, bufferReader}
-				options := blockblob.StageBlockOptions{
-					// Specify the transactional md5 for the body, to be validated by the service.
-					TransactionalValidation: blob.TransferValidationTypeMD5(transactionalMD5),
-				}
-				_, err = blb.StageBlock(ctx, blockID, &rs, &options)
-				return o.fs.shouldRetry(ctx, err)
-			})
-			if err != nil {
-				return fmt.Errorf("multipart upload failed to upload part: %w", err)
-			}
-			return nil
-		})
-
-		// ready for next block
-		if size >= 0 {
-			remaining -= partSize
-		}
-		position += partSize
+	chunkWriter := &azChunkWriter{
+		chunkSize: int64(partSize),
+		size:      size,
+		f:         f,
+		ui:        ui,
+		o:         o,
 	}
-	err = g.Wait()
+	info = fs.ChunkWriterInfo{
+		ChunkSize:   int64(partSize),
+		Concurrency: o.fs.opt.UploadConcurrency,
+		//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
+	}
+	fs.Debugf(o, "open chunk writer: started multipart upload")
+	return info, chunkWriter, nil
+}
+
+// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
+func (w *azChunkWriter) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (int64, error) {
+	if chunkNumber < 0 {
+		err := fmt.Errorf("invalid chunk number provided: %v", chunkNumber)
+		return -1, err
+	}
+
+	// Upload the block, with MD5 for check
+	m := md5.New()
+	currentChunkSize, err := io.Copy(m, reader)
 	if err != nil {
-		return err
+		return -1, err
 	}
-
-	options := blockblob.CommitBlockListOptions{
-		Metadata:    o.getMetadata(),
-		Tier:        parseTier(o.fs.opt.AccessTier),
-		HTTPHeaders: httpHeaders,
+	// If no data read, don't write the chunk
+	if currentChunkSize == 0 {
+		return 0, nil
 	}
+	md5sum := m.Sum(nil)
 
-	// Finalise the upload session
-	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err := blb.CommitBlockList(ctx, blocks, &options)
-		return o.fs.shouldRetry(ctx, err)
+	// increment the blockID and save the blocks for finalize
+	var binaryBlockID [8]byte // block counter as LSB first 8 bytes
+	binary.LittleEndian.PutUint64(binaryBlockID[:], uint64(chunkNumber))
+	blockID := base64.StdEncoding.EncodeToString(binaryBlockID[:])
+
+	// Save the blockID for the commit
+	w.blocksMu.Lock()
+	w.blocks = append(w.blocks, azBlock{
+		chunkNumber: uint64(chunkNumber),
+		id:          blockID,
+	})
+	w.blocksMu.Unlock()
+
+	err = w.f.pacer.Call(func() (bool, error) {
+		// rewind the reader on retry and after reading md5
+		_, err = reader.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		options := blockblob.StageBlockOptions{
+			// Specify the transactional md5 for the body, to be validated by the service.
+			TransactionalValidation: blob.TransferValidationTypeMD5(md5sum),
+		}
+		_, err = w.ui.blb.StageBlock(ctx, blockID, &readSeekCloser{Reader: reader, Seeker: reader}, &options)
+		if err != nil {
+			if chunkNumber <= 8 {
+				return w.f.shouldRetry(ctx, err)
+			}
+			// retry all chunks once have done the first few
+			return true, err
+		}
+		return false, nil
 	})
 	if err != nil {
-		return fmt.Errorf("multipart upload failed to finalize: %w", err)
+		return -1, fmt.Errorf("failed to upload chunk %d with %v bytes: %w", chunkNumber+1, currentChunkSize, err)
 	}
+
+	fs.Debugf(w.o, "multipart upload wrote chunk %d with %v bytes", chunkNumber+1, currentChunkSize)
+	return currentChunkSize, err
+}
+
+// Abort the multipart upload.
+//
+// FIXME it would be nice to delete uncommitted blocks.
+//
+// See: https://github.com/rclone/rclone/issues/5583
+//
+// However there doesn't seem to be an easy way of doing this other than
+// by deleting the target.
+//
+// This means that a failed upload deletes the target which isn't ideal.
+//
+// Uploading a zero length blob and deleting it will remove the
+// uncommitted blocks I think.
+//
+// Could check to see if a file exists already and if it doesn't then
+// create a 0 length file and delete it to flush the uncommitted
+// blocks.
+//
+// This is what azcopy does
+// https://github.com/MicrosoftDocs/azure-docs/issues/36347#issuecomment-541457962
+func (w *azChunkWriter) Abort(ctx context.Context) error {
+	fs.Debugf(w.o, "multipart upload aborted (did nothing - see issue #5583)")
 	return nil
 }
 
-// uploadSinglepart uploads a short blob using a single part upload
-func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, blb *blockblob.Client, httpHeaders *blob.HTTPHeaders) (err error) {
-	// fs.Debugf(o, "Single part upload starting of object %d bytes", size)
-	if size > o.fs.poolSize || size < 0 {
-		return fmt.Errorf("internal error: single part upload size too big %d > %d", size, o.fs.opt.ChunkSize)
-	}
+// Close and finalise the multipart upload
+func (w *azChunkWriter) Close(ctx context.Context) (err error) {
+	// sort the completed parts by part number
+	sort.Slice(w.blocks, func(i, j int) bool {
+		return w.blocks[i].chunkNumber < w.blocks[j].chunkNumber
+	})
 
-	buf := o.fs.pool.Get()
-	defer o.fs.pool.Put(buf)
-
-	n, err := readers.ReadFill(in, buf)
-	if err == nil {
-		// Check to see whether in is exactly len(buf) or bigger
-		var buf2 = []byte{0}
-		n2, err2 := readers.ReadFill(in, buf2)
-		if n2 != 0 || err2 != io.EOF {
-			return fmt.Errorf("single part upload read failed: object longer than expected (expecting %d but got > %d)", size, len(buf))
+	// Create and check a list of block IDs
+	blockIDs := make([]string, len(w.blocks))
+	for i := range w.blocks {
+		if w.blocks[i].chunkNumber != uint64(i) {
+			return fmt.Errorf("internal error: expecting chunkNumber %d but got %d", i, w.blocks[i].chunkNumber)
 		}
+		chunkBytes, err := base64.StdEncoding.DecodeString(w.blocks[i].id)
+		if err != nil {
+			return fmt.Errorf("internal error: bad block ID: %w", err)
+		}
+		chunkNumber := binary.LittleEndian.Uint64(chunkBytes)
+		if w.blocks[i].chunkNumber != chunkNumber {
+			return fmt.Errorf("internal error: expecting decoded chunkNumber %d but got %d", w.blocks[i].chunkNumber, chunkNumber)
+		}
+		blockIDs[i] = w.blocks[i].id
 	}
+
+	options := blockblob.CommitBlockListOptions{
+		Metadata:    w.o.getMetadata(),
+		Tier:        parseTier(w.f.opt.AccessTier),
+		HTTPHeaders: &w.ui.httpHeaders,
+	}
+
+	// Finalise the upload session
+	err = w.f.pacer.Call(func() (bool, error) {
+		_, err := w.ui.blb.CommitBlockList(ctx, blockIDs, &options)
+		return w.f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to complete multipart upload: %w", err)
+	}
+	fs.Debugf(w.o, "multipart upload finished")
+	return err
+}
+
+var warnStreamUpload sync.Once
+
+// uploadMultipart uploads a file using multipart upload
+//
+// Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (ui uploadInfo, err error) {
+	chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
+		Open:        o.fs,
+		OpenOptions: options,
+	})
+	if err != nil {
+		return ui, err
+	}
+	return chunkWriter.(*azChunkWriter).ui, nil
+}
+
+// uploadSinglepart uploads a short blob using a single part upload
+func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, ui uploadInfo) (err error) {
+	chunkSize := int64(o.fs.opt.ChunkSize)
+	// fs.Debugf(o, "Single part upload starting of object %d bytes", size)
+	if size > chunkSize || size < 0 {
+		return fmt.Errorf("internal error: single part upload size too big %d > %d", size, chunkSize)
+	}
+
+	rw := multipart.NewRW()
+	defer fs.CheckClose(rw, &err)
+
+	n, err := io.CopyN(rw, in, size+1)
 	if err != nil && err != io.EOF {
 		return fmt.Errorf("single part upload read failed: %w", err)
 	}
-	if int64(n) != size {
+	if n != size {
 		return fmt.Errorf("single part upload: expecting to read %d bytes but read %d", size, n)
 	}
 
-	b := bytes.NewReader(buf[:n])
-	rs := &readSeekCloser{Reader: b, Seeker: b}
+	rs := &readSeekCloser{Reader: rw, Seeker: rw}
 
 	options := blockblob.UploadOptions{
 		Metadata:    o.getMetadata(),
 		Tier:        parseTier(o.fs.opt.AccessTier),
-		HTTPHeaders: httpHeaders,
+		HTTPHeaders: &ui.httpHeaders,
 	}
 
-	// Don't retry, return a retry error instead
-	return o.fs.pacer.CallNoRetry(func() (bool, error) {
-		_, err = blb.Upload(ctx, rs, &options)
+	return o.fs.pacer.Call(func() (bool, error) {
+		// rewind the reader on retry
+		_, err = rs.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+		_, err = ui.blb.Upload(ctx, rs, &options)
 		return o.fs.shouldRetry(ctx, err)
 	})
 }
 
+// Info needed for an upload
+type uploadInfo struct {
+	blb         *blockblob.Client
+	httpHeaders blob.HTTPHeaders
+	isDirMarker bool
+}
+
+// Prepare the object for upload
+func (o *Object) prepareUpload(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (ui uploadInfo, err error) {
+	container, containerPath := o.split()
+	if container == "" || containerPath == "" {
+		return ui, fmt.Errorf("can't upload to root - need a container")
+	}
+	// Create parent dir/bucket if not saving directory marker
+	metadataMu.Lock()
+	_, ui.isDirMarker = o.meta[dirMetaKey]
+	metadataMu.Unlock()
+	if !ui.isDirMarker {
+		err = o.fs.mkdirParent(ctx, o.remote)
+		if err != nil {
+			return ui, err
+		}
+	}
+
+	// Update Mod time
+	o.updateMetadataWithModTime(src.ModTime(ctx))
+	if err != nil {
+		return ui, err
+	}
+
+	// Create the HTTP headers for the upload
+	ui.httpHeaders = blob.HTTPHeaders{
+		BlobContentType: pString(fs.MimeType(ctx, src)),
+	}
+
+	// Compute the Content-MD5 of the file. As we stream all uploads it
+	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
+	if !o.fs.opt.DisableCheckSum {
+		if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
+			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+			if err == nil {
+				ui.httpHeaders.BlobContentMD5 = sourceMD5bytes
+			} else {
+				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+			}
+		}
+	}
+
+	// Apply upload options (also allows one to overwrite content-type)
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			ui.httpHeaders.BlobCacheControl = pString(value)
+		case "content-disposition":
+			ui.httpHeaders.BlobContentDisposition = pString(value)
+		case "content-encoding":
+			ui.httpHeaders.BlobContentEncoding = pString(value)
+		case "content-language":
+			ui.httpHeaders.BlobContentLanguage = pString(value)
+		case "content-type":
+			ui.httpHeaders.BlobContentType = pString(value)
+		}
+	}
+
+	ui.blb = o.fs.getBlockBlobSVC(container, containerPath)
+	return ui, nil
+}
+
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // The new object may have been created if an error is returned
@@ -2221,80 +2344,26 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			return errCantUpdateArchiveTierBlobs
 		}
 	}
-	container, containerPath := o.split()
-	if container == "" || containerPath == "" {
-		return fmt.Errorf("can't upload to root - need a container")
-	}
-	// Create parent dir/bucket if not saving directory marker
-	_, isDirMarker := o.meta[dirMetaKey]
-	if !isDirMarker {
-		err = o.fs.mkdirParent(ctx, o.remote)
-		if err != nil {
-			return err
-		}
-	}
 
-	// Update Mod time
-	fs.Debugf(nil, "o.meta = %+v", o.meta)
-	o.updateMetadataWithModTime(src.ModTime(ctx))
-	if err != nil {
-		return err
-	}
-
-	// Create the HTTP headers for the upload
-	httpHeaders := blob.HTTPHeaders{
-		BlobContentType: pString(fs.MimeType(ctx, src)),
-	}
-
-	// Compute the Content-MD5 of the file. As we stream all uploads it
-	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	if !o.fs.opt.DisableCheckSum {
-		if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
-			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-			if err == nil {
-				httpHeaders.BlobContentMD5 = sourceMD5bytes
-			} else {
-				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
-			}
-		}
-	}
-
-	// Apply upload options (also allows one to overwrite content-type)
-	for _, option := range options {
-		key, value := option.Header()
-		lowerKey := strings.ToLower(key)
-		switch lowerKey {
-		case "":
-			// ignore
-		case "cache-control":
-			httpHeaders.BlobCacheControl = pString(value)
-		case "content-disposition":
-			httpHeaders.BlobContentDisposition = pString(value)
-		case "content-encoding":
-			httpHeaders.BlobContentEncoding = pString(value)
-		case "content-language":
-			httpHeaders.BlobContentLanguage = pString(value)
-		case "content-type":
-			httpHeaders.BlobContentType = pString(value)
-		}
-	}
-
-	blb := o.fs.getBlockBlobSVC(container, containerPath)
 	size := src.Size()
-	multipartUpload := size < 0 || size > o.fs.poolSize
+	multipartUpload := size < 0 || size > int64(o.fs.opt.ChunkSize)
+	var ui uploadInfo
 
-	fs.Debugf(nil, "o.meta = %+v", o.meta)
 	if multipartUpload {
-		err = o.uploadMultipart(ctx, in, size, blb, &httpHeaders)
+		ui, err = o.uploadMultipart(ctx, in, src, options...)
 	} else {
-		err = o.uploadSinglepart(ctx, in, size, blb, &httpHeaders)
+		ui, err = o.prepareUpload(ctx, src, options)
+		if err != nil {
+			return fmt.Errorf("failed to prepare upload: %w", err)
+		}
+		err = o.uploadSinglepart(ctx, in, size, ui)
 	}
 	if err != nil {
 		return err
 	}
 
 	// Refresh metadata on object
-	if !isDirMarker {
+	if !ui.isDirMarker {
 		o.clearMetaData()
 		err = o.readMetaData(ctx)
 		if err != nil {
@@ -2314,9 +2383,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	blb := o.getBlobSVC()
-	//only := blob.DeleteSnapshotsOptionTypeOnly
-	opt := blob.DeleteOptions{
-		//DeleteSnapshots: &only,
+	opt := blob.DeleteOptions{}
+	if o.fs.opt.DeleteSnapshots != "" {
+		action := blob.DeleteSnapshotsOptionType(o.fs.opt.DeleteSnapshots)
+		opt.DeleteSnapshots = &action
 	}
 	return o.fs.pacer.Call(func() (bool, error) {
 		_, err := blb.Delete(ctx, &opt)
@@ -2383,13 +2453,14 @@ func parseTier(tier string) *blob.AccessTier {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs          = &Fs{}
-	_ fs.Copier      = &Fs{}
-	_ fs.PutStreamer = &Fs{}
-	_ fs.Purger      = &Fs{}
-	_ fs.ListRer     = &Fs{}
-	_ fs.Object      = &Object{}
-	_ fs.MimeTyper   = &Object{}
-	_ fs.GetTierer   = &Object{}
-	_ fs.SetTierer   = &Object{}
+	_ fs.Fs              = &Fs{}
+	_ fs.Copier          = &Fs{}
+	_ fs.PutStreamer     = &Fs{}
+	_ fs.Purger          = &Fs{}
+	_ fs.ListRer         = &Fs{}
+	_ fs.OpenChunkWriter = &Fs{}
+	_ fs.Object          = &Object{}
+	_ fs.MimeTyper       = &Object{}
+	_ fs.GetTierer       = &Object{}
+	_ fs.SetTierer       = &Object{}
 )

backend/azureblob/azureblob_internal_test.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
 
 package azureblob
 
@@ -17,20 +16,3 @@ func (f *Fs) InternalTest(t *testing.T) {
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
-
-func TestIncrement(t *testing.T) {
-	for _, test := range []struct {
-		in   []byte
-		want []byte
-	}{
-		{[]byte{0, 0, 0, 0}, []byte{1, 0, 0, 0}},
-		{[]byte{0xFE, 0, 0, 0}, []byte{0xFF, 0, 0, 0}},
-		{[]byte{0xFF, 0, 0, 0}, []byte{0, 1, 0, 0}},
-		{[]byte{0, 1, 0, 0}, []byte{1, 1, 0, 0}},
-		{[]byte{0xFF, 0xFF, 0xFF, 0xFE}, []byte{0, 0, 0, 0xFF}},
-		{[]byte{0xFF, 0xFF, 0xFF, 0xFF}, []byte{0, 0, 0, 0}},
-	} {
-		increment(test.in)
-		assert.Equal(t, test.want, test.in)
-	}
-}

backend/azureblob/azureblob_test.go

@@ -1,7 +1,6 @@
 // Test AzureBlob filesystem interface
 
 //go:build !plan9 && !solaris && !js
-// +build !plan9,!solaris,!js
 
 package azureblob
 
@@ -19,7 +18,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:  "TestAzureBlob:",
 		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool"},
+		TiersToTest: []string{"Hot", "Cool", "Cold"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
 			MinChunkSize: defaultChunkSize,
 		},
@@ -31,11 +30,11 @@ func TestIntegration2(t *testing.T) {
 	if *fstest.RemoteName != "" {
 		t.Skip("Skipping as -remote set")
 	}
-	name := "TestAzureBlob:"
+	name := "TestAzureBlob"
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:  name,
+		RemoteName:  name + ":",
 		NilObject:   (*Object)(nil),
-		TiersToTest: []string{"Hot", "Cool"},
+		TiersToTest: []string{"Hot", "Cool", "Cold"},
 		ChunkedUpload: fstests.ChunkedUploadConfig{
 			MinChunkSize: defaultChunkSize,
 		},
@@ -62,6 +61,7 @@ func TestValidateAccessTier(t *testing.T) {
 		"HOT":     {"HOT", true},
 		"Hot":     {"Hot", true},
 		"cool":    {"cool", true},
+		"cold":    {"cold", true},
 		"archive": {"archive", true},
 		"empty":   {"", false},
 		"unknown": {"unknown", false},

backend/azureblob/azureblob_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || solaris || js
-// +build plan9 solaris js
 
+// Package azureblob provides an interface to the Microsoft Azure blob object storage system
 package azureblob

backend/azurefiles/azurefiles_internal_test.go

@@ -0,0 +1,69 @@
+//go:build !plan9 && !js
+
+package azurefiles
+
+import (
+	"context"
+	"math/rand"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/stretchr/testify/assert"
+)
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Authentication", f.InternalTestAuth)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)
+
+func (f *Fs) InternalTestAuth(t *testing.T) {
+	t.Skip("skipping since this requires authentication credentials which are not part of repo")
+	shareName := "test-rclone-oct-2023"
+	testCases := []struct {
+		name    string
+		options *Options
+	}{
+		{
+			name: "ConnectionString",
+			options: &Options{
+				ShareName:        shareName,
+				ConnectionString: "",
+			},
+		},
+		{
+			name: "AccountAndKey",
+			options: &Options{
+				ShareName: shareName,
+				Account:   "",
+				Key:       "",
+			}},
+		{
+			name: "SASUrl",
+			options: &Options{
+				ShareName: shareName,
+				SASURL:    "",
+			}},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			fs, err := newFsFromOptions(context.TODO(), "TestAzureFiles", "", tc.options)
+			assert.NoError(t, err)
+			dirName := randomString(10)
+			assert.NoError(t, fs.Mkdir(context.TODO(), dirName))
+		})
+	}
+}
+
+const chars = "abcdefghijklmnopqrstuvwzyxABCDEFGHIJKLMNOPQRSTUVWZYX"
+
+func randomString(charCount int) string {
+	strBldr := strings.Builder{}
+	for i := 0; i < charCount; i++ {
+		randPos := rand.Int63n(52)
+		strBldr.WriteByte(chars[randPos])
+	}
+	return strBldr.String()
+}

backend/azurefiles/azurefiles_test.go

@@ -0,0 +1,17 @@
+//go:build !plan9 && !js
+
+package azurefiles
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+func TestIntegration(t *testing.T) {
+	var objPtr *Object
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestAzureFiles:",
+		NilObject:  objPtr,
+	})
+}

backend/azurefiles/azurefiles_unsupported.go

@@ -0,0 +1,7 @@
+// Build for azurefiles for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+//go:build plan9 || js
+
+// Package azurefiles provides an interface to Microsoft Azure Files
+package azurefiles

backend/b2/api/types.go

@@ -33,10 +33,18 @@ var _ fserrors.Fataler = (*Error)(nil)
 
 // Bucket describes a B2 bucket
 type Bucket struct {
-	ID        string `json:"bucketId"`
-	AccountID string `json:"accountId"`
-	Name      string `json:"bucketName"`
-	Type      string `json:"bucketType"`
+	ID             string          `json:"bucketId"`
+	AccountID      string          `json:"accountId"`
+	Name           string          `json:"bucketName"`
+	Type           string          `json:"bucketType"`
+	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
+}
+
+// LifecycleRule is a single lifecycle rule
+type LifecycleRule struct {
+	DaysFromHidingToDeleting  *int   `json:"daysFromHidingToDeleting"`
+	DaysFromUploadingToHiding *int   `json:"daysFromUploadingToHiding"`
+	FileNamePrefix            string `json:"fileNamePrefix"`
 }
 
 // Timestamp is a UTC time when this file was uploaded. It is a base
@@ -206,9 +214,10 @@ type FileInfo struct {
 
 // CreateBucketRequest is used to create a bucket
 type CreateBucketRequest struct {
-	AccountID string `json:"accountId"`
-	Name      string `json:"bucketName"`
-	Type      string `json:"bucketType"`
+	AccountID      string          `json:"accountId"`
+	Name           string          `json:"bucketName"`
+	Type           string          `json:"bucketType"`
+	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
 }
 
 // DeleteBucketRequest is used to create a bucket
@@ -331,3 +340,11 @@ type CopyPartRequest struct {
 	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
 	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
 }
+
+// UpdateBucketRequest describes a request to modify a B2 bucket
+type UpdateBucketRequest struct {
+	ID             string          `json:"bucketId"`
+	AccountID      string          `json:"accountId"`
+	Type           string          `json:"bucketType,omitempty"`
+	LifecycleRules []LifecycleRule `json:"lifecycleRules,omitempty"`
+}

backend/b2/api/types_test.go

@@ -42,11 +42,11 @@ func TestTimestampIsZero(t *testing.T) {
 }
 
 func TestTimestampEqual(t *testing.T) {
-	assert.False(t, emptyT.Equal(emptyT))
+	assert.False(t, emptyT.Equal(emptyT)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
 	assert.False(t, t0.Equal(emptyT))
 	assert.False(t, emptyT.Equal(t0))
 	assert.False(t, t0.Equal(t1))
 	assert.False(t, t1.Equal(t0))
-	assert.True(t, t0.Equal(t0))
-	assert.True(t, t1.Equal(t1))
+	assert.True(t, t0.Equal(t0)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
+	assert.True(t, t1.Equal(t1)) //nolint:gocritic // Don't include gocritic when running golangci-lint to avoid dupArg: suspicious method call with the same argument and receiver
 }

backend/b2/b2.go

@@ -9,6 +9,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/sha1"
+	"encoding/json"
 	"errors"
 	"fmt"
 	gohash "hash"
@@ -32,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/multipart"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
@@ -57,9 +59,8 @@ const (
 	minChunkSize        = 5 * fs.Mebi
 	defaultChunkSize    = 96 * fs.Mebi
 	defaultUploadCutoff = 200 * fs.Mebi
-	largeFileCopyCutoff = 4 * fs.Gibi              // 5E9 is the max
-	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
-	memoryPoolUseMmap   = false
+	largeFileCopyCutoff = 4 * fs.Gibi // 5E9 is the max
+	defaultMaxAge       = 24 * time.Hour
 )
 
 // Globals
@@ -74,6 +75,7 @@ func init() {
 		Name:        "b2",
 		Description: "Backblaze B2",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:      "account",
 			Help:      "Account ID or Application Key ID.",
@@ -100,7 +102,7 @@ below will cause b2 to return specific errors:
   * "force_cap_exceeded"
 
 These will be set in the "X-Bz-Test-Mode" header which is documented
-in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration_checklist.html).`,
+in the [b2 integrations checklist](https://www.backblaze.com/docs/cloud-storage-integration-checklist).`,
 			Default:  "",
 			Hide:     fs.OptionHideConfigurator,
 			Advanced: true,
@@ -149,6 +151,18 @@ might a maximum of "--transfers" chunks in progress at once.
 5,000,000 Bytes is the minimum size.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
+		}, {
+			Name: "upload_concurrency",
+			Help: `Concurrency for multipart uploads.
+
+This is the number of chunks of the same file that are uploaded
+concurrently.
+
+Note that chunks are stored in memory and there may be up to
+"--transfers" * "--b2-upload-concurrency" chunks stored at once
+in memory.`,
+			Default:  4,
+			Advanced: true,
 		}, {
 			Name: "disable_checksum",
 			Help: `Disable checksums for large (> upload cutoff) files.
@@ -180,29 +194,57 @@ Example:
 			Advanced: true,
 		}, {
 			Name: "download_auth_duration",
-			Help: `Time before the authorization token will expire in s or suffix ms|s|m|h|d.
+			Help: `Time before the public link authorization token will expire in s or suffix ms|s|m|h|d.
+
+This is used in combination with "rclone link" for making files
+accessible to the public and sets the duration before the download
+authorization token will expire.
 
-The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
 		}, {
 			Name:     "memory_pool_flush_time",
-			Default:  memoryPoolFlushTime,
+			Default:  fs.Duration(time.Minute),
 			Advanced: true,
-			Help: `How often internal memory buffer pools will be flushed.
-Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
-This option controls how often unused buffers will be removed from the pool.`,
+			Hide:     fs.OptionHideBoth,
+			Help:     `How often internal memory buffer pools will be flushed. (no longer used)`,
 		}, {
 			Name:     "memory_pool_use_mmap",
-			Default:  memoryPoolUseMmap,
+			Default:  false,
+			Advanced: true,
+			Hide:     fs.OptionHideBoth,
+			Help:     `Whether to use mmap buffers in internal memory pool. (no longer used)`,
+		}, {
+			Name: "lifecycle",
+			Help: `Set the number of days deleted files should be kept when creating a bucket.
+
+On bucket creation, this parameter is used to create a lifecycle rule
+for the entire bucket.
+
+If lifecycle is 0 (the default) it does not create a lifecycle rule so
+the default B2 behaviour applies. This is to create versions of files
+on delete and overwrite and to keep them indefinitely.
+
+If lifecycle is >0 then it creates a single rule setting the number of
+days before a file that is deleted or overwritten is deleted
+permanently. This is known as daysFromHidingToDeleting in the b2 docs.
+
+The minimum value for this parameter is 1 day.
+
+You can also enable hard_delete in the config also which will mean
+deletions won't cause versions but overwrites will still cause
+versions to be made.
+
+See: [rclone backend lifecycle](#lifecycle) for setting lifecycles after bucket creation.
+`,
+			Default:  0,
 			Advanced: true,
-			Help:     `Whether to use mmap buffers in internal memory pool.`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			// See: https://www.backblaze.com/b2/docs/files.html
+			// See: https://www.backblaze.com/docs/cloud-storage-files
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// FIXME: allow /, but not leading, trailing or double
 			Default: (encoder.Display |
@@ -224,11 +266,11 @@ type Options struct {
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
 	ChunkSize                     fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency             int                  `config:"upload_concurrency"`
 	DisableCheckSum               bool                 `config:"disable_checksum"`
 	DownloadURL                   string               `config:"download_url"`
 	DownloadAuthorizationDuration fs.Duration          `config:"download_auth_duration"`
-	MemoryPoolFlushTime           fs.Duration          `config:"memory_pool_flush_time"`
-	MemoryPoolUseMmap             bool                 `config:"memory_pool_use_mmap"`
+	Lifecycle                     int                  `config:"lifecycle"`
 	Enc                           encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -253,18 +295,18 @@ type Fs struct {
 	authMu          sync.Mutex                             // lock for authorizing the account
 	pacer           *fs.Pacer                              // To pace and retry the API calls
 	uploadToken     *pacer.TokenDispenser                  // control concurrency
-	pool            *pool.Pool                             // memory pool
 }
 
 // Object describes a b2 object
 type Object struct {
-	fs       *Fs       // what this object is part of
-	remote   string    // The remote path
-	id       string    // b2 id of the file
-	modTime  time.Time // The modified time of the object if known
-	sha1     string    // SHA-1 hash if known
-	size     int64     // Size of the object
-	mimeType string    // Content-Type of the object
+	fs       *Fs               // what this object is part of
+	remote   string            // The remote path
+	id       string            // b2 id of the file
+	modTime  time.Time         // The modified time of the object if known
+	sha1     string            // SHA-1 hash if known
+	size     int64             // Size of the object
+	mimeType string            // Content-Type of the object
+	meta     map[string]string // The object metadata if known - may be nil - with lower case keys
 }
 
 // ------------------------------------------------------------
@@ -322,7 +364,7 @@ var retryErrorCodes = []int{
 	504, // Gateway Time-out
 }
 
-// shouldRetryNoAuth returns a boolean as to whether this resp and err
+// shouldRetryNoReauth returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func (f *Fs) shouldRetryNoReauth(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if fserrors.ContextError(ctx, &err) {
@@ -363,11 +405,18 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (b
 
 // errorHandler parses a non 2xx error response into an error
 func errorHandler(resp *http.Response) error {
-	// Decode error response
-	errResponse := new(api.Error)
-	err := rest.DecodeJSON(resp, &errResponse)
+	body, err := rest.ReadBody(resp)
 	if err != nil {
-		fs.Debugf(nil, "Couldn't decode error response: %v", err)
+		fs.Errorf(nil, "Couldn't read error out of body: %v", err)
+		body = nil
+	}
+	// Decode error response if there was one - they can be blank
+	errResponse := new(api.Error)
+	if len(body) > 0 {
+		err = json.Unmarshal(body, errResponse)
+		if err != nil {
+			fs.Errorf(nil, "Couldn't decode error response: %v", err)
+		}
 	}
 	if errResponse.Code == "" {
 		errResponse.Code = "unknown"
@@ -411,6 +460,14 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
+func (f *Fs) setCopyCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadChunkSize(cs)
+	if err == nil {
+		old, f.opt.CopyCutoff = f.opt.CopyCutoff, cs
+	}
+	return
+}
+
 // setRoot changes the root of the Fs
 func (f *Fs) setRoot(root string) {
 	f.root = parsePath(root)
@@ -458,19 +515,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		uploads:     make(map[string][]*api.GetUploadURLResponse),
 		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
-		pool: pool.New(
-			time.Duration(opt.MemoryPoolFlushTime),
-			int(opt.ChunkSize),
-			ci.Transfers,
-			opt.MemoryPoolUseMmap,
-		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
-		ReadMimeType:      true,
-		WriteMimeType:     true,
-		BucketBased:       true,
-		BucketBasedRootOK: true,
+		ReadMimeType:          true,
+		WriteMimeType:         true,
+		BucketBased:           true,
+		BucketBasedRootOK:     true,
+		ChunkWriterDoesntSeek: true,
 	}).Fill(ctx, f)
 	// Set the test flag if required
 	if opt.TestMode != "" {
@@ -597,23 +649,24 @@ func (f *Fs) clearUploadURL(bucketID string) {
 	f.uploadMu.Unlock()
 }
 
-// getBuf gets a buffer of f.opt.ChunkSize and an upload token
+// getRW gets a RW buffer and an upload token
 //
 // If noBuf is set then it just gets an upload token
-func (f *Fs) getBuf(noBuf bool) (buf []byte) {
+func (f *Fs) getRW(noBuf bool) (rw *pool.RW) {
 	f.uploadToken.Get()
 	if !noBuf {
-		buf = f.pool.Get()
+		rw = multipart.NewRW()
 	}
-	return buf
+	return rw
 }
 
-// putBuf returns a buffer to the memory pool and an upload token
+// putRW returns a RW buffer to the memory pool and returns an upload
+// token
 //
-// If noBuf is set then it just returns the upload token
-func (f *Fs) putBuf(buf []byte, noBuf bool) {
-	if !noBuf {
-		f.pool.Put(buf)
+// If buf is nil then it just returns the upload token
+func (f *Fs) putRW(rw *pool.RW) {
+	if rw != nil {
+		_ = rw.Close()
 	}
 	f.uploadToken.Put()
 }
@@ -820,7 +873,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
 
 // listBuckets returns all the buckets to out
 func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
-	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, "", func(bucket *api.Bucket) error {
 		d := fs.NewDir(bucket.Name, time.Time{})
 		entries = append(entries, d)
 		return nil
@@ -913,11 +966,14 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 type listBucketFn func(*api.Bucket) error
 
 // listBucketsToFn lists the buckets to the function supplied
-func (f *Fs) listBucketsToFn(ctx context.Context, fn listBucketFn) error {
+func (f *Fs) listBucketsToFn(ctx context.Context, bucketName string, fn listBucketFn) error {
 	var account = api.ListBucketsRequest{
 		AccountID: f.info.AccountID,
 		BucketID:  f.info.Allowed.BucketID,
 	}
+	if bucketName != "" && account.BucketID == "" {
+		account.BucketName = f.opt.Enc.FromStandardName(bucketName)
+	}
 
 	var response api.ListBucketsResponse
 	opts := rest.Opts{
@@ -963,7 +1019,7 @@ func (f *Fs) getbucketType(ctx context.Context, bucket string) (bucketType strin
 	if bucketType != "" {
 		return bucketType, nil
 	}
-	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, bucket, func(bucket *api.Bucket) error {
 		// listBucketsToFn reads bucket Types
 		return nil
 	})
@@ -998,7 +1054,7 @@ func (f *Fs) getBucketID(ctx context.Context, bucket string) (bucketID string, e
 	if bucketID != "" {
 		return bucketID, nil
 	}
-	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, bucket, func(bucket *api.Bucket) error {
 		// listBucketsToFn sets IDs
 		return nil
 	})
@@ -1062,6 +1118,11 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 			Name:      f.opt.Enc.FromStandardName(bucket),
 			Type:      "allPrivate",
 		}
+		if f.opt.Lifecycle > 0 {
+			request.LifecycleRules = []api.LifecycleRule{{
+				DaysFromHidingToDeleting: &f.opt.Lifecycle,
+			}}
+		}
 		var response api.Bucket
 		err := f.pacer.Call(func() (bool, error) {
 			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
@@ -1189,7 +1250,7 @@ func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
 // if oldOnly is true then it deletes only non current files.
 //
 // Implemented here so we can make sure we delete old versions.
-func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
+func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden bool, deleteUnfinished bool, maxAge time.Duration) error {
 	bucket, directory := f.split(dir)
 	if bucket == "" {
 		return errors.New("can't purge from root")
@@ -1207,7 +1268,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 		}
 	}
 	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
-		return time.Since(time.Time(timestamp)).Hours() > 24
+		return time.Since(time.Time(timestamp)) > maxAge
 	}
 
 	// Delete Config.Transfers in parallel
@@ -1230,6 +1291,21 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			}
 		}()
 	}
+	if oldOnly {
+		if deleteHidden && deleteUnfinished {
+			fs.Infof(f, "cleaning bucket %q of all hidden files, and pending multipart uploads older than %v", bucket, maxAge)
+		} else if deleteHidden {
+			fs.Infof(f, "cleaning bucket %q of all hidden files", bucket)
+		} else if deleteUnfinished {
+			fs.Infof(f, "cleaning bucket %q of pending multipart uploads older than %v", bucket, maxAge)
+		} else {
+			fs.Errorf(f, "cleaning bucket %q of nothing. This should never happen!", bucket)
+			return nil
+		}
+	} else {
+		fs.Infof(f, "cleaning bucket %q of all files", bucket)
+	}
+
 	last := ""
 	checkErr(f.list(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "", true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {
 		if !isDirectory {
@@ -1240,14 +1316,14 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 			tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
 			if oldOnly && last != remote {
 				// Check current version of the file
-				if object.Action == "hide" {
+				if deleteHidden && object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
 					toBeDeleted <- object
-				} else if object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
+				} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
 					toBeDeleted <- object
 				} else {
-					fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
+					fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))
 				}
 			} else {
 				fs.Debugf(remote, "Deleting (id %q)", object.ID)
@@ -1269,12 +1345,17 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 
 // Purge deletes all the files and directories including the old versions.
 func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purge(ctx, dir, false)
+	return f.purge(ctx, dir, false, false, false, defaultMaxAge)
 }
 
-// CleanUp deletes all the hidden files.
+// CleanUp deletes all hidden files and pending multipart uploads older than 24 hours.
 func (f *Fs) CleanUp(ctx context.Context) error {
-	return f.purge(ctx, "", true)
+	return f.purge(ctx, "", true, true, true, defaultMaxAge)
+}
+
+// cleanUp deletes all hidden files and/or pending multipart uploads older than the specified age.
+func (f *Fs) cleanUp(ctx context.Context, deleteHidden bool, deleteUnfinished bool, maxAge time.Duration) (err error) {
+	return f.purge(ctx, "", true, deleteHidden, deleteUnfinished, maxAge)
 }
 
 // copy does a server-side copy from dstObj <- srcObj
@@ -1282,7 +1363,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 // If newInfo is nil then the metadata will be copied otherwise it
 // will be replaced with newInfo
 func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *api.File) (err error) {
-	if srcObj.size >= int64(f.opt.CopyCutoff) {
+	if srcObj.size > int64(f.opt.CopyCutoff) {
 		if newInfo == nil {
 			newInfo, err = srcObj.getMetaData(ctx)
 			if err != nil {
@@ -1293,7 +1374,11 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
 		if err != nil {
 			return err
 		}
-		return up.Upload(ctx)
+		err = up.Copy(ctx)
+		if err != nil {
+			return err
+		}
+		return dstObj.decodeMetaDataFileInfo(up.info)
 	}
 
 	dstBucket, dstPath := dstObj.split()
@@ -1422,7 +1507,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	if err != nil {
 		return "", err
 	}
-	absPath := "/" + bucketPath
+	absPath := "/" + urlEncode(bucketPath)
 	link = RootURL + "/file/" + urlEncode(bucket) + absPath
 	bucketType, err := f.getbucketType(ctx, bucket)
 	if err != nil {
@@ -1482,7 +1567,7 @@ func (o *Object) Size() int64 {
 //
 // Make sure it is lower case.
 //
-// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
+// Remove unverified prefix - see https://www.backblaze.com/docs/cloud-storage-upload-files-with-the-native-api
 // Some tools (e.g. Cyberduck) use this
 func cleanSHA1(sha1 string) string {
 	const unverified = "unverified:"
@@ -1509,7 +1594,14 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	o.size = Size
 	// Use the UploadTimestamp if can't get file info
 	o.modTime = time.Time(UploadTimestamp)
-	return o.parseTimeString(Info[timeKey])
+	err = o.parseTimeString(Info[timeKey])
+	if err != nil {
+		return err
+	}
+	// For now, just set "mtime" in metadata
+	o.meta = make(map[string]string, 1)
+	o.meta["mtime"] = o.modTime.Format(time.RFC3339Nano)
+	return nil
 }
 
 // decodeMetaData sets the metadata in the object from an api.File
@@ -1611,6 +1703,16 @@ func timeString(modTime time.Time) string {
 	return strconv.FormatInt(modTime.UnixNano()/1e6, 10)
 }
 
+// parseTimeStringHelper converts a decimal string number of milliseconds
+// elapsed since January 1, 1970 UTC into a time.Time
+func parseTimeStringHelper(timeString string) (time.Time, error) {
+	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
+	if err != nil {
+		return time.Time{}, err
+	}
+	return time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC(), nil
+}
+
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.
@@ -1618,12 +1720,12 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 	if timeString == "" {
 		return nil
 	}
-	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
+	modTime, err := parseTimeStringHelper(timeString)
 	if err != nil {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
 		return nil
 	}
-	o.modTime = time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC()
+	o.modTime = modTime
 	return nil
 }
 
@@ -1700,14 +1802,14 @@ func (file *openFile) Close() (err error) {
 
 	// Check to see we read the correct number of bytes
 	if file.o.Size() != file.bytes {
-		return fmt.Errorf("object corrupted on transfer - length mismatch (want %d got %d)", file.o.Size(), file.bytes)
+		return fmt.Errorf("corrupted on transfer: lengths differ want %d vs got %d", file.o.Size(), file.bytes)
 	}
 
 	// Check the SHA1
 	receivedSHA1 := file.o.sha1
 	calculatedSHA1 := fmt.Sprintf("%x", file.hash.Sum(nil))
 	if receivedSHA1 != "" && receivedSHA1 != calculatedSHA1 {
-		return fmt.Errorf("object corrupted on transfer - SHA1 mismatch (want %q got %q)", receivedSHA1, calculatedSHA1)
+		return fmt.Errorf("corrupted on transfer: SHA1 hashes differ want %q vs got %q", receivedSHA1, calculatedSHA1)
 	}
 
 	return nil
@@ -1777,6 +1879,14 @@ func (o *Object) getOrHead(ctx context.Context, method string, options []fs.Open
 		ContentType:     resp.Header.Get("Content-Type"),
 		Info:            Info,
 	}
+
+	// Embryonic metadata support - just mtime
+	o.meta = make(map[string]string, 1)
+	modTime, err := parseTimeStringHelper(info.Info[timeKey])
+	if err == nil {
+		o.meta["mtime"] = modTime.Format(time.RFC3339Nano)
+	}
+
 	// When reading files from B2 via cloudflare using
 	// --b2-download-url cloudflare strips the Content-Length
 	// headers (presumably so it can inject stuff) so use the old
@@ -1861,11 +1971,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	if size == -1 {
+	if size < 0 {
 		// Check if the file is large enough for a chunked upload (needs to be at least two chunks)
-		buf := o.fs.getBuf(false)
+		rw := o.fs.getRW(false)
 
-		n, err := io.ReadFull(in, buf)
+		n, err := io.CopyN(rw, in, int64(o.fs.opt.ChunkSize))
 		if err == nil {
 			bufReader := bufio.NewReader(in)
 			in = bufReader
@@ -1874,31 +1984,42 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		if err == nil {
 			fs.Debugf(o, "File is big enough for chunked streaming")
-			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
+			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil, options...)
 			if err != nil {
-				o.fs.putBuf(buf, false)
+				o.fs.putRW(rw)
 				return err
 			}
 			// NB Stream returns the buffer and token
-			return up.Stream(ctx, buf)
-		} else if err == io.EOF || err == io.ErrUnexpectedEOF {
+			err = up.Stream(ctx, rw)
+			if err != nil {
+				return err
+			}
+			return o.decodeMetaDataFileInfo(up.info)
+		} else if err == io.EOF {
 			fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
-			defer o.fs.putBuf(buf, false)
-			size = int64(n)
-			in = bytes.NewReader(buf[:n])
+			defer o.fs.putRW(rw)
+			size = n
+			in = rw
 		} else {
-			o.fs.putBuf(buf, false)
+			o.fs.putRW(rw)
 			return err
 		}
 	} else if size > int64(o.fs.opt.UploadCutoff) {
-		up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
+		chunkWriter, err := multipart.UploadMultipart(ctx, src, in, multipart.UploadMultipartOptions{
+			Open:        o.fs,
+			OpenOptions: options,
+		})
 		if err != nil {
 			return err
 		}
-		return up.Upload(ctx)
+		up := chunkWriter.(*largeUpload)
+		return o.decodeMetaDataFileInfo(up.info)
 	}
 
-	modTime := src.ModTime(ctx)
+	modTime, err := o.getModTime(ctx, src, options)
+	if err != nil {
+		return err
+	}
 
 	calculatedSha1, _ := src.Hash(ctx, hash.SHA1)
 	if calculatedSha1 == "" {
@@ -2003,6 +2124,71 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.decodeMetaDataFileInfo(&response)
 }
 
+// Get modTime from the source; if --metadata is set, fetch the src metadata and get it from there.
+// When metadata support is added to b2, this method will need a more generic name
+func (o *Object) getModTime(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption) (time.Time, error) {
+	modTime := src.ModTime(ctx)
+
+	// Fetch metadata if --metadata is in use
+	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
+	if err != nil {
+		return time.Time{}, fmt.Errorf("failed to read metadata from source object: %w", err)
+	}
+	// merge metadata into request and user metadata
+	for k, v := range meta {
+		k = strings.ToLower(k)
+		// For now, the only metadata we're concerned with is "mtime"
+		switch k {
+		case "mtime":
+			// mtime in meta overrides source ModTime
+			metaModTime, err := time.Parse(time.RFC3339Nano, v)
+			if err != nil {
+				fs.Debugf(o, "failed to parse metadata %s: %q: %v", k, v, err)
+			} else {
+				modTime = metaModTime
+			}
+		default:
+			// Do nothing for now
+		}
+	}
+	return modTime, nil
+}
+
+// OpenChunkWriter returns the chunk size and a ChunkWriter
+//
+// Pass in the remote and the src object
+// You can also use options to hint at the desired chunk size
+func (f *Fs) OpenChunkWriter(ctx context.Context, remote string, src fs.ObjectInfo, options ...fs.OpenOption) (info fs.ChunkWriterInfo, writer fs.ChunkWriter, err error) {
+	// FIXME what if file is smaller than 1 chunk?
+	if f.opt.Versions {
+		return info, nil, errNotWithVersions
+	}
+	if f.opt.VersionAt.IsSet() {
+		return info, nil, errNotWithVersionAt
+	}
+	//size := src.Size()
+
+	// Temporary Object under construction
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+
+	bucket, _ := o.split()
+	err = f.makeBucket(ctx, bucket)
+	if err != nil {
+		return info, nil, err
+	}
+
+	info = fs.ChunkWriterInfo{
+		ChunkSize:   int64(f.opt.ChunkSize),
+		Concurrency: o.fs.opt.UploadConcurrency,
+		//LeavePartsOnError: o.fs.opt.LeavePartsOnError,
+	}
+	up, err := f.newLargeUpload(ctx, o, nil, src, f.opt.ChunkSize, false, nil, options...)
+	return info, up, err
+}
+
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
 	bucket, bucketPath := o.split()
@@ -2028,16 +2214,201 @@ func (o *Object) ID() string {
 	return o.id
 }
 
+var lifecycleHelp = fs.CommandHelp{
+	Name:  "lifecycle",
+	Short: "Read or set the lifecycle for a bucket",
+	Long: `This command can be used to read or set the lifecycle for a bucket.
+
+Usage Examples:
+
+To show the current lifecycle rules:
+
+    rclone backend lifecycle b2:bucket
+
+This will dump something like this showing the lifecycle rules.
+
+    [
+        {
+            "daysFromHidingToDeleting": 1,
+            "daysFromUploadingToHiding": null,
+            "fileNamePrefix": ""
+        }
+    ]
+
+If there are no lifecycle rules (the default) then it will just return [].
+
+To reset the current lifecycle rules:
+
+    rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=30
+    rclone backend lifecycle b2:bucket -o daysFromUploadingToHiding=5 -o daysFromHidingToDeleting=1
+
+This will run and then print the new lifecycle rules as above.
+
+Rclone only lets you set lifecycles for the whole bucket with the
+fileNamePrefix = "".
+
+You can't disable versioning with B2. The best you can do is to set
+the daysFromHidingToDeleting to 1 day. You can enable hard_delete in
+the config also which will mean deletions won't cause versions but
+overwrites will still cause versions to be made.
+
+    rclone backend lifecycle b2:bucket -o daysFromHidingToDeleting=1
+
+See: https://www.backblaze.com/docs/cloud-storage-lifecycle-rules
+`,
+	Opts: map[string]string{
+		"daysFromHidingToDeleting":  "After a file has been hidden for this many days it is deleted. 0 is off.",
+		"daysFromUploadingToHiding": "This many days after uploading a file is hidden",
+	},
+}
+
+func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	var newRule api.LifecycleRule
+	if daysStr := opt["daysFromHidingToDeleting"]; daysStr != "" {
+		days, err := strconv.Atoi(daysStr)
+		if err != nil {
+			return nil, fmt.Errorf("bad daysFromHidingToDeleting: %w", err)
+		}
+		newRule.DaysFromHidingToDeleting = &days
+	}
+	if daysStr := opt["daysFromUploadingToHiding"]; daysStr != "" {
+		days, err := strconv.Atoi(daysStr)
+		if err != nil {
+			return nil, fmt.Errorf("bad daysFromUploadingToHiding: %w", err)
+		}
+		newRule.DaysFromUploadingToHiding = &days
+	}
+	bucketName, _ := f.split("")
+	if bucketName == "" {
+		return nil, errors.New("bucket required")
+
+	}
+
+	var bucket *api.Bucket
+	if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil {
+		bucketID, err := f.getBucketID(ctx, bucketName)
+		if err != nil {
+			return nil, err
+		}
+		opts := rest.Opts{
+			Method: "POST",
+			Path:   "/b2_update_bucket",
+		}
+		var request = api.UpdateBucketRequest{
+			ID:             bucketID,
+			AccountID:      f.info.AccountID,
+			LifecycleRules: []api.LifecycleRule{newRule},
+		}
+		var response api.Bucket
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+			return f.shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return nil, err
+		}
+		bucket = &response
+	} else {
+		err = f.listBucketsToFn(ctx, bucketName, func(b *api.Bucket) error {
+			bucket = b
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	if bucket == nil {
+		return nil, fs.ErrorDirNotFound
+	}
+	return bucket.LifecycleRules, nil
+}
+
+var cleanupHelp = fs.CommandHelp{
+	Name:  "cleanup",
+	Short: "Remove unfinished large file uploads.",
+	Long: `This command removes unfinished large file uploads of age greater than
+max-age, which defaults to 24 hours.
+
+Note that you can use --interactive/-i or --dry-run with this command to see what
+it would do.
+
+    rclone backend cleanup b2:bucket/path/to/object
+    rclone backend cleanup -o max-age=7w b2:bucket/path/to/object
+
+Durations are parsed as per the rest of rclone, 2h, 7d, 7w etc.
+`,
+	Opts: map[string]string{
+		"max-age": "Max age of upload to delete",
+	},
+}
+
+func (f *Fs) cleanupCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	maxAge := defaultMaxAge
+	if opt["max-age"] != "" {
+		maxAge, err = fs.ParseDuration(opt["max-age"])
+		if err != nil {
+			return nil, fmt.Errorf("bad max-age: %w", err)
+		}
+	}
+	return nil, f.cleanUp(ctx, false, true, maxAge)
+}
+
+var cleanupHiddenHelp = fs.CommandHelp{
+	Name:  "cleanup-hidden",
+	Short: "Remove old versions of files.",
+	Long: `This command removes any old hidden versions of files.
+
+Note that you can use --interactive/-i or --dry-run with this command to see what
+it would do.
+
+    rclone backend cleanup-hidden b2:bucket/path/to/dir
+`,
+}
+
+func (f *Fs) cleanupHiddenCommand(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	return nil, f.cleanUp(ctx, true, false, 0)
+}
+
+var commandHelp = []fs.CommandHelp{
+	lifecycleHelp,
+	cleanupHelp,
+	cleanupHiddenHelp,
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "lifecycle":
+		return f.lifecycleCommand(ctx, name, arg, opt)
+	case "cleanup":
+		return f.cleanupCommand(ctx, name, arg, opt)
+	case "cleanup-hidden":
+		return f.cleanupHiddenCommand(ctx, name, arg, opt)
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs           = &Fs{}
-	_ fs.Purger       = &Fs{}
-	_ fs.Copier       = &Fs{}
-	_ fs.PutStreamer  = &Fs{}
-	_ fs.CleanUpper   = &Fs{}
-	_ fs.ListRer      = &Fs{}
-	_ fs.PublicLinker = &Fs{}
-	_ fs.Object       = &Object{}
-	_ fs.MimeTyper    = &Object{}
-	_ fs.IDer         = &Object{}
+	_ fs.Fs              = &Fs{}
+	_ fs.Purger          = &Fs{}
+	_ fs.Copier          = &Fs{}
+	_ fs.PutStreamer     = &Fs{}
+	_ fs.CleanUpper      = &Fs{}
+	_ fs.ListRer         = &Fs{}
+	_ fs.PublicLinker    = &Fs{}
+	_ fs.OpenChunkWriter = &Fs{}
+	_ fs.Commander       = &Fs{}
+	_ fs.Object          = &Object{}
+	_ fs.MimeTyper       = &Object{}
+	_ fs.IDer            = &Object{}
 )

backend/b2/b2_internal_test.go

@@ -1,14 +1,29 @@
 package b2
 
 import (
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"path"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/rclone/rclone/backend/b2/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/cache"
+	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/lib/version"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 // Test b2 string encoding
-// https://www.backblaze.com/b2/docs/string_encoding.html
+// https://www.backblaze.com/docs/cloud-storage-native-api-string-encoding
 
 var encodeTest = []struct {
 	fullyEncoded     string
@@ -168,3 +183,304 @@ func TestParseTimeString(t *testing.T) {
 	}
 
 }
+
+// Return a map of the headers in the options with keys stripped of the "x-bz-info-" prefix
+func OpenOptionToMetaData(options []fs.OpenOption) map[string]string {
+	var headers = make(map[string]string)
+	for _, option := range options {
+		k, v := option.Header()
+		k = strings.ToLower(k)
+		if strings.HasPrefix(k, headerPrefix) {
+			headers[k[len(headerPrefix):]] = v
+		}
+	}
+
+	return headers
+}
+
+func (f *Fs) internalTestMetadata(t *testing.T, size string, uploadCutoff string, chunkSize string) {
+	what := fmt.Sprintf("Size%s/UploadCutoff%s/ChunkSize%s", size, uploadCutoff, chunkSize)
+	t.Run(what, func(t *testing.T) {
+		ctx := context.Background()
+
+		ss := fs.SizeSuffix(0)
+		err := ss.Set(size)
+		require.NoError(t, err)
+		original := random.String(int(ss))
+
+		contents := fstest.Gz(t, original)
+		mimeType := "text/html"
+
+		if chunkSize != "" {
+			ss := fs.SizeSuffix(0)
+			err := ss.Set(chunkSize)
+			require.NoError(t, err)
+			_, err = f.SetUploadChunkSize(ss)
+			require.NoError(t, err)
+		}
+
+		if uploadCutoff != "" {
+			ss := fs.SizeSuffix(0)
+			err := ss.Set(uploadCutoff)
+			require.NoError(t, err)
+			_, err = f.SetUploadCutoff(ss)
+			require.NoError(t, err)
+		}
+
+		item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499Z"))
+		btime := time.Now()
+		metadata := fs.Metadata{
+			// Just mtime for now - limit to milliseconds since x-bz-info-src_last_modified_millis can't support any
+
+			"mtime": "2009-05-06T04:05:06.499Z",
+		}
+
+		// Need to specify HTTP options with the header prefix since they are passed as-is
+		options := []fs.OpenOption{
+			&fs.HTTPOption{Key: "X-Bz-Info-a", Value: "1"},
+			&fs.HTTPOption{Key: "X-Bz-Info-b", Value: "2"},
+		}
+
+		obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, true, contents, true, mimeType, metadata, options...)
+		defer func() {
+			assert.NoError(t, obj.Remove(ctx))
+		}()
+		o := obj.(*Object)
+		gotMetadata, err := o.getMetaData(ctx)
+		require.NoError(t, err)
+
+		// X-Bz-Info-a & X-Bz-Info-b
+		optMetadata := OpenOptionToMetaData(options)
+		for k, v := range optMetadata {
+			got := gotMetadata.Info[k]
+			assert.Equal(t, v, got, k)
+		}
+
+		// mtime
+		for k, v := range metadata {
+			got := o.meta[k]
+			assert.Equal(t, v, got, k)
+		}
+
+		assert.Equal(t, mimeType, gotMetadata.ContentType, "Content-Type")
+
+		// Modification time from the x-bz-info-src_last_modified_millis header
+		var mtime api.Timestamp
+		err = mtime.UnmarshalJSON([]byte(gotMetadata.Info[timeKey]))
+		if err != nil {
+			fs.Debugf(o, "Bad "+timeHeader+" header: %v", err)
+		}
+		assert.Equal(t, item.ModTime, time.Time(mtime), "Modification time")
+
+		// Upload time
+		gotBtime := time.Time(gotMetadata.UploadTimestamp)
+		dt := gotBtime.Sub(btime)
+		assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("btime more than 1 minute out want %v got %v delta %v", btime, gotBtime, dt))
+
+		t.Run("GzipEncoding", func(t *testing.T) {
+			// Test that the gzipped file we uploaded can be
+			// downloaded
+			checkDownload := func(wantContents string, wantSize int64, wantHash string) {
+				gotContents := fstests.ReadObject(ctx, t, o, -1)
+				assert.Equal(t, wantContents, gotContents)
+				assert.Equal(t, wantSize, o.Size())
+				gotHash, err := o.Hash(ctx, hash.SHA1)
+				require.NoError(t, err)
+				assert.Equal(t, wantHash, gotHash)
+			}
+
+			t.Run("NoDecompress", func(t *testing.T) {
+				checkDownload(contents, int64(len(contents)), sha1Sum(t, contents))
+			})
+		})
+	})
+}
+
+func (f *Fs) InternalTestMetadata(t *testing.T) {
+	// 1 kB regular file
+	f.internalTestMetadata(t, "1kiB", "", "")
+
+	// 10 MiB large file
+	f.internalTestMetadata(t, "10MiB", "6MiB", "6MiB")
+}
+
+func sha1Sum(t *testing.T, s string) string {
+	hash := sha1.Sum([]byte(s))
+	return fmt.Sprintf("%x", hash)
+}
+
+// This is adapted from the s3 equivalent.
+func (f *Fs) InternalTestVersions(t *testing.T) {
+	ctx := context.Background()
+
+	// Small pause to make the LastModified different since AWS
+	// only seems to track them to 1 second granularity
+	time.Sleep(2 * time.Second)
+
+	// Create an object
+	const dirName = "versions"
+	const fileName = dirName + "/" + "test-versions.txt"
+	contents := random.String(100)
+	item := fstest.NewItem(fileName, contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+	obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+	defer func() {
+		assert.NoError(t, obj.Remove(ctx))
+	}()
+	objMetadata, err := obj.(*Object).getMetaData(ctx)
+	require.NoError(t, err)
+
+	// Small pause
+	time.Sleep(2 * time.Second)
+
+	// Remove it
+	assert.NoError(t, obj.Remove(ctx))
+
+	// Small pause to make the LastModified different since AWS only seems to track them to 1 second granularity
+	time.Sleep(2 * time.Second)
+
+	// And create it with different size and contents
+	newContents := random.String(101)
+	newItem := fstest.NewItem(fileName, newContents, fstest.Time("2002-05-06T04:05:06.499999999Z"))
+	newObj := fstests.PutTestContents(ctx, t, f, &newItem, newContents, true)
+	newObjMetadata, err := newObj.(*Object).getMetaData(ctx)
+	require.NoError(t, err)
+
+	t.Run("Versions", func(t *testing.T) {
+		// Set --b2-versions for this test
+		f.opt.Versions = true
+		defer func() {
+			f.opt.Versions = false
+		}()
+
+		// Read the contents
+		entries, err := f.List(ctx, dirName)
+		require.NoError(t, err)
+		tests := 0
+		var fileNameVersion string
+		for _, entry := range entries {
+			t.Log(entry)
+			remote := entry.Remote()
+			if remote == fileName {
+				t.Run("ReadCurrent", func(t *testing.T) {
+					assert.Equal(t, newContents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
+				})
+				tests++
+			} else if versionTime, p := version.Remove(remote); !versionTime.IsZero() && p == fileName {
+				t.Run("ReadVersion", func(t *testing.T) {
+					assert.Equal(t, contents, fstests.ReadObject(ctx, t, entry.(fs.Object), -1))
+				})
+				assert.WithinDuration(t, time.Time(objMetadata.UploadTimestamp), versionTime, time.Second, "object time must be with 1 second of version time")
+				fileNameVersion = remote
+				tests++
+			}
+		}
+		assert.Equal(t, 2, tests, "object missing from listing")
+
+		// Check we can read the object with a version suffix
+		t.Run("NewObject", func(t *testing.T) {
+			o, err := f.NewObject(ctx, fileNameVersion)
+			require.NoError(t, err)
+			require.NotNil(t, o)
+			assert.Equal(t, int64(100), o.Size(), o.Remote())
+		})
+
+		// Check we can make a NewFs from that object with a version suffix
+		t.Run("NewFs", func(t *testing.T) {
+			newPath := bucket.Join(fs.ConfigStringFull(f), fileNameVersion)
+			// Make sure --b2-versions is set in the config of the new remote
+			fs.Debugf(nil, "oldPath = %q", newPath)
+			lastColon := strings.LastIndex(newPath, ":")
+			require.True(t, lastColon >= 0)
+			newPath = newPath[:lastColon] + ",versions" + newPath[lastColon:]
+			fs.Debugf(nil, "newPath = %q", newPath)
+			fNew, err := cache.Get(ctx, newPath)
+			// This should return pointing to a file
+			require.Equal(t, fs.ErrorIsFile, err)
+			require.NotNil(t, fNew)
+			// With the directory above
+			assert.Equal(t, dirName, path.Base(fs.ConfigStringFull(fNew)))
+		})
+	})
+
+	t.Run("VersionAt", func(t *testing.T) {
+		// We set --b2-version-at for this test so make sure we reset it at the end
+		defer func() {
+			f.opt.VersionAt = fs.Time{}
+		}()
+
+		var (
+			firstObjectTime  = time.Time(objMetadata.UploadTimestamp)
+			secondObjectTime = time.Time(newObjMetadata.UploadTimestamp)
+		)
+
+		for _, test := range []struct {
+			what     string
+			at       time.Time
+			want     []fstest.Item
+			wantErr  error
+			wantSize int64
+		}{
+			{
+				what:    "Before",
+				at:      firstObjectTime.Add(-time.Second),
+				want:    fstests.InternalTestFiles,
+				wantErr: fs.ErrorObjectNotFound,
+			},
+			{
+				what:     "AfterOne",
+				at:       firstObjectTime.Add(time.Second),
+				want:     append([]fstest.Item{item}, fstests.InternalTestFiles...),
+				wantSize: 100,
+			},
+			{
+				what:    "AfterDelete",
+				at:      secondObjectTime.Add(-time.Second),
+				want:    fstests.InternalTestFiles,
+				wantErr: fs.ErrorObjectNotFound,
+			},
+			{
+				what:     "AfterTwo",
+				at:       secondObjectTime.Add(time.Second),
+				want:     append([]fstest.Item{newItem}, fstests.InternalTestFiles...),
+				wantSize: 101,
+			},
+		} {
+			t.Run(test.what, func(t *testing.T) {
+				f.opt.VersionAt = fs.Time(test.at)
+				t.Run("List", func(t *testing.T) {
+					fstest.CheckListing(t, f, test.want)
+				})
+				// b2 NewObject doesn't work with VersionAt
+				//t.Run("NewObject", func(t *testing.T) {
+				//	gotObj, gotErr := f.NewObject(ctx, fileName)
+				//	assert.Equal(t, test.wantErr, gotErr)
+				//	if gotErr == nil {
+				//		assert.Equal(t, test.wantSize, gotObj.Size())
+				//	}
+				//})
+			})
+		}
+	})
+
+	t.Run("Cleanup", func(t *testing.T) {
+		require.NoError(t, f.cleanUp(ctx, true, false, 0))
+		items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)
+		fstest.CheckListing(t, f, items)
+		// Set --b2-versions for this test
+		f.opt.Versions = true
+		defer func() {
+			f.opt.Versions = false
+		}()
+		fstest.CheckListing(t, f, items)
+	})
+
+	// Purge gets tested later
+}
+
+// -run TestIntegration/FsMkdir/FsPutFiles/Internal
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Metadata", f.InternalTestMetadata)
+	t.Run("Versions", f.InternalTestVersions)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

backend/b2/b2_test.go

@@ -28,7 +28,12 @@ func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadCutoff(cs)
 }
 
+func (f *Fs) SetCopyCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setCopyCutoff(cs)
+}
+
 var (
 	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
 	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+	_ fstests.SetCopyCutoffer     = (*Fs)(nil)
 )

backend/b2/upload.go

@@ -1,11 +1,10 @@
 // Upload large files for b2
 //
-// Docs - https://www.backblaze.com/b2/docs/large_files.html
+// Docs - https://www.backblaze.com/docs/cloud-storage-large-files
 
 package b2
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha1"
 	"encoding/hex"
@@ -14,7 +13,6 @@ import (
 	"io"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
@@ -80,36 +78,31 @@ type largeUpload struct {
 	wrap      accounting.WrapFn               // account parts being transferred
 	id        string                          // ID of the file being uploaded
 	size      int64                           // total size
-	parts     int64                           // calculated number of parts, if known
+	parts     int                             // calculated number of parts, if known
+	sha1smu   sync.Mutex                      // mutex to protect sha1s
 	sha1s     []string                        // slice of SHA1s for each part
 	uploadMu  sync.Mutex                      // lock for upload variable
 	uploads   []*api.GetUploadPartURLResponse // result of get upload URL calls
 	chunkSize int64                           // chunk size to use
 	src       *Object                         // if copying, object we are reading from
+	info      *api.FileInfo                   // final response with info about the object
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
 //
 // If newInfo is set then metadata from that will be used instead of reading it from src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File, options ...fs.OpenOption) (up *largeUpload, err error) {
 	size := src.Size()
-	parts := int64(0)
-	sha1SliceSize := int64(maxParts)
+	parts := 0
 	chunkSize := defaultChunkSize
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
 		chunkSize = chunksize.Calculator(o, size, maxParts, defaultChunkSize)
-		parts = size / int64(chunkSize)
+		parts = int(size / int64(chunkSize))
 		if size%int64(chunkSize) != 0 {
 			parts++
 		}
-		sha1SliceSize = parts
-	}
-
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_start_large_file",
 	}
 	bucket, bucketPath := o.split()
 	bucketID, err := f.getBucketID(ctx, bucket)
@@ -120,12 +113,27 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		BucketID: bucketID,
 		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
+	optionsToSend := make([]fs.OpenOption, 0, len(options))
 	if newInfo == nil {
-		modTime := src.ModTime(ctx)
+		modTime, err := o.getModTime(ctx, src, options)
+		if err != nil {
+			return nil, err
+		}
+
 		request.ContentType = fs.MimeType(ctx, src)
 		request.Info = map[string]string{
 			timeKey: timeString(modTime),
 		}
+		// Custom upload headers - remove header prefix since they are sent in the body
+		for _, option := range options {
+			k, v := option.Header()
+			k = strings.ToLower(k)
+			if strings.HasPrefix(k, headerPrefix) {
+				request.Info[k[len(headerPrefix):]] = v
+			} else {
+				optionsToSend = append(optionsToSend, option)
+			}
+		}
 		// Set the SHA1 if known
 		if !o.fs.opt.DisableCheckSum || doCopy {
 			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
@@ -136,6 +144,11 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		request.ContentType = newInfo.ContentType
 		request.Info = newInfo.Info
 	}
+	opts := rest.Opts{
+		Method:  "POST",
+		Path:    "/b2_start_large_file",
+		Options: optionsToSend,
+	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
@@ -152,7 +165,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		id:        response.ID,
 		size:      size,
 		parts:     parts,
-		sha1s:     make([]string, sha1SliceSize),
+		sha1s:     make([]string, 0, 16),
 		chunkSize: int64(chunkSize),
 	}
 	// unwrap the accounting from the input, we use wrap to put it
@@ -171,24 +184,26 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 // This should be returned with returnUploadURL when finished
 func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) {
 	up.uploadMu.Lock()
-	defer up.uploadMu.Unlock()
-	if len(up.uploads) == 0 {
-		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/b2_get_upload_part_url",
-		}
-		var request = api.GetUploadPartURLRequest{
-			ID: up.id,
-		}
-		err := up.f.pacer.Call(func() (bool, error) {
-			resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
-			return up.f.shouldRetry(ctx, resp, err)
-		})
-		if err != nil {
-			return nil, fmt.Errorf("failed to get upload URL: %w", err)
-		}
-	} else {
+	if len(up.uploads) > 0 {
 		upload, up.uploads = up.uploads[0], up.uploads[1:]
+		up.uploadMu.Unlock()
+		return upload, nil
+	}
+	up.uploadMu.Unlock()
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_get_upload_part_url",
+	}
+	var request = api.GetUploadPartURLRequest{
+		ID: up.id,
+	}
+	err = up.f.pacer.Call(func() (bool, error) {
+		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
+		return up.f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get upload URL: %w", err)
 	}
 	return upload, nil
 }
@@ -203,10 +218,39 @@ func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) {
 	up.uploadMu.Unlock()
 }
 
-// Transfer a chunk
-func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error {
-	err := up.f.pacer.Call(func() (bool, error) {
-		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
+// Add an sha1 to the being built up sha1s
+func (up *largeUpload) addSha1(chunkNumber int, sha1 string) {
+	up.sha1smu.Lock()
+	defer up.sha1smu.Unlock()
+	if len(up.sha1s) < chunkNumber+1 {
+		up.sha1s = append(up.sha1s, make([]string, chunkNumber+1-len(up.sha1s))...)
+	}
+	up.sha1s[chunkNumber] = sha1
+}
+
+// WriteChunk will write chunk number with reader bytes, where chunk number >= 0
+func (up *largeUpload) WriteChunk(ctx context.Context, chunkNumber int, reader io.ReadSeeker) (size int64, err error) {
+	// Only account after the checksum reads have been done
+	if do, ok := reader.(pool.DelayAccountinger); ok {
+		// To figure out this number, do a transfer and if the accounted size is 0 or a
+		// multiple of what it should be, increase or decrease this number.
+		do.DelayAccounting(1)
+	}
+
+	err = up.f.pacer.Call(func() (bool, error) {
+		// Discover the size by seeking to the end
+		size, err = reader.Seek(0, io.SeekEnd)
+		if err != nil {
+			return false, err
+		}
+
+		// rewind the reader on retry and after reading size
+		_, err = reader.Seek(0, io.SeekStart)
+		if err != nil {
+			return false, err
+		}
+
+		fs.Debugf(up.o, "Sending chunk %d length %d", chunkNumber, size)
 
 		// Get upload URL
 		upload, err := up.getUploadURL(ctx)
@@ -214,8 +258,8 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 			return false, err
 		}
 
-		in := newHashAppendingReader(bytes.NewReader(body), sha1.New())
-		size := int64(len(body)) + int64(in.AdditionalLength())
+		in := newHashAppendingReader(reader, sha1.New())
+		sizeWithHash := size + int64(in.AdditionalLength())
 
 		// Authorization
 		//
@@ -245,10 +289,10 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 			Body:    up.wrap(in),
 			ExtraHeaders: map[string]string{
 				"Authorization":    upload.AuthorizationToken,
-				"X-Bz-Part-Number": fmt.Sprintf("%d", part),
+				"X-Bz-Part-Number": fmt.Sprintf("%d", chunkNumber+1),
 				sha1Header:         "hex_digits_at_end",
 			},
-			ContentLength: &size,
+			ContentLength: &sizeWithHash,
 		}
 
 		var response api.UploadPartResponse
@@ -256,7 +300,7 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 		resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
 		retry, err := up.f.shouldRetry(ctx, resp, err)
 		if err != nil {
-			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err)
+			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", chunkNumber, retry, err, err)
 		}
 		// On retryable error clear PartUploadURL
 		if retry {
@@ -264,30 +308,30 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 			upload = nil
 		}
 		up.returnUploadURL(upload)
-		up.sha1s[part-1] = in.HexSum()
+		up.addSha1(chunkNumber, in.HexSum())
 		return retry, err
 	})
 	if err != nil {
-		fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
+		fs.Debugf(up.o, "Error sending chunk %d: %v", chunkNumber, err)
 	} else {
-		fs.Debugf(up.o, "Done sending chunk %d", part)
+		fs.Debugf(up.o, "Done sending chunk %d", chunkNumber)
 	}
-	return err
+	return size, err
 }
 
 // Copy a chunk
-func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
+func (up *largeUpload) copyChunk(ctx context.Context, part int, partSize int64) error {
 	err := up.f.pacer.Call(func() (bool, error) {
 		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
 		opts := rest.Opts{
 			Method: "POST",
 			Path:   "/b2_copy_part",
 		}
-		offset := (part - 1) * up.chunkSize // where we are in the source file
+		offset := int64(part) * up.chunkSize // where we are in the source file
 		var request = api.CopyPartRequest{
 			SourceID:    up.src.id,
 			LargeFileID: up.id,
-			PartNumber:  part,
+			PartNumber:  int64(part + 1),
 			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
 		}
 		var response api.UploadPartResponse
@@ -296,7 +340,7 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64
 		if err != nil {
 			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
 		}
-		up.sha1s[part-1] = response.SHA1
+		up.addSha1(part, response.SHA1)
 		return retry, err
 	})
 	if err != nil {
@@ -307,8 +351,8 @@ func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64
 	return err
 }
 
-// finish closes off the large upload
-func (up *largeUpload) finish(ctx context.Context) error {
+// Close closes off the large upload
+func (up *largeUpload) Close(ctx context.Context) error {
 	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
 	opts := rest.Opts{
 		Method: "POST",
@@ -326,11 +370,12 @@ func (up *largeUpload) finish(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	return up.o.decodeMetaDataFileInfo(&response)
+	up.info = &response
+	return nil
 }
 
-// cancel aborts the large upload
-func (up *largeUpload) cancel(ctx context.Context) error {
+// Abort aborts the large upload
+func (up *largeUpload) Abort(ctx context.Context) error {
 	fs.Debugf(up.o, "Cancelling large file %s", up.what)
 	opts := rest.Opts{
 		Method: "POST",
@@ -355,157 +400,105 @@ func (up *largeUpload) cancel(ctx context.Context) error {
 // reaches EOF.
 //
 // Note that initialUploadBlock must be returned to f.putBuf()
-func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
+func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock *pool.RW) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
 	var (
 		g, gCtx      = errgroup.WithContext(ctx)
 		hasMoreParts = true
 	)
-	up.size = int64(len(initialUploadBlock))
-	g.Go(func() error {
-		for part := int64(1); hasMoreParts; part++ {
-			// Get a block of memory from the pool and token which limits concurrency.
-			var buf []byte
-			if part == 1 {
-				buf = initialUploadBlock
-			} else {
-				buf = up.f.getBuf(false)
-			}
-
-			// Fail fast, in case an errgroup managed function returns an error
-			// gCtx is cancelled. There is no point in uploading all the other parts.
-			if gCtx.Err() != nil {
-				up.f.putBuf(buf, false)
-				return nil
-			}
-
-			// Read the chunk
-			var n int
-			if part == 1 {
-				n = len(buf)
-			} else {
-				n, err = io.ReadFull(up.in, buf)
-				if err == io.ErrUnexpectedEOF {
-					fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
-					buf = buf[:n]
-					hasMoreParts = false
-				} else if err == io.EOF {
-					fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
-					up.f.putBuf(buf, false)
-					return nil
-				} else if err != nil {
-					// other kinds of errors indicate failure
-					up.f.putBuf(buf, false)
-					return err
-				}
-			}
-
-			// Keep stats up to date
-			up.parts = part
-			up.size += int64(n)
-			if part > maxParts {
-				up.f.putBuf(buf, false)
-				return fmt.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
-			}
-
-			part := part // for the closure
-			g.Go(func() (err error) {
-				defer up.f.putBuf(buf, false)
-				return up.transferChunk(gCtx, part, buf)
-			})
+	up.size = initialUploadBlock.Size()
+	up.parts = 0
+	for part := 0; hasMoreParts; part++ {
+		// Get a block of memory from the pool and token which limits concurrency.
+		var rw *pool.RW
+		if part == 0 {
+			rw = initialUploadBlock
+		} else {
+			rw = up.f.getRW(false)
 		}
-		return nil
-	})
+
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in uploading all the other parts.
+		if gCtx.Err() != nil {
+			up.f.putRW(rw)
+			break
+		}
+
+		// Read the chunk
+		var n int64
+		if part == 0 {
+			n = rw.Size()
+		} else {
+			n, err = io.CopyN(rw, up.in, up.chunkSize)
+			if err == io.EOF {
+				if n == 0 {
+					fs.Debugf(up.o, "Not sending empty chunk after EOF - ending.")
+					up.f.putRW(rw)
+					break
+				} else {
+					fs.Debugf(up.o, "Read less than a full chunk %d, making this the last one.", n)
+				}
+				hasMoreParts = false
+			} else if err != nil {
+				// other kinds of errors indicate failure
+				up.f.putRW(rw)
+				return err
+			}
+		}
+
+		// Keep stats up to date
+		up.parts += 1
+		up.size += n
+		if part > maxParts {
+			up.f.putRW(rw)
+			return fmt.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+		}
+
+		part := part // for the closure
+		g.Go(func() (err error) {
+			defer up.f.putRW(rw)
+			_, err = up.WriteChunk(gCtx, part, rw)
+			return err
+		})
+	}
 	err = g.Wait()
 	if err != nil {
 		return err
 	}
-	up.sha1s = up.sha1s[:up.parts]
-	return up.finish(ctx)
+	return up.Close(ctx)
 }
 
-// Upload uploads the chunks from the input
-func (up *largeUpload) Upload(ctx context.Context) (err error) {
-	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
+// Copy the chunks from the source to the destination
+func (up *largeUpload) Copy(ctx context.Context) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.Abort(ctx) })()
 	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
 	var (
-		g, gCtx    = errgroup.WithContext(ctx)
-		remaining  = up.size
-		uploadPool *pool.Pool
-		ci         = fs.GetConfig(ctx)
+		g, gCtx   = errgroup.WithContext(ctx)
+		remaining = up.size
 	)
-	// If using large chunk size then make a temporary pool
-	if up.chunkSize <= int64(up.f.opt.ChunkSize) {
-		uploadPool = up.f.pool
-	} else {
-		uploadPool = pool.New(
-			time.Duration(up.f.opt.MemoryPoolFlushTime),
-			int(up.chunkSize),
-			ci.Transfers,
-			up.f.opt.MemoryPoolUseMmap,
-		)
-		defer uploadPool.Flush()
-	}
-	// Get an upload token and a buffer
-	getBuf := func() (buf []byte) {
-		up.f.getBuf(true)
-		if !up.doCopy {
-			buf = uploadPool.Get()
+	g.SetLimit(up.f.opt.UploadConcurrency)
+	for part := 0; part < up.parts; part++ {
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in copying all the other parts.
+		if gCtx.Err() != nil {
+			break
 		}
-		return buf
-	}
-	// Put an upload token and a buffer
-	putBuf := func(buf []byte) {
-		if !up.doCopy {
-			uploadPool.Put(buf)
+
+		reqSize := remaining
+		if reqSize >= up.chunkSize {
+			reqSize = up.chunkSize
 		}
-		up.f.putBuf(nil, true)
+
+		part := part // for the closure
+		g.Go(func() (err error) {
+			return up.copyChunk(gCtx, part, reqSize)
+		})
+		remaining -= reqSize
 	}
-	g.Go(func() error {
-		for part := int64(1); part <= up.parts; part++ {
-			// Get a block of memory from the pool and token which limits concurrency.
-			buf := getBuf()
-
-			// Fail fast, in case an errgroup managed function returns an error
-			// gCtx is cancelled. There is no point in uploading all the other parts.
-			if gCtx.Err() != nil {
-				putBuf(buf)
-				return nil
-			}
-
-			reqSize := remaining
-			if reqSize >= up.chunkSize {
-				reqSize = up.chunkSize
-			}
-
-			if !up.doCopy {
-				// Read the chunk
-				buf = buf[:reqSize]
-				_, err = io.ReadFull(up.in, buf)
-				if err != nil {
-					putBuf(buf)
-					return err
-				}
-			}
-
-			part := part // for the closure
-			g.Go(func() (err error) {
-				defer putBuf(buf)
-				if !up.doCopy {
-					err = up.transferChunk(gCtx, part, buf)
-				} else {
-					err = up.copyChunk(gCtx, part, reqSize)
-				}
-				return err
-			})
-			remaining -= reqSize
-		}
-		return nil
-	})
 	err = g.Wait()
 	if err != nil {
 		return err
 	}
-	return up.finish(ctx)
+	return up.Close(ctx)
 }

backend/box/api/types.go

@@ -52,7 +52,7 @@ func (e *Error) Error() string {
 		out += ": " + e.Message
 	}
 	if e.ContextInfo != nil {
-		out += fmt.Sprintf(" (%+v)", e.ContextInfo)
+		out += fmt.Sprintf(" (%s)", string(e.ContextInfo))
 	}
 	return out
 }
@@ -63,7 +63,7 @@ var _ error = (*Error)(nil)
 // ItemFields are the fields needed for FileInfo
 var ItemFields = "type,id,sequence_id,etag,sha1,name,size,created_at,modified_at,content_created_at,content_modified_at,item_status,shared_link,owned_by"
 
-// Types of things in Item
+// Types of things in Item/ItemMini
 const (
 	ItemTypeFolder    = "folder"
 	ItemTypeFile      = "file"
@@ -72,20 +72,31 @@ const (
 	ItemStatusDeleted = "deleted"
 )
 
+// ItemMini is a subset of the elements in a full Item returned by some API calls
+type ItemMini struct {
+	Type       string `json:"type"`
+	ID         string `json:"id"`
+	SequenceID int64  `json:"sequence_id,string"`
+	Etag       string `json:"etag"`
+	SHA1       string `json:"sha1"`
+	Name       string `json:"name"`
+}
+
 // Item describes a folder or a file as returned by Get Folder Items and others
 type Item struct {
-	Type              string  `json:"type"`
-	ID                string  `json:"id"`
-	SequenceID        string  `json:"sequence_id"`
-	Etag              string  `json:"etag"`
-	SHA1              string  `json:"sha1"`
-	Name              string  `json:"name"`
-	Size              float64 `json:"size"` // box returns this in xEyy format for very large numbers - see #2261
-	CreatedAt         Time    `json:"created_at"`
-	ModifiedAt        Time    `json:"modified_at"`
-	ContentCreatedAt  Time    `json:"content_created_at"`
-	ContentModifiedAt Time    `json:"content_modified_at"`
-	ItemStatus        string  `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
+	Type              string   `json:"type"`
+	ID                string   `json:"id"`
+	SequenceID        int64    `json:"sequence_id,string"`
+	Etag              string   `json:"etag"`
+	SHA1              string   `json:"sha1"`
+	Name              string   `json:"name"`
+	Size              float64  `json:"size"` // box returns this in xEyy format for very large numbers - see #2261
+	CreatedAt         Time     `json:"created_at"`
+	ModifiedAt        Time     `json:"modified_at"`
+	ContentCreatedAt  Time     `json:"content_created_at"`
+	ContentModifiedAt Time     `json:"content_modified_at"`
+	ItemStatus        string   `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
+	Parent            ItemMini `json:"parent"`
 	SharedLink        struct {
 		URL    string `json:"url,omitempty"`
 		Access string `json:"access,omitempty"`
@@ -156,19 +167,7 @@ type PreUploadCheckResponse struct {
 // PreUploadCheckConflict is returned in the ContextInfo error field
 // from PreUploadCheck when the error code is "item_name_in_use"
 type PreUploadCheckConflict struct {
-	Conflicts struct {
-		Type        string `json:"type"`
-		ID          string `json:"id"`
-		FileVersion struct {
-			Type string `json:"type"`
-			ID   string `json:"id"`
-			Sha1 string `json:"sha1"`
-		} `json:"file_version"`
-		SequenceID string `json:"sequence_id"`
-		Etag       string `json:"etag"`
-		Sha1       string `json:"sha1"`
-		Name       string `json:"name"`
-	} `json:"conflicts"`
+	Conflicts ItemMini `json:"conflicts"`
 }
 
 // UpdateFileModTime is used in Update File Info
@@ -281,3 +280,30 @@ type User struct {
 	Address       string    `json:"address"`
 	AvatarURL     string    `json:"avatar_url"`
 }
+
+// FileTreeChangeEventTypes are the events that can require cache invalidation
+var FileTreeChangeEventTypes = map[string]struct{}{
+	"ITEM_COPY":                 {},
+	"ITEM_CREATE":               {},
+	"ITEM_MAKE_CURRENT_VERSION": {},
+	"ITEM_MODIFY":               {},
+	"ITEM_MOVE":                 {},
+	"ITEM_RENAME":               {},
+	"ITEM_TRASH":                {},
+	"ITEM_UNDELETE_VIA_TRASH":   {},
+	"ITEM_UPLOAD":               {},
+}
+
+// Event is an array element in the response returned from /events
+type Event struct {
+	EventType string `json:"event_type"`
+	EventID   string `json:"event_id"`
+	Source    Item   `json:"source"`
+}
+
+// Events is returned from /events
+type Events struct {
+	ChunkSize          int64   `json:"chunk_size"`
+	Entries            []Event `json:"entries"`
+	NextStreamPosition int64   `json:"next_stream_position"`
+}

backend/box/box.go

@@ -149,6 +149,23 @@ func init() {
 			Default:  "",
 			Help:     "Only show items owned by the login (email address) passed in.",
 			Advanced: true,
+		}, {
+			Name:    "impersonate",
+			Default: "",
+			Help: `Impersonate this user ID when using a service account.
+
+Setting this flag allows rclone, when using a JWT service account, to
+act on behalf of another user by setting the as-user header.
+
+The user ID is the Box identifier for a user. User IDs can found for
+any user via the GET /users endpoint, which is only available to
+admins, or by calling the GET /users/me endpoint with an authenticated
+user session.
+
+See: https://developer.box.com/guides/authentication/jwt/as-user/
+`,
+			Advanced:  true,
+			Sensitive: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -262,19 +279,29 @@ type Options struct {
 	AccessToken   string               `config:"access_token"`
 	ListChunk     int                  `config:"list_chunk"`
 	OwnedBy       string               `config:"owned_by"`
+	Impersonate   string               `config:"impersonate"`
+}
+
+// ItemMeta defines metadata we cache for each Item ID
+type ItemMeta struct {
+	SequenceID int64  // the most recent event processed for this item
+	ParentID   string // ID of the parent directory of this item
+	Name       string // leaf name of this item
 }
 
 // Fs represents a remote box
 type Fs struct {
-	name         string                // name of this remote
-	root         string                // the path we are working on
-	opt          Options               // parsed options
-	features     *fs.Features          // optional features
-	srv          *rest.Client          // the connection to the server
-	dirCache     *dircache.DirCache    // Map of directory path to directory id
-	pacer        *fs.Pacer             // pacer for API calls
-	tokenRenewer *oauthutil.Renew      // renew the token on expiry
-	uploadToken  *pacer.TokenDispenser // control concurrency
+	name            string                // name of this remote
+	root            string                // the path we are working on
+	opt             Options               // parsed options
+	features        *fs.Features          // optional features
+	srv             *rest.Client          // the connection to the server
+	dirCache        *dircache.DirCache    // Map of directory path to directory id
+	pacer           *fs.Pacer             // pacer for API calls
+	tokenRenewer    *oauthutil.Renew      // renew the token on expiry
+	uploadToken     *pacer.TokenDispenser // control concurrency
+	itemMetaCacheMu *sync.Mutex           // protects itemMetaCache
+	itemMetaCache   map[string]ItemMeta   // map of Item ID to selected metadata
 }
 
 // Object describes a box object
@@ -353,7 +380,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
-	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
+	// defer log.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
 	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
@@ -362,20 +389,30 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		return nil, err
 	}
 
-	found, err := f.listAll(ctx, directoryID, false, true, true, func(item *api.Item) bool {
-		if strings.EqualFold(item.Name, leaf) {
-			info = item
-			return true
-		}
-		return false
+	// Use preupload to find the ID
+	itemMini, err := f.preUploadCheck(ctx, leaf, directoryID, -1)
+	if err != nil {
+		return nil, err
+	}
+	if itemMini == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	// Now we have the ID we can look up the object proper
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/files/" + itemMini.ID,
+		Parameters: fieldsValue(),
+	}
+	var item api.Item
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &item)
+		return shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
 	}
-	if !found {
-		return nil, fs.ErrorObjectNotFound
-	}
-	return info, nil
+	return &item, nil
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -422,12 +459,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:        name,
-		root:        root,
-		opt:         *opt,
-		srv:         rest.NewClient(client).SetRoot(rootURL),
-		pacer:       fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
-		uploadToken: pacer.NewTokenDispenser(ci.Transfers),
+		name:            name,
+		root:            root,
+		opt:             *opt,
+		srv:             rest.NewClient(client).SetRoot(rootURL),
+		pacer:           fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken:     pacer.NewTokenDispenser(ci.Transfers),
+		itemMetaCacheMu: new(sync.Mutex),
+		itemMetaCache:   make(map[string]ItemMeta),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -440,6 +479,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		f.srv.SetHeader("Authorization", "Bearer "+f.opt.AccessToken)
 	}
 
+	// If using impersonate set an as-user header
+	if f.opt.Impersonate != "" {
+		f.srv.SetHeader("as-user", f.opt.Impersonate)
+	}
+
 	jsonFile, ok := m.Get("box_config_file")
 	boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 
@@ -682,6 +726,17 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			}
 			entries = append(entries, o)
 		}
+
+		// Cache some metadata for this Item to help us process events later
+		// on. In particular, the box event API does not provide the old path
+		// of the Item when it is renamed/deleted/moved/etc.
+		f.itemMetaCacheMu.Lock()
+		cachedItemMeta, found := f.itemMetaCache[info.ID]
+		if !found || cachedItemMeta.SequenceID < info.SequenceID {
+			f.itemMetaCache[info.ID] = ItemMeta{SequenceID: info.SequenceID, ParentID: directoryID, Name: info.Name}
+		}
+		f.itemMetaCacheMu.Unlock()
+
 		return false
 	})
 	if err != nil {
@@ -717,7 +772,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 //
 // It returns "", nil if the file is good to go
 // It returns "ID", nil if the file must be updated
-func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (ID string, err error) {
+func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size int64) (item *api.ItemMini, err error) {
 	check := api.PreUploadCheck{
 		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
@@ -742,16 +797,16 @@ func (f *Fs) preUploadCheck(ctx context.Context, leaf, directoryID string, size
 			var conflict api.PreUploadCheckConflict
 			err = json.Unmarshal(apiErr.ContextInfo, &conflict)
 			if err != nil {
-				return "", fmt.Errorf("pre-upload check: JSON decode failed: %w", err)
+				return nil, fmt.Errorf("pre-upload check: JSON decode failed: %w", err)
 			}
 			if conflict.Conflicts.Type != api.ItemTypeFile {
-				return "", fmt.Errorf("pre-upload check: can't overwrite non file with file: %w", err)
+				return nil, fs.ErrorIsDir
 			}
-			return conflict.Conflicts.ID, nil
+			return &conflict.Conflicts, nil
 		}
-		return "", fmt.Errorf("pre-upload check: %w", err)
+		return nil, fmt.Errorf("pre-upload check: %w", err)
 	}
-	return "", nil
+	return nil, nil
 }
 
 // Put the object
@@ -772,11 +827,11 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 	// Preflight check the upload, which returns the ID if the
 	// object already exists
-	ID, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
+	item, err := f.preUploadCheck(ctx, leaf, directoryID, src.Size())
 	if err != nil {
 		return nil, err
 	}
-	if ID == "" {
+	if item == nil {
 		return f.PutUnchecked(ctx, in, src, options...)
 	}
 
@@ -784,7 +839,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	o := &Object{
 		fs:     f,
 		remote: remote,
-		id:     ID,
+		id:     item.ID,
 	}
 	return o, o.Update(ctx, in, src, options...)
 }
@@ -1121,7 +1176,7 @@ func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
 // CleanUp empties the trash
 func (f *Fs) CleanUp(ctx context.Context) (err error) {
 	var (
-		deleteErrors       = int64(0)
+		deleteErrors       atomic.Uint64
 		concurrencyControl = make(chan struct{}, fs.GetConfig(ctx).Checkers)
 		wg                 sync.WaitGroup
 	)
@@ -1137,7 +1192,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 				err := f.deletePermanently(ctx, item.Type, item.ID)
 				if err != nil {
 					fs.Errorf(f, "failed to delete trash item %q (%q): %v", item.Name, item.ID, err)
-					atomic.AddInt64(&deleteErrors, 1)
+					deleteErrors.Add(1)
 				}
 			}()
 		} else {
@@ -1146,12 +1201,283 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 		return false
 	})
 	wg.Wait()
-	if deleteErrors != 0 {
-		return fmt.Errorf("failed to delete %d trash items", deleteErrors)
+	if deleteErrors.Load() != 0 {
+		return fmt.Errorf("failed to delete %d trash items", deleteErrors.Load())
 	}
 	return err
 }
 
+// Shutdown shutdown the fs
+func (f *Fs) Shutdown(ctx context.Context) error {
+	f.tokenRenewer.Shutdown()
+	return nil
+}
+
+// ChangeNotify calls the passed function with a path that has had changes.
+// If the implementation uses polling, it should adhere to the given interval.
+//
+// Automatically restarts itself in case of unexpected behavior of the remote.
+//
+// Close the returned channel to stop being notified.
+func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
+	go func() {
+		// get the `stream_position` early so all changes from now on get processed
+		streamPosition, err := f.changeNotifyStreamPosition(ctx)
+		if err != nil {
+			fs.Infof(f, "Failed to get StreamPosition: %s", err)
+		}
+
+		// box can send duplicate Event IDs. Use this map to track and filter
+		// the ones we've already processed.
+		processedEventIDs := make(map[string]time.Time)
+
+		var ticker *time.Ticker
+		var tickerC <-chan time.Time
+		for {
+			select {
+			case pollInterval, ok := <-pollIntervalChan:
+				if !ok {
+					if ticker != nil {
+						ticker.Stop()
+					}
+					return
+				}
+				if ticker != nil {
+					ticker.Stop()
+					ticker, tickerC = nil, nil
+				}
+				if pollInterval != 0 {
+					ticker = time.NewTicker(pollInterval)
+					tickerC = ticker.C
+				}
+			case <-tickerC:
+				if streamPosition == "" {
+					streamPosition, err = f.changeNotifyStreamPosition(ctx)
+					if err != nil {
+						fs.Infof(f, "Failed to get StreamPosition: %s", err)
+						continue
+					}
+				}
+
+				// Garbage collect EventIDs older than 1 minute
+				for eventID, timestamp := range processedEventIDs {
+					if time.Since(timestamp) > time.Minute {
+						delete(processedEventIDs, eventID)
+					}
+				}
+
+				streamPosition, err = f.changeNotifyRunner(ctx, notifyFunc, streamPosition, processedEventIDs)
+				if err != nil {
+					fs.Infof(f, "Change notify listener failure: %s", err)
+				}
+			}
+		}
+	}()
+}
+
+func (f *Fs) changeNotifyStreamPosition(ctx context.Context) (streamPosition string, err error) {
+	opts := rest.Opts{
+		Method:     "GET",
+		Path:       "/events",
+		Parameters: fieldsValue(),
+	}
+	opts.Parameters.Set("stream_position", "now")
+	opts.Parameters.Set("stream_type", "changes")
+
+	var result api.Events
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return "", err
+	}
+
+	return strconv.FormatInt(result.NextStreamPosition, 10), nil
+}
+
+// Attempts to construct the full path for an object, given the ID of its
+// parent directory and the name of the object.
+//
+// Can return "" if the parentID is not currently in the directory cache.
+func (f *Fs) getFullPath(parentID string, childName string) (fullPath string) {
+	fullPath = ""
+	name := f.opt.Enc.ToStandardName(childName)
+	if parentID != "" {
+		if parentDir, ok := f.dirCache.GetInv(parentID); ok {
+			if len(parentDir) > 0 {
+				fullPath = parentDir + "/" + name
+			} else {
+				fullPath = name
+			}
+		}
+	} else {
+		// No parent, this object is at the root
+		fullPath = name
+	}
+	return fullPath
+}
+
+func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), streamPosition string, processedEventIDs map[string]time.Time) (nextStreamPosition string, err error) {
+	nextStreamPosition = streamPosition
+
+	for {
+		limit := f.opt.ListChunk
+
+		// box only allows a max of 500 events
+		if limit > 500 {
+			limit = 500
+		}
+
+		opts := rest.Opts{
+			Method:     "GET",
+			Path:       "/events",
+			Parameters: fieldsValue(),
+		}
+		opts.Parameters.Set("stream_position", nextStreamPosition)
+		opts.Parameters.Set("stream_type", "changes")
+		opts.Parameters.Set("limit", strconv.Itoa(limit))
+
+		var result api.Events
+		var resp *http.Response
+		fs.Debugf(f, "Checking for changes on remote (next_stream_position: %q)", nextStreamPosition)
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(ctx, resp, err)
+		})
+		if err != nil {
+			return "", err
+		}
+
+		if result.ChunkSize != int64(len(result.Entries)) {
+			return "", fmt.Errorf("invalid response to event request, chunk_size (%v) not equal to number of entries (%v)", result.ChunkSize, len(result.Entries))
+		}
+
+		nextStreamPosition = strconv.FormatInt(result.NextStreamPosition, 10)
+		if result.ChunkSize == 0 {
+			return nextStreamPosition, nil
+		}
+
+		type pathToClear struct {
+			path      string
+			entryType fs.EntryType
+		}
+		var pathsToClear []pathToClear
+		newEventIDs := 0
+		for _, entry := range result.Entries {
+			eventDetails := fmt.Sprintf("[%q(%d)|%s|%s|%s|%s]", entry.Source.Name, entry.Source.SequenceID,
+				entry.Source.Type, entry.EventType, entry.Source.ID, entry.EventID)
+
+			if entry.EventID == "" {
+				fs.Debugf(f, "%s ignored due to missing EventID", eventDetails)
+				continue
+			}
+			if _, ok := processedEventIDs[entry.EventID]; ok {
+				fs.Debugf(f, "%s ignored due to duplicate EventID", eventDetails)
+				continue
+			}
+			processedEventIDs[entry.EventID] = time.Now()
+			newEventIDs++
+
+			if entry.Source.ID == "" { // missing File or Folder ID
+				fs.Debugf(f, "%s ignored due to missing SourceID", eventDetails)
+				continue
+			}
+			if entry.Source.Type != api.ItemTypeFile && entry.Source.Type != api.ItemTypeFolder { // event is not for a file or folder
+				fs.Debugf(f, "%s ignored due to unsupported SourceType", eventDetails)
+				continue
+			}
+
+			// Only interested in event types that result in a file tree change
+			if _, found := api.FileTreeChangeEventTypes[entry.EventType]; !found {
+				fs.Debugf(f, "%s ignored due to unsupported EventType", eventDetails)
+				continue
+			}
+
+			f.itemMetaCacheMu.Lock()
+			itemMeta, cachedItemMetaFound := f.itemMetaCache[entry.Source.ID]
+			if cachedItemMetaFound {
+				if itemMeta.SequenceID >= entry.Source.SequenceID {
+					// Item in the cache has the same or newer SequenceID than
+					// this event. Ignore this event, it must be old.
+					f.itemMetaCacheMu.Unlock()
+					fs.Debugf(f, "%s ignored due to old SequenceID (%q)", eventDetails, itemMeta.SequenceID)
+					continue
+				}
+
+				// This event is newer. Delete its entry from the cache,
+				// we'll notify about its change below, then it's up to a
+				// future list operation to repopulate the cache.
+				delete(f.itemMetaCache, entry.Source.ID)
+			}
+			f.itemMetaCacheMu.Unlock()
+
+			entryType := fs.EntryDirectory
+			if entry.Source.Type == api.ItemTypeFile {
+				entryType = fs.EntryObject
+			}
+
+			// The box event only includes the new path for the object (e.g.
+			// the path after the object was moved). If there was an old path
+			// saved in our cache, it must be cleared.
+			if cachedItemMetaFound {
+				path := f.getFullPath(itemMeta.ParentID, itemMeta.Name)
+				if path != "" {
+					fs.Debugf(f, "%s added old path (%q) for notify", eventDetails, path)
+					pathsToClear = append(pathsToClear, pathToClear{path: path, entryType: entryType})
+				} else {
+					fs.Debugf(f, "%s old parent not cached", eventDetails)
+				}
+
+				// If this is a directory, also delete it from the dir cache.
+				// This will effectively invalidate the item metadata cache
+				// entries for all descendents of this directory, since we
+				// will no longer be able to construct a full path for them.
+				// This is exactly what we want, since we don't want to notify
+				// on the paths of these descendents if one of their ancestors
+				// has been renamed/deleted.
+				if entry.Source.Type == api.ItemTypeFolder {
+					f.dirCache.FlushDir(path)
+				}
+			}
+
+			// If the item is "active", then it is not trashed or deleted, so
+			// it potentially has a valid parent.
+			//
+			// Construct the new path of the object, based on the Parent ID
+			// and its name. If we get an empty result, it means we don't
+			// currently know about this object so notification is unnecessary.
+			if entry.Source.ItemStatus == api.ItemStatusActive {
+				path := f.getFullPath(entry.Source.Parent.ID, entry.Source.Name)
+				if path != "" {
+					fs.Debugf(f, "%s added new path (%q) for notify", eventDetails, path)
+					pathsToClear = append(pathsToClear, pathToClear{path: path, entryType: entryType})
+				} else {
+					fs.Debugf(f, "%s new parent not found", eventDetails)
+				}
+			}
+		}
+
+		// box can sometimes repeatedly return the same Event IDs within a
+		// short period of time. If it stops giving us new ones, treat it
+		// the same as if it returned us none at all.
+		if newEventIDs == 0 {
+			return nextStreamPosition, nil
+		}
+
+		notifiedPaths := make(map[string]bool)
+		for _, p := range pathsToClear {
+			if _, ok := notifiedPaths[p.path]; ok {
+				continue
+			}
+			notifiedPaths[p.path] = true
+			notifyFunc(p.path, p.entryType)
+		}
+		fs.Debugf(f, "Received %v events, resulting in %v paths and %v notifications", len(result.Entries), len(pathsToClear), len(notifiedPaths))
+	}
+}
+
 // DirCacheFlush resets the directory cache - used in testing as an
 // optional interface
 func (f *Fs) DirCacheFlush() {
@@ -1399,6 +1725,7 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
+	_ fs.Shutdowner      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/cache/cache.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 // Package cache implements a virtual provider to cache existing remotes.
 package cache
@@ -410,18 +409,16 @@ func NewFs(ctx context.Context, name, rootPath string, m configmap.Mapper) (fs.F
 			if err != nil {
 				return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
 			}
-		} else {
-			if opt.PlexPassword != "" && opt.PlexUsername != "" {
-				decPass, err := obscure.Reveal(opt.PlexPassword)
-				if err != nil {
-					decPass = opt.PlexPassword
-				}
-				f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
-					m.Set("plex_token", token)
-				})
-				if err != nil {
-					return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
-				}
+		} else if opt.PlexPassword != "" && opt.PlexUsername != "" {
+			decPass, err := obscure.Reveal(opt.PlexPassword)
+			if err != nil {
+				decPass = opt.PlexPassword
+			}
+			f.plexConnector, err = newPlexConnector(f, opt.PlexURL, opt.PlexUsername, decPass, opt.PlexInsecure, func(token string) {
+				m.Set("plex_token", token)
+			})
+			if err != nil {
+				return nil, fmt.Errorf("failed to connect to the Plex API %v: %w", opt.PlexURL, err)
 			}
 		}
 	}

backend/cache/cache_internal_test.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js && !race
-// +build !plan9,!js,!race
 
 package cache_test
 
@@ -11,7 +10,6 @@ import (
 	goflag "flag"
 	"fmt"
 	"io"
-	"log"
 	"math/rand"
 	"os"
 	"path"
@@ -30,10 +28,11 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
-	"github.com/rclone/rclone/vfs/vfsflags"
+	"github.com/rclone/rclone/vfs/vfscommon"
 	"github.com/stretchr/testify/require"
 )
 
@@ -93,7 +92,7 @@ func TestMain(m *testing.M) {
 	goflag.Parse()
 	var rc int
 
-	log.Printf("Running with the following params: \n remote: %v", remoteName)
+	fs.Logf(nil, "Running with the following params: \n remote: %v", remoteName)
 	runInstance = newRun()
 	rc = m.Run()
 	os.Exit(rc)
@@ -123,10 +122,10 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 
 /* TODO: is this testing something?
 func TestInternalVfsCache(t *testing.T) {
-	vfsflags.Opt.DirCacheTime = time.Second * 30
+	vfscommon.Opt.DirCacheTime = time.Second * 30
 	testSize := int64(524288000)
 
-	vfsflags.Opt.CacheMode = vfs.CacheModeWrites
+	vfscommon.Opt.CacheMode = vfs.CacheModeWrites
 	id := "tiuufo"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, map[string]string{"writes": "true", "info_age": "1h"})
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -338,7 +337,7 @@ func TestInternalCachedUpdatedContentMatches(t *testing.T) {
 
 func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tiwwcm%v", time.Now().Unix())
-	vfsflags.Opt.DirCacheTime = time.Second
+	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
@@ -368,7 +367,7 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 func TestInternalLargeWrittenContentMatches(t *testing.T) {
 	id := fmt.Sprintf("tilwcm%v", time.Now().Unix())
-	vfsflags.Opt.DirCacheTime = time.Second
+	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second)
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	if runInstance.rootIsCrypt {
 		t.Skip("test skipped with crypt remote")
@@ -408,7 +407,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	// update in the wrapped fs
 	originalSize, err := runInstance.size(t, rootFs, "data.bin")
 	require.NoError(t, err)
-	log.Printf("original size: %v", originalSize)
+	fs.Logf(nil, "original size: %v", originalSize)
 
 	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
@@ -417,7 +416,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	if runInstance.rootIsCrypt {
 		data2, err = base64.StdEncoding.DecodeString(cryptedText3Base64)
 		require.NoError(t, err)
-		expectedSize = expectedSize + 1 // FIXME newline gets in, likely test data issue
+		expectedSize++ // FIXME newline gets in, likely test data issue
 	} else {
 		data2 = []byte("test content")
 	}
@@ -425,7 +424,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	err = o.Update(context.Background(), bytes.NewReader(data2), objInfo)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(data2)), o.Size())
-	log.Printf("updated size: %v", len(data2))
+	fs.Logf(nil, "updated size: %v", len(data2))
 
 	// get a new instance from the cache
 	if runInstance.wrappedIsExternal {
@@ -485,49 +484,49 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	err = runInstance.retryBlock(func() error {
 		li, err := runInstance.list(t, rootFs, "test")
 		if err != nil {
-			log.Printf("err: %v", err)
+			fs.Logf(nil, "err: %v", err)
 			return err
 		}
 		if len(li) != 2 {
-			log.Printf("not expected listing /test: %v", li)
+			fs.Logf(nil, "not expected listing /test: %v", li)
 			return fmt.Errorf("not expected listing /test: %v", li)
 		}
 
 		li, err = runInstance.list(t, rootFs, "test/one")
 		if err != nil {
-			log.Printf("err: %v", err)
+			fs.Logf(nil, "err: %v", err)
 			return err
 		}
 		if len(li) != 0 {
-			log.Printf("not expected listing /test/one: %v", li)
+			fs.Logf(nil, "not expected listing /test/one: %v", li)
 			return fmt.Errorf("not expected listing /test/one: %v", li)
 		}
 
 		li, err = runInstance.list(t, rootFs, "test/second")
 		if err != nil {
-			log.Printf("err: %v", err)
+			fs.Logf(nil, "err: %v", err)
 			return err
 		}
 		if len(li) != 1 {
-			log.Printf("not expected listing /test/second: %v", li)
+			fs.Logf(nil, "not expected listing /test/second: %v", li)
 			return fmt.Errorf("not expected listing /test/second: %v", li)
 		}
 		if fi, ok := li[0].(os.FileInfo); ok {
 			if fi.Name() != "data.bin" {
-				log.Printf("not expected name: %v", fi.Name())
+				fs.Logf(nil, "not expected name: %v", fi.Name())
 				return fmt.Errorf("not expected name: %v", fi.Name())
 			}
 		} else if di, ok := li[0].(fs.DirEntry); ok {
 			if di.Remote() != "test/second/data.bin" {
-				log.Printf("not expected remote: %v", di.Remote())
+				fs.Logf(nil, "not expected remote: %v", di.Remote())
 				return fmt.Errorf("not expected remote: %v", di.Remote())
 			}
 		} else {
-			log.Printf("unexpected listing: %v", li)
+			fs.Logf(nil, "unexpected listing: %v", li)
 			return fmt.Errorf("unexpected listing: %v", li)
 		}
 
-		log.Printf("complete listing: %v", li)
+		fs.Logf(nil, "complete listing: %v", li)
 		return nil
 	}, 12, time.Second*10)
 	require.NoError(t, err)
@@ -577,43 +576,43 @@ func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
 	err = runInstance.retryBlock(func() error {
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
 		if !found {
-			log.Printf("not found /test")
+			fs.Logf(nil, "not found /test")
 			return fmt.Errorf("not found /test")
 		}
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one")))
 		if !found {
-			log.Printf("not found /test/one")
+			fs.Logf(nil, "not found /test/one")
 			return fmt.Errorf("not found /test/one")
 		}
 		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"), runInstance.encryptRemoteIfNeeded(t, "test2")))
 		if !found {
-			log.Printf("not found /test/one/test2")
+			fs.Logf(nil, "not found /test/one/test2")
 			return fmt.Errorf("not found /test/one/test2")
 		}
 		li, err := runInstance.list(t, rootFs, "test/one")
 		if err != nil {
-			log.Printf("err: %v", err)
+			fs.Logf(nil, "err: %v", err)
 			return err
 		}
 		if len(li) != 1 {
-			log.Printf("not expected listing /test/one: %v", li)
+			fs.Logf(nil, "not expected listing /test/one: %v", li)
 			return fmt.Errorf("not expected listing /test/one: %v", li)
 		}
 		if fi, ok := li[0].(os.FileInfo); ok {
 			if fi.Name() != "test2" {
-				log.Printf("not expected name: %v", fi.Name())
+				fs.Logf(nil, "not expected name: %v", fi.Name())
 				return fmt.Errorf("not expected name: %v", fi.Name())
 			}
 		} else if di, ok := li[0].(fs.DirEntry); ok {
 			if di.Remote() != "test/one/test2" {
-				log.Printf("not expected remote: %v", di.Remote())
+				fs.Logf(nil, "not expected remote: %v", di.Remote())
 				return fmt.Errorf("not expected remote: %v", di.Remote())
 			}
 		} else {
-			log.Printf("unexpected listing: %v", li)
+			fs.Logf(nil, "unexpected listing: %v", li)
 			return fmt.Errorf("unexpected listing: %v", li)
 		}
-		log.Printf("complete listing /test/one/test2")
+		fs.Logf(nil, "complete listing /test/one/test2")
 		return nil
 	}, 12, time.Second*10)
 	require.NoError(t, err)
@@ -708,7 +707,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 
 func TestInternalExpiredEntriesRemoved(t *testing.T) {
 	id := fmt.Sprintf("tieer%v", time.Now().Unix())
-	vfsflags.Opt.DirCacheTime = time.Second * 4 // needs to be lower than the defined
+	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 4) // needs to be lower than the defined
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, true, true, nil)
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
@@ -743,7 +742,7 @@ func TestInternalExpiredEntriesRemoved(t *testing.T) {
 }
 
 func TestInternalBug2117(t *testing.T) {
-	vfsflags.Opt.DirCacheTime = time.Second * 10
+	vfscommon.Opt.DirCacheTime = fs.Duration(time.Second * 10)
 
 	id := fmt.Sprintf("tib2117%v", time.Now().Unix())
 	rootFs, _ := runInstance.newCacheFs(t, remoteName, id, false, true, map[string]string{"info_age": "72h", "chunk_clean_interval": "15m"})
@@ -771,24 +770,24 @@ func TestInternalBug2117(t *testing.T) {
 
 	di, err := runInstance.list(t, rootFs, "test/dir1/dir2")
 	require.NoError(t, err)
-	log.Printf("len: %v", len(di))
+	fs.Logf(nil, "len: %v", len(di))
 	require.Len(t, di, 1)
 
 	time.Sleep(time.Second * 30)
 
 	di, err = runInstance.list(t, rootFs, "test/dir1/dir2")
 	require.NoError(t, err)
-	log.Printf("len: %v", len(di))
+	fs.Logf(nil, "len: %v", len(di))
 	require.Len(t, di, 1)
 
 	di, err = runInstance.list(t, rootFs, "test/dir1")
 	require.NoError(t, err)
-	log.Printf("len: %v", len(di))
+	fs.Logf(nil, "len: %v", len(di))
 	require.Len(t, di, 4)
 
 	di, err = runInstance.list(t, rootFs, "test")
 	require.NoError(t, err)
-	log.Printf("len: %v", len(di))
+	fs.Logf(nil, "len: %v", len(di))
 	require.Len(t, di, 4)
 }
 
@@ -829,7 +828,7 @@ func newRun() *run {
 	} else {
 		r.tmpUploadDir = uploadDir
 	}
-	log.Printf("Temp Upload Dir: %v", r.tmpUploadDir)
+	fs.Logf(nil, "Temp Upload Dir: %v", r.tmpUploadDir)
 
 	return r
 }
@@ -850,8 +849,8 @@ func (r *run) encryptRemoteIfNeeded(t *testing.T, remote string) string {
 func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool, flags map[string]string) (fs.Fs, *cache.Persistent) {
 	fstest.Initialise()
 	remoteExists := false
-	for _, s := range config.FileSections() {
-		if s == remote {
+	for _, s := range config.GetRemotes() {
+		if s.Name == remote {
 			remoteExists = true
 		}
 	}
@@ -875,12 +874,12 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	cacheRemote := remote
 	if !remoteExists {
 		localRemote := remote + "-local"
-		config.FileSet(localRemote, "type", "local")
-		config.FileSet(localRemote, "nounc", "true")
+		config.FileSetValue(localRemote, "type", "local")
+		config.FileSetValue(localRemote, "nounc", "true")
 		m.Set("type", "cache")
 		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
-		remoteType := config.FileGet(remote, "type")
+		remoteType := config.GetValue(remote, "type")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -891,14 +890,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				m.Set("password", cryptPassword1)
 				m.Set("password2", cryptPassword2)
 			}
-			remoteRemote := config.FileGet(remote, "remote")
+			remoteRemote := config.GetValue(remote, "remote")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := config.FileGet(remoteWrapping, "type")
+			remoteType := config.GetValue(remoteWrapping, "type")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -935,8 +934,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 
 	if purge {
-		_ = f.Features().Purge(context.Background(), "")
-		require.NoError(t, err)
+		_ = operations.Purge(context.Background(), f, "")
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
@@ -949,7 +947,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 }
 
 func (r *run) cleanupFs(t *testing.T, f fs.Fs) {
-	err := f.Features().Purge(context.Background(), "")
+	err := operations.Purge(context.Background(), f, "")
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
@@ -1193,7 +1191,7 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 func (r *run) cleanSize(t *testing.T, size int64) int64 {
 	if r.rootIsCrypt {
 		denominator := int64(65536 + 16)
-		size = size - 32
+		size -= 32
 		quotient := size / denominator
 		remainder := size % denominator
 		return (quotient*65536 + remainder - 16)

backend/cache/cache_test.go

@@ -1,7 +1,6 @@
 // Test Cache filesystem interface
 
 //go:build !plan9 && !js && !race
-// +build !plan9,!js,!race
 
 package cache_test
 
@@ -16,10 +15,11 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName:                   "TestCache:",
-		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
-		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata"},
-		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
+		RemoteName:                      "TestCache:",
+		NilObject:                       (*cache.Object)(nil),
+		UnimplementableFsMethods:        []string{"PublicLink", "OpenWriterAt", "OpenChunkWriter", "DirSetModTime", "MkdirMetadata"},
+		UnimplementableObjectMethods:    []string{"MimeType", "ID", "GetTier", "SetTier", "Metadata", "SetMetadata"},
+		UnimplementableDirectoryMethods: []string{"Metadata", "SetMetadata", "SetModTime"},
+		SkipInvalidUTF8:                 true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9 || js
-// +build plan9 js
 
+// Package cache implements a virtual provider to cache existing remotes.
 package cache

backend/cache/cache_upload_test.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js && !race
-// +build !plan9,!js,!race
 
 package cache_test
 

backend/cache/directory.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 package cache
 

backend/cache/handle.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 package cache
 
@@ -119,7 +118,7 @@ func (r *Handle) startReadWorkers() {
 	r.scaleWorkers(totalWorkers)
 }
 
-// scaleOutWorkers will increase the worker pool count by the provided amount
+// scaleWorkers will increase the worker pool count by the provided amount
 func (r *Handle) scaleWorkers(desired int) {
 	current := r.workers
 	if current == desired {
@@ -209,7 +208,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 	offset := chunkStart % int64(r.cacheFs().opt.ChunkSize)
 
 	// we align the start offset of the first chunk to a likely chunk in the storage
-	chunkStart = chunkStart - offset
+	chunkStart -= offset
 	r.queueOffset(chunkStart)
 	found := false
 
@@ -328,7 +327,7 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 
 	chunkStart := r.offset - (r.offset % int64(r.cacheFs().opt.ChunkSize))
 	if chunkStart >= int64(r.cacheFs().opt.ChunkSize) {
-		chunkStart = chunkStart - int64(r.cacheFs().opt.ChunkSize)
+		chunkStart -= int64(r.cacheFs().opt.ChunkSize)
 	}
 	r.queueOffset(chunkStart)
 
@@ -416,10 +415,8 @@ func (w *worker) run() {
 					continue
 				}
 			}
-		} else {
-			if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
-				continue
-			}
+		} else if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
+			continue
 		}
 
 		chunkEnd := chunkStart + int64(w.r.cacheFs().opt.ChunkSize)

backend/cache/object.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 package cache
 

backend/cache/plex.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 package cache
 

backend/cache/storage_memory.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 package cache
 

backend/cache/storage_persistent.go

@@ -1,5 +1,4 @@
 //go:build !plan9 && !js
-// +build !plan9,!js
 
 package cache
 

backend/cache/utils_test.go

@@ -1,3 +1,6 @@
+//go:build !plan9 && !js
+// +build !plan9,!js
+
 package cache
 
 import bolt "go.etcd.io/bbolt"

backend/chunker/chunker.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/lib/encoder"
 )
 
 // Chunker's composite files have one or more chunks
@@ -101,8 +102,10 @@ var (
 //
 // And still chunker's primary function is to chunk large files
 // rather than serve as a generic metadata container.
-const maxMetadataSize = 1023
-const maxMetadataSizeWritten = 255
+const (
+	maxMetadataSize        = 1023
+	maxMetadataSizeWritten = 255
+)
 
 // Current/highest supported metadata format.
 const metadataVersion = 2
@@ -305,7 +308,6 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		root: rpath,
 		opt:  *opt,
 	}
-	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
 	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType, opt.Transactions); err != nil {
@@ -317,26 +319,41 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 	// i.e. `rpath` does not exist in the wrapped remote, but chunker
 	// detects a composite file because it finds the first chunk!
 	// (yet can't satisfy fstest.CheckListing, will ignore)
-	if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
+	if err == nil && !f.useMeta {
 		firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
-		_, testErr := cache.Get(ctx, baseName+firstChunkPath)
+		newBase, testErr := cache.Get(ctx, baseName+firstChunkPath)
 		if testErr == fs.ErrorIsFile {
+			f.base = newBase
 			err = testErr
 		}
 	}
+	cache.PinUntilFinalized(f.base, f)
+
+	// Correct root if definitely pointing to a file
+	if err == fs.ErrorIsFile {
+		f.root = path.Dir(f.root)
+		if f.root == "." || f.root == "/" {
+			f.root = ""
+		}
+	}
 
 	// Note 1: the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs.
 	// Note 2: features.Fill() points features.PutStream to our PutStream,
 	// but features.Mask() will nullify it if wrappedFs does not have it.
 	f.features = (&fs.Features{
-		CaseInsensitive:         true,
-		DuplicateFiles:          true,
-		ReadMimeType:            false, // Object.MimeType not supported
-		WriteMimeType:           true,
-		BucketBased:             true,
-		CanHaveEmptyDirectories: true,
-		ServerSideAcrossConfigs: true,
+		CaseInsensitive:          true,
+		DuplicateFiles:           true,
+		ReadMimeType:             false, // Object.MimeType not supported
+		WriteMimeType:            true,
+		BucketBased:              true,
+		CanHaveEmptyDirectories:  true,
+		ServerSideAcrossConfigs:  true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          true,
+		DirModTimeUpdatesOnWrite: true,
 	}).Fill(ctx, f).Mask(ctx, baseFs).WrapsFs(f, baseFs)
 
 	f.features.Disable("ListR") // Recursive listing may cause chunker skip files
@@ -813,8 +830,7 @@ func (f *Fs) processEntries(ctx context.Context, origEntries fs.DirEntries, dirP
 			}
 		case fs.Directory:
 			isSubdir[entry.Remote()] = true
-			wrapDir := fs.NewDirCopy(ctx, entry)
-			wrapDir.SetRemote(entry.Remote())
+			wrapDir := fs.NewDirWrapper(entry.Remote(), entry)
 			tempEntries = append(tempEntries, wrapDir)
 		default:
 			if f.opt.FailHard {
@@ -947,6 +963,11 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 		}
 		if caseInsensitive {
 			sameMain = strings.EqualFold(mainRemote, remote)
+			if sameMain && f.base.Features().IsLocal {
+				// on local, make sure the EqualFold still holds true when accounting for encoding.
+				// sometimes paths with special characters will only normalize the same way in Standard Encoding.
+				sameMain = strings.EqualFold(encoder.OS.FromStandardPath(mainRemote), encoder.OS.FromStandardPath(remote))
+			}
 		} else {
 			sameMain = mainRemote == remote
 		}
@@ -960,13 +981,13 @@ func (f *Fs) scanObject(ctx context.Context, remote string, quickScan bool) (fs.
 			}
 			continue
 		}
-		//fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
+		// fs.Debugf(f, "%q belongs to %q as chunk %d", entryRemote, mainRemote, chunkNo)
 		if err := o.addChunk(entry, chunkNo); err != nil {
 			return nil, err
 		}
 	}
 
-	if o.main == nil && (o.chunks == nil || len(o.chunks) == 0) {
+	if o.main == nil && len(o.chunks) == 0 {
 		// Scanning hasn't found data chunks with conforming names.
 		if f.useMeta || quickScan {
 			// Metadata is required but absent and there are no chunks.
@@ -1122,8 +1143,8 @@ func (o *Object) readXactID(ctx context.Context) (xactID string, err error) {
 // put implements Put, PutStream, PutUnchecked, Update
 func (f *Fs) put(
 	ctx context.Context, in io.Reader, src fs.ObjectInfo, remote string, options []fs.OpenOption,
-	basePut putFn, action string, target fs.Object) (obj fs.Object, err error) {
-
+	basePut putFn, action string, target fs.Object,
+) (obj fs.Object, err error) {
 	// Perform consistency checks
 	if err := f.forbidChunk(src, remote); err != nil {
 		return nil, fmt.Errorf("%s refused: %w", action, err)
@@ -1563,6 +1584,14 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.base.Mkdir(ctx, dir)
 }
 
+// MkdirMetadata makes the root directory of the Fs object
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	if do := f.base.Features().MkdirMetadata; do != nil {
+		return do(ctx, dir, metadata)
+	}
+	return nil, fs.ErrorNotImplemented
+}
+
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -1880,6 +1909,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return do(ctx, srcFs.base, srcRemote, dstRemote)
 }
 
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	if do := f.base.Features().DirSetModTime; do != nil {
+		return do(ctx, dir, modTime)
+	}
+	return fs.ErrorNotImplemented
+}
+
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1928,7 +1965,7 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		return
 	}
 	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
-		//fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
+		// fs.Debugf(f, "ChangeNotify: path %q entryType %d", path, entryType)
 		if entryType == fs.EntryObject {
 			mainPath, _, _, xactID := f.parseChunkName(path)
 			metaXactID := ""
@@ -2540,6 +2577,8 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.DirSetModTimer  = (*Fs)(nil)
+	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -36,10 +36,12 @@ func TestIntegration(t *testing.T) {
 			"GetTier",
 			"SetTier",
 			"Metadata",
+			"SetMetadata",
 		},
 		UnimplementableFsMethods: []string{
 			"PublicLink",
 			"OpenWriterAt",
+			"OpenChunkWriter",
 			"MergeDirs",
 			"DirCacheFlush",
 			"UserInfo",

backend/combine/combine.go

@@ -222,18 +222,23 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
 	}
 	// check features
 	var features = (&fs.Features{
-		CaseInsensitive:         true,
-		DuplicateFiles:          false,
-		ReadMimeType:            true,
-		WriteMimeType:           true,
-		CanHaveEmptyDirectories: true,
-		BucketBased:             true,
-		SetTier:                 true,
-		GetTier:                 true,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            true,
-		PartialUploads:          true,
+		CaseInsensitive:          true,
+		DuplicateFiles:           false,
+		ReadMimeType:             true,
+		WriteMimeType:            true,
+		CanHaveEmptyDirectories:  true,
+		BucketBased:              true,
+		SetTier:                  true,
+		GetTier:                  true,
+		ReadMetadata:             true,
+		WriteMetadata:            true,
+		UserMetadata:             true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          true,
+		DirModTimeUpdatesOnWrite: true,
+		PartialUploads:           true,
 	}).Fill(ctx, f)
 	canMove := true
 	for _, u := range f.upstreams {
@@ -440,6 +445,32 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return u.f.Mkdir(ctx, uRemote)
 }
 
+// MkdirMetadata makes the root directory of the Fs object
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	u, uRemote, err := f.findUpstream(dir)
+	if err != nil {
+		return nil, err
+	}
+	do := u.f.Features().MkdirMetadata
+	if do == nil {
+		return nil, fs.ErrorNotImplemented
+	}
+	newDir, err := do(ctx, uRemote, metadata)
+	if err != nil {
+		return nil, err
+	}
+	entries := fs.DirEntries{newDir}
+	entries, err = u.wrapEntries(ctx, entries)
+	if err != nil {
+		return nil, err
+	}
+	newDir, ok := entries[0].(fs.Directory)
+	if !ok {
+		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
+	}
+	return newDir, nil
+}
+
 // purge the upstream or fallback to a slow way
 func (u *upstream) purge(ctx context.Context, dir string) (err error) {
 	if do := u.f.Features().Purge; do != nil {
@@ -755,12 +786,11 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D
 		case fs.Object:
 			entries[i] = u.newObject(x)
 		case fs.Directory:
-			newDir := fs.NewDirCopy(ctx, x)
-			newPath, err := u.pathAdjustment.do(newDir.Remote())
+			newPath, err := u.pathAdjustment.do(x.Remote())
 			if err != nil {
 				return nil, err
 			}
-			newDir.SetRemote(newPath)
+			newDir := fs.NewDirWrapper(newPath, x)
 			entries[i] = newDir
 		default:
 			return nil, fmt.Errorf("unknown entry type %T", entry)
@@ -783,7 +813,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if f.root == "" && dir == "" {
 		entries = make(fs.DirEntries, 0, len(f.upstreams))
 		for combineDir := range f.upstreams {
-			d := fs.NewDir(combineDir, f.when)
+			d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))
 			entries = append(entries, d)
 		}
 		return entries, nil
@@ -914,7 +944,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	return do(ctx, uRemote, expire, unlink)
 }
 
-// Put in to the remote path with the modTime given of the given size
+// PutUnchecked in to the remote path with the modTime given of the given size
 //
 // May create the object even if it returns an error - if so
 // will return the object and the error, otherwise will return
@@ -965,6 +995,22 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	return do(ctx, uDirs)
 }
 
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	u, uDir, err := f.findUpstream(dir)
+	if err != nil {
+		return err
+	}
+	if uDir == "" {
+		fs.Debugf(dir, "Can't set modtime on upstream root. skipping.")
+		return nil
+	}
+	if do := u.f.Features().DirSetModTime; do != nil {
+		return do(ctx, uDir, modTime)
+	}
+	return fs.ErrorNotImplemented
+}
+
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -1073,6 +1119,17 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
+// SetMetadata sets metadata for an Object
+//
+// It should return fs.ErrorNotImplemented if it can't set metadata
+func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
+	do, ok := o.Object.(fs.SetMetadataer)
+	if !ok {
+		return fs.ErrorNotImplemented
+	}
+	return do.SetMetadata(ctx, metadata)
+}
+
 // SetTier performs changing storage tier of the Object if
 // multiple storage classes supported
 func (o *Object) SetTier(tier string) error {
@@ -1099,6 +1156,8 @@ var (
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirSetModTimer  = (*Fs)(nil)
+	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.OpenWriterAter  = (*Fs)(nil)
 	_ fs.FullObject      = (*Object)(nil)

backend/combine/combine_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect"}
+	unimplementableFsMethods     = []string{"UnWrap", "WrapFs", "SetWrapper", "UserInfo", "Disconnect", "OpenChunkWriter"}
 	unimplementableObjectMethods = []string{}
 )
 

backend/compress/compress.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"regexp"
 	"strings"
 	"time"
@@ -37,6 +38,7 @@ import (
 const (
 	initialChunkSize = 262144  // Initial and max sizes of chunks when reading parts of the file. Currently
 	maxChunkSize     = 8388608 // at 256 KiB and 8 MiB.
+	chunkStreams     = 0       // Streams to use for reading
 
 	bufferSize          = 8388608
 	heuristicBytes      = 1048576
@@ -172,21 +174,33 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		opt:  *opt,
 		mode: compressionModeFromName(opt.CompressionMode),
 	}
+	// Correct root if definitely pointing to a file
+	if err == fs.ErrorIsFile {
+		f.root = path.Dir(f.root)
+		if f.root == "." || f.root == "/" {
+			f.root = ""
+		}
+	}
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:         true,
-		DuplicateFiles:          false,
-		ReadMimeType:            false,
-		WriteMimeType:           false,
-		GetTier:                 true,
-		SetTier:                 true,
-		BucketBased:             true,
-		CanHaveEmptyDirectories: true,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            true,
-		PartialUploads:          true,
+		CaseInsensitive:          true,
+		DuplicateFiles:           false,
+		ReadMimeType:             false,
+		WriteMimeType:            false,
+		GetTier:                  true,
+		SetTier:                  true,
+		BucketBased:              true,
+		CanHaveEmptyDirectories:  true,
+		ReadMetadata:             true,
+		WriteMetadata:            true,
+		UserMetadata:             true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          true,
+		DirModTimeUpdatesOnWrite: true,
+		PartialUploads:           true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 	// We support reading MIME types no matter the wrapped fs
 	f.features.ReadMimeType = true
@@ -257,6 +271,16 @@ func isMetadataFile(filename string) bool {
 	return strings.HasSuffix(filename, metaFileExt)
 }
 
+// Checks whether a file is a metadata file and returns the original
+// file name and a flag indicating whether it was a metadata file or
+// not.
+func unwrapMetadataFile(filename string) (string, bool) {
+	if !isMetadataFile(filename) {
+		return "", false
+	}
+	return filename[:len(filename)-len(metaFileExt)], true
+}
+
 // makeDataName generates the file name for a data file with specified compression mode
 func makeDataName(remote string, size int64, mode int) (newRemote string) {
 	if mode != Uncompressed {
@@ -432,7 +456,7 @@ func (f *Fs) verifyObjectHash(ctx context.Context, o fs.Object, hasher *hash.Mul
 		if err != nil {
 			fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 		}
-		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ %q vs %q", ht, srcHash, dstHash)
+		return fmt.Errorf("corrupted on transfer: %v compressed hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs, srcHash, o.Fs(), dstHash)
 	}
 	return nil
 }
@@ -766,6 +790,14 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.Fs.Mkdir(ctx, dir)
 }
 
+// MkdirMetadata makes the root directory of the Fs object
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	if do := f.Fs.Features().MkdirMetadata; do != nil {
+		return do(ctx, dir, metadata)
+	}
+	return nil, fs.ErrorNotImplemented
+}
+
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -909,6 +941,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return do(ctx, srcFs.Fs, srcRemote, dstRemote)
 }
 
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	if do := f.Fs.Features().DirSetModTime; do != nil {
+		return do(ctx, dir, modTime)
+	}
+	return fs.ErrorNotImplemented
+}
+
 // CleanUp the trash in the Fs
 //
 // Implement this if you have a way of emptying the trash or
@@ -979,7 +1019,8 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 	wrappedNotifyFunc := func(path string, entryType fs.EntryType) {
 		fs.Logf(f, "path %q entryType %d", path, entryType)
 		var (
-			wrappedPath string
+			wrappedPath    string
+			isMetadataFile bool
 		)
 		switch entryType {
 		case fs.EntryDirectory:
@@ -987,7 +1028,10 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 		case fs.EntryObject:
 			// Note: All we really need to do to monitor the object is to check whether the metadata changed,
 			// as the metadata contains the hash. This will work unless there's a hash collision and the sizes stay the same.
-			wrappedPath = makeMetadataName(path)
+			wrappedPath, isMetadataFile = unwrapMetadataFile(path)
+			if !isMetadataFile {
+				return
+			}
 		default:
 			fs.Errorf(path, "press ChangeNotify: ignoring unknown EntryType %d", entryType)
 			return
@@ -1243,6 +1287,17 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
+// SetMetadata sets metadata for an Object
+//
+// It should return fs.ErrorNotImplemented if it can't set metadata
+func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
+	do, ok := o.Object.(fs.SetMetadataer)
+	if !ok {
+		return fs.ErrorNotImplemented
+	}
+	return do.SetMetadata(ctx, metadata)
+}
+
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
@@ -1308,7 +1363,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 		}
 	}
 	// Get a chunkedreader for the wrapped object
-	chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize)
+	chunkedReader := chunkedreader.New(ctx, o.Object, initialChunkSize, maxChunkSize, chunkStreams)
 	// Get file handle
 	var file io.Reader
 	if offset != 0 {
@@ -1475,6 +1530,8 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.DirSetModTimer  = (*Fs)(nil)
+	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.UnWrapper       = (*Fs)(nil)

backend/compress/compress_test.go

@@ -14,23 +14,26 @@ import (
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
+var defaultOpt = fstests.Opt{
+	RemoteName: "TestCompress:",
+	NilObject:  (*Object)(nil),
+	UnimplementableFsMethods: []string{
+		"OpenWriterAt",
+		"OpenChunkWriter",
+		"MergeDirs",
+		"DirCacheFlush",
+		"PutUnchecked",
+		"PutStream",
+		"UserInfo",
+		"Disconnect",
+	},
+	TiersToTest:                  []string{"STANDARD", "STANDARD_IA"},
+	UnimplementableObjectMethods: []string{},
+}
+
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
-	opt := fstests.Opt{
-		RemoteName: *fstest.RemoteName,
-		NilObject:  (*Object)(nil),
-		UnimplementableFsMethods: []string{
-			"OpenWriterAt",
-			"MergeDirs",
-			"DirCacheFlush",
-			"PutUnchecked",
-			"PutStream",
-			"UserInfo",
-			"Disconnect",
-		},
-		TiersToTest:                  []string{"STANDARD", "STANDARD_IA"},
-		UnimplementableObjectMethods: []string{}}
-	fstests.Run(t, &opt)
+	fstests.Run(t, &defaultOpt)
 }
 
 // TestRemoteGzip tests GZIP compression
@@ -40,27 +43,13 @@ func TestRemoteGzip(t *testing.T) {
 	}
 	tempdir := filepath.Join(os.TempDir(), "rclone-compress-test-gzip")
 	name := "TestCompressGzip"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		NilObject:  (*Object)(nil),
-		UnimplementableFsMethods: []string{
-			"OpenWriterAt",
-			"MergeDirs",
-			"DirCacheFlush",
-			"PutUnchecked",
-			"PutStream",
-			"UserInfo",
-			"Disconnect",
-		},
-		UnimplementableObjectMethods: []string{
-			"GetTier",
-			"SetTier",
-		},
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "compress"},
-			{Name: name, Key: "remote", Value: tempdir},
-			{Name: name, Key: "compression_mode", Value: "gzip"},
-		},
-		QuickTestOK: true,
-	})
+	opt := defaultOpt
+	opt.RemoteName = name + ":"
+	opt.ExtraConfig = []fstests.ExtraConfigItem{
+		{Name: name, Key: "type", Value: "compress"},
+		{Name: name, Key: "remote", Value: tempdir},
+		{Name: name, Key: "compression_mode", Value: "gzip"},
+	}
+	opt.QuickTestOK = true
+	fstests.Run(t, &opt)
 }

backend/crypt/cipher.go

@@ -329,7 +329,7 @@ func (c *Cipher) obfuscateSegment(plaintext string) string {
 	for _, runeValue := range plaintext {
 		dir += int(runeValue)
 	}
-	dir = dir % 256
+	dir %= 256
 
 	// We'll use this number to store in the result filename...
 	var result bytes.Buffer
@@ -450,7 +450,7 @@ func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if pos >= 26 {
 				pos -= 6
 			}
-			pos = pos - thisdir
+			pos -= thisdir
 			if pos < 0 {
 				pos += 52
 			}
@@ -888,7 +888,7 @@ func (fh *decrypter) fillBuffer() (err error) {
 		fs.Errorf(nil, "crypt: ignoring: %v", ErrorEncryptedBadBlock)
 		// Zero out the bad block and continue
 		for i := range (*fh.buf)[:n] {
-			(*fh.buf)[i] = 0
+			fh.buf[i] = 0
 		}
 	}
 	fh.bufIndex = 0

backend/crypt/crypt.go

@@ -130,6 +130,16 @@ trying to recover an encrypted file with errors and it is desired to
 recover as much of the file as possible.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "strict_names",
+			Help: `If set, this will raise an error when crypt comes across a filename that can't be decrypted.
+
+(By default, rclone will just log a NOTICE and continue as normal.)
+This can happen if encrypted and unencrypted files are stored in the same
+directory (which is not recommended.) It may also indicate a more serious
+problem that should be investigated.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name: "filename_encoding",
 			Help: `How to encode the encrypted filename to text string.
@@ -253,22 +263,34 @@ func NewFs(ctx context.Context, name, rpath string, m configmap.Mapper) (fs.Fs,
 		cipher: cipher,
 	}
 	cache.PinUntilFinalized(f.Fs, f)
+	// Correct root if definitely pointing to a file
+	if err == fs.ErrorIsFile {
+		f.root = path.Dir(f.root)
+		if f.root == "." || f.root == "/" {
+			f.root = ""
+		}
+	}
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
-		CaseInsensitive:         !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
-		DuplicateFiles:          true,
-		ReadMimeType:            false, // MimeTypes not supported with crypt
-		WriteMimeType:           false,
-		BucketBased:             true,
-		CanHaveEmptyDirectories: true,
-		SetTier:                 true,
-		GetTier:                 true,
-		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            true,
-		PartialUploads:          true,
+		CaseInsensitive:          !cipher.dirNameEncrypt || cipher.NameEncryptionMode() == NameEncryptionOff,
+		DuplicateFiles:           true,
+		ReadMimeType:             false, // MimeTypes not supported with crypt
+		WriteMimeType:            false,
+		BucketBased:              true,
+		CanHaveEmptyDirectories:  true,
+		SetTier:                  true,
+		GetTier:                  true,
+		ServerSideAcrossConfigs:  opt.ServerSideAcrossConfigs,
+		ReadMetadata:             true,
+		WriteMetadata:            true,
+		UserMetadata:             true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          true,
+		DirModTimeUpdatesOnWrite: true,
+		PartialUploads:           true,
 	}).Fill(ctx, f).Mask(ctx, wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -287,6 +309,7 @@ type Options struct {
 	PassBadBlocks           bool   `config:"pass_bad_blocks"`
 	FilenameEncoding        string `config:"filename_encoding"`
 	Suffix                  string `config:"suffix"`
+	StrictNames             bool   `config:"strict_names"`
 }
 
 // Fs represents a wrapped fs.Fs
@@ -321,45 +344,64 @@ func (f *Fs) String() string {
 }
 
 // Encrypt an object file name to entries.
-func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
+func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) error {
 	remote := obj.Remote()
 	decryptedRemote, err := f.cipher.DecryptFileName(remote)
 	if err != nil {
-		fs.Debugf(remote, "Skipping undecryptable file name: %v", err)
-		return
+		if f.opt.StrictNames {
+			return fmt.Errorf("%s: undecryptable file name detected: %v", remote, err)
+		}
+		fs.Logf(remote, "Skipping undecryptable file name: %v", err)
+		return nil
 	}
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
 	*entries = append(*entries, f.newObject(obj))
+	return nil
 }
 
 // Encrypt a directory file name to entries.
-func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
+func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) error {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
-		fs.Debugf(remote, "Skipping undecryptable dir name: %v", err)
-		return
+		if f.opt.StrictNames {
+			return fmt.Errorf("%s: undecryptable dir name detected: %v", remote, err)
+		}
+		fs.Logf(remote, "Skipping undecryptable dir name: %v", err)
+		return nil
 	}
 	if f.opt.ShowMapping {
 		fs.Logf(decryptedRemote, "Encrypts to %q", remote)
 	}
 	*entries = append(*entries, f.newDir(ctx, dir))
+	return nil
 }
 
 // Encrypt some directory entries.  This alters entries returning it as newEntries.
 func (f *Fs) encryptEntries(ctx context.Context, entries fs.DirEntries) (newEntries fs.DirEntries, err error) {
 	newEntries = entries[:0] // in place filter
+	errors := 0
+	var firsterr error
 	for _, entry := range entries {
 		switch x := entry.(type) {
 		case fs.Object:
-			f.add(&newEntries, x)
+			err = f.add(&newEntries, x)
 		case fs.Directory:
-			f.addDir(ctx, &newEntries, x)
+			err = f.addDir(ctx, &newEntries, x)
 		default:
 			return nil, fmt.Errorf("unknown object type %T", entry)
 		}
+		if err != nil {
+			errors++
+			if firsterr == nil {
+				firsterr = err
+			}
+		}
+	}
+	if firsterr != nil {
+		return nil, fmt.Errorf("there were %v undecryptable name errors. first error: %v", errors, firsterr)
 	}
 	return newEntries, nil
 }
@@ -478,7 +520,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 				if err != nil {
 					fs.Errorf(o, "Failed to remove corrupted object: %v", err)
 				}
-				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hash differ src %q vs dst %q", ht, srcHash, dstHash)
+				return nil, fmt.Errorf("corrupted on transfer: %v encrypted hashes differ src(%s) %q vs dst(%s) %q", ht, f.Fs, srcHash, o.Fs(), dstHash)
 			}
 			fs.Debugf(src, "%v = %s OK", ht, srcHash)
 		}
@@ -513,6 +555,37 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return f.Fs.Mkdir(ctx, f.cipher.EncryptDirName(dir))
 }
 
+// MkdirMetadata makes the root directory of the Fs object
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	do := f.Fs.Features().MkdirMetadata
+	if do == nil {
+		return nil, fs.ErrorNotImplemented
+	}
+	newDir, err := do(ctx, f.cipher.EncryptDirName(dir), metadata)
+	if err != nil {
+		return nil, err
+	}
+	var entries = make(fs.DirEntries, 0, 1)
+	err = f.addDir(ctx, &entries, newDir)
+	if err != nil {
+		return nil, err
+	}
+	newDir, ok := entries[0].(fs.Directory)
+	if !ok {
+		return nil, fmt.Errorf("internal error: expecting %T to be fs.Directory", entries[0])
+	}
+	return newDir, nil
+}
+
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	do := f.Fs.Features().DirSetModTime
+	if do == nil {
+		return fs.ErrorNotImplemented
+	}
+	return do(ctx, f.cipher.EncryptDirName(dir), modTime)
+}
+
 // Rmdir removes the directory (container, bucket) if empty
 //
 // Return an error if it doesn't exist or isn't empty
@@ -754,7 +827,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	}
 	out := make([]fs.Directory, len(dirs))
 	for i, dir := range dirs {
-		out[i] = fs.NewDirCopy(ctx, dir).SetRemote(f.cipher.EncryptDirName(dir.Remote()))
+		out[i] = fs.NewDirWrapper(f.cipher.EncryptDirName(dir.Remote()), dir)
 	}
 	return do(ctx, out)
 }
@@ -990,14 +1063,14 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // newDir returns a dir with the Name decrypted
 func (f *Fs) newDir(ctx context.Context, dir fs.Directory) fs.Directory {
-	newDir := fs.NewDirCopy(ctx, dir)
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
 	if err != nil {
 		fs.Debugf(remote, "Undecryptable dir name: %v", err)
 	} else {
-		newDir.SetRemote(decryptedRemote)
+		remote = decryptedRemote
 	}
+	newDir := fs.NewDirWrapper(remote, dir)
 	return newDir
 }
 
@@ -1175,6 +1248,17 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
+// SetMetadata sets metadata for an Object
+//
+// It should return fs.ErrorNotImplemented if it can't set metadata
+func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
+	do, ok := o.Object.(fs.SetMetadataer)
+	if !ok {
+		return fs.ErrorNotImplemented
+	}
+	return do.SetMetadata(ctx, metadata)
+}
+
 // MimeType returns the content type of the Object if
 // known, or "" if not
 //
@@ -1200,6 +1284,8 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Wrapper         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirSetModTimer  = (*Fs)(nil)
+	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)

backend/crypt/crypt_test.go

@@ -24,7 +24,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   *fstest.RemoteName,
 		NilObject:                    (*crypt.Object)(nil),
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
@@ -45,7 +45,7 @@ func TestStandardBase32(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -67,7 +67,7 @@ func TestStandardBase64(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 			{Name: name, Key: "filename_encoding", Value: "base64"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -89,7 +89,7 @@ func TestStandardBase32768(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "standard"},
 			{Name: name, Key: "filename_encoding", Value: "base32768"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -111,7 +111,7 @@ func TestOff(t *testing.T) {
 			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
 			{Name: name, Key: "filename_encryption", Value: "off"},
 		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -137,7 +137,7 @@ func TestObfuscate(t *testing.T) {
 			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
 		},
 		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})
@@ -164,7 +164,7 @@ func TestNoDataObfuscate(t *testing.T) {
 			{Name: name, Key: "no_data_encryption", Value: "true"},
 		},
 		SkipBadWindowsCharacters:     true,
-		UnimplementableFsMethods:     []string{"OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"OpenWriterAt", "OpenChunkWriter"},
 		UnimplementableObjectMethods: []string{"MimeType"},
 		QuickTestOK:                  true,
 	})

backend/drive/drive.go

@@ -71,7 +71,7 @@ const (
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	minChunkSize     = fs.SizeSuffix(googleapi.MinUploadChunkSize)
 	defaultChunkSize = 8 * fs.Mebi
-	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
+	partialFields    = "id,name,size,md5Checksum,sha1Checksum,sha256Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
 	listRGrouping    = 50   // number of IDs to search at once when using ListR
 	listRInputBuffer = 1000 // size of input buffer when using ListR
 	defaultXDGIcon   = "text-html"
@@ -143,6 +143,46 @@ var (
 	_linkTemplates   map[string]*template.Template // available link types
 )
 
+// rwChoices type for fs.Bits
+type rwChoices struct{}
+
+func (rwChoices) Choices() []fs.BitsChoicesInfo {
+	return []fs.BitsChoicesInfo{
+		{Bit: uint64(rwOff), Name: "off"},
+		{Bit: uint64(rwRead), Name: "read"},
+		{Bit: uint64(rwWrite), Name: "write"},
+		{Bit: uint64(rwFailOK), Name: "failok"},
+	}
+}
+
+// rwChoice type alias
+type rwChoice = fs.Bits[rwChoices]
+
+const (
+	rwRead rwChoice = 1 << iota
+	rwWrite
+	rwFailOK
+	rwOff rwChoice = 0
+)
+
+// Examples for the options
+var rwExamples = fs.OptionExamples{{
+	Value: rwOff.String(),
+	Help:  "Do not read or write the value",
+}, {
+	Value: rwRead.String(),
+	Help:  "Read the value only",
+}, {
+	Value: rwWrite.String(),
+	Help:  "Write the value only",
+}, {
+	Value: rwFailOK.String(),
+	Help:  "If writing fails log errors only, don't fail the transfer",
+}, {
+	Value: (rwRead | rwWrite).String(),
+	Help:  "Read and Write the value.",
+}}
+
 // Parse the scopes option returning a slice of scopes
 func driveScopes(scopesString string) (scopes []string) {
 	if scopesString == "" {
@@ -250,9 +290,16 @@ func init() {
 			}
 			return nil, fmt.Errorf("unknown state %q", config.State)
 		},
+		MetadataInfo: &fs.MetadataInfo{
+			System: systemMetadataInfo,
+			Help: `User metadata is stored in the properties field of the drive object.
+
+Metadata is supported on files and directories.
+`,
+		},
 		Options: append(driveOAuthOptions(), []fs.Option{{
 			Name: "scope",
-			Help: "Scope that rclone should use when requesting access from drive.",
+			Help: "Comma separated list of scopes that rclone should use when requesting access from drive.",
 			Examples: []fs.OptionExample{{
 				Value: "drive",
 				Help:  "Full access all files, excluding Application Data Folder.",
@@ -320,16 +367,35 @@ rather than shortcuts themselves when doing server side copies.`,
 			Default:  false,
 			Help:     "Skip google documents in all listings.\n\nIf given, gdocs practically become invisible to rclone.",
 			Advanced: true,
+		}, {
+			Name:    "show_all_gdocs",
+			Default: false,
+			Help: `Show all Google Docs including non-exportable ones in listings.
+
+If you try a server side copy on a Google Form without this flag, you
+will get this error:
+
+    No export formats found for "application/vnd.google-apps.form"
+
+However adding this flag will allow the form to be server side copied.
+
+Note that rclone doesn't add extensions to the Google Docs file names
+in this mode.
+
+Do **not** use this flag when trying to download Google Docs - rclone
+will fail to download them.
+`,
+			Advanced: true,
 		}, {
 			Name:    "skip_checksum_gphotos",
 			Default: false,
-			Help: `Skip MD5 checksum on Google photos and videos only.
+			Help: `Skip checksums on Google photos and videos only.
 
 Use this if you get checksum errors when transferring Google photos or
 videos.
 
 Setting this flag will cause Google photos and videos to return a
-blank MD5 checksum.
+blank checksums.
 
 Google photos are identified by being in the "photos" space.
 
@@ -594,10 +660,82 @@ This resource key requirement only applies to a subset of old files.
 
 Note also that opening the folder once in the web interface (with the
 user you've authenticated rclone with) seems to be enough so that the
-resource key is no needed.
+resource key is not needed.
 `,
 			Advanced:  true,
 			Sensitive: true,
+		}, {
+			Name: "fast_list_bug_fix",
+			Help: `Work around a bug in Google Drive listing.
+
+Normally rclone will work around a bug in Google Drive when using
+--fast-list (ListR) where the search "(A in parents) or (B in
+parents)" returns nothing sometimes. See #3114, #4289 and
+https://issuetracker.google.com/issues/149522397
+
+Rclone detects this by finding no items in more than one directory
+when listing and retries them as lists of individual directories.
+
+This means that if you have a lot of empty directories rclone will end
+up listing them all individually and this can take many more API
+calls.
+
+This flag allows the work-around to be disabled. This is **not**
+recommended in normal use - only if you have a particular case you are
+having trouble with like many empty directories.
+`,
+			Advanced: true,
+			Default:  true,
+		}, {
+			Name: "metadata_owner",
+			Help: `Control whether owner should be read or written in metadata.
+
+Owner is a standard part of the file metadata so is easy to read. But it
+isn't always desirable to set the owner from the metadata.
+
+Note that you can't set the owner on Shared Drives, and that setting
+ownership will generate an email to the new owner (this can't be
+disabled), and you can't transfer ownership to someone outside your
+organization.
+`,
+			Advanced: true,
+			Default:  rwRead,
+			Examples: rwExamples,
+		}, {
+			Name: "metadata_permissions",
+			Help: `Control whether permissions should be read or written in metadata.
+
+Reading permissions metadata from files can be done quickly, but it
+isn't always desirable to set the permissions from the metadata.
+
+Note that rclone drops any inherited permissions on Shared Drives and
+any owner permission on My Drives as these are duplicated in the owner
+metadata.
+`,
+			Advanced: true,
+			Default:  rwOff,
+			Examples: rwExamples,
+		}, {
+			Name: "metadata_labels",
+			Help: `Control whether labels should be read or written in metadata.
+
+Reading labels metadata from files takes an extra API transaction and
+will slow down listings. It isn't always desirable to set the labels
+from the metadata.
+
+The format of labels is documented in the drive API documentation at
+https://developers.google.com/drive/api/reference/rest/v3/Label -
+rclone just provides a JSON dump of this format.
+
+When setting labels, the label and fields must already exist - rclone
+will not create them. This means that if you are transferring labels
+from two different accounts you will have to create the labels in
+advance and use the metadata mapper to translate the IDs between the
+two accounts.
+`,
+			Advanced: true,
+			Default:  rwOff,
+			Examples: rwExamples,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -645,6 +783,7 @@ type Options struct {
 	UseTrash                  bool                 `config:"use_trash"`
 	CopyShortcutContent       bool                 `config:"copy_shortcut_content"`
 	SkipGdocs                 bool                 `config:"skip_gdocs"`
+	ShowAllGdocs              bool                 `config:"show_all_gdocs"`
 	SkipChecksumGphotos       bool                 `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool                 `config:"shared_with_me"`
 	TrashedOnly               bool                 `config:"trashed_only"`
@@ -672,6 +811,10 @@ type Options struct {
 	SkipShortcuts             bool                 `config:"skip_shortcuts"`
 	SkipDanglingShortcuts     bool                 `config:"skip_dangling_shortcuts"`
 	ResourceKey               string               `config:"resource_key"`
+	FastListBugFix            bool                 `config:"fast_list_bug_fix"`
+	MetadataOwner             rwChoice             `config:"metadata_owner"`
+	MetadataPermissions       rwChoice             `config:"metadata_permissions"`
+	MetadataLabels            rwChoice             `config:"metadata_labels"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 	EnvAuth                   bool                 `config:"env_auth"`
 }
@@ -693,23 +836,25 @@ type Fs struct {
 	exportExtensions []string           // preferred extensions to download docs
 	importMimeTypes  []string           // MIME types to convert to docs
 	isTeamDrive      bool               // true if this is a team drive
-	fileFields       googleapi.Field    // fields to fetch file info with
 	m                configmap.Mapper
-	grouping         int32               // number of IDs to search at once in ListR - read with atomic
-	listRmu          *sync.Mutex         // protects listRempties
-	listRempties     map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
-	dirResourceKeys  *sync.Map           // map directory ID to resource key
+	grouping         int32                        // number of IDs to search at once in ListR - read with atomic
+	listRmu          *sync.Mutex                  // protects listRempties
+	listRempties     map[string]struct{}          // IDs of supposedly empty directories which triggered grouping disable
+	dirResourceKeys  *sync.Map                    // map directory ID to resource key
+	permissionsMu    *sync.Mutex                  // protect the below
+	permissions      map[string]*drive.Permission // map permission IDs to Permissions
 }
 
 type baseObject struct {
-	fs           *Fs      // what this object is part of
-	remote       string   // The remote path
-	id           string   // Drive Id of this object
-	modifiedDate string   // RFC3339 time it was last modified
-	mimeType     string   // The object MIME type
-	bytes        int64    // size of the object
-	parents      []string // IDs of the parent directories
-	resourceKey  *string  // resourceKey is needed for link shared objects
+	fs           *Fs          // what this object is part of
+	remote       string       // The remote path
+	id           string       // Drive Id of this object
+	modifiedDate string       // RFC3339 time it was last modified
+	mimeType     string       // The object MIME type
+	bytes        int64        // size of the object
+	parents      []string     // IDs of the parent directories
+	resourceKey  *string      // resourceKey is needed for link shared objects
+	metadata     *fs.Metadata // metadata if known
 }
 type documentObject struct {
 	baseObject
@@ -728,9 +873,16 @@ type Object struct {
 	baseObject
 	url        string // Download URL of this object
 	md5sum     string // md5sum of the object
+	sha1sum    string // sha1sum of the object
+	sha256sum  string // sha256sum of the object
 	v2Download bool   // generate v2 download link ondemand
 }
 
+// Directory describes a drive directory
+type Directory struct {
+	baseObject
+}
+
 // ------------------------------------------------------------
 
 // Name of the remote (as passed into NewFs)
@@ -956,7 +1108,7 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		list.Header().Add("X-Goog-Drive-Resource-Keys", resourceKeysHeader)
 	}
 
-	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.fileFields)
+	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
 
 OUTER:
 	for {
@@ -1230,16 +1382,25 @@ func newFs(ctx context.Context, name, path string, m configmap.Mapper) (*Fs, err
 		listRmu:         new(sync.Mutex),
 		listRempties:    make(map[string]struct{}),
 		dirResourceKeys: new(sync.Map),
+		permissionsMu:   new(sync.Mutex),
+		permissions:     make(map[string]*drive.Permission),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
-	f.fileFields = f.getFileFields()
 	f.features = (&fs.Features{
-		DuplicateFiles:          true,
-		ReadMimeType:            true,
-		WriteMimeType:           true,
-		CanHaveEmptyDirectories: true,
-		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
-		FilterAware:             true,
+		DuplicateFiles:           true,
+		ReadMimeType:             true,
+		WriteMimeType:            true,
+		CanHaveEmptyDirectories:  true,
+		ServerSideAcrossConfigs:  opt.ServerSideAcrossConfigs,
+		FilterAware:              true,
+		ReadMetadata:             true,
+		WriteMetadata:            true,
+		UserMetadata:             true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          true,
+		DirModTimeUpdatesOnWrite: false, // FIXME need to check!
 	}).Fill(ctx, f)
 
 	// Create a new authorized Drive client.
@@ -1344,7 +1505,7 @@ func NewFs(ctx context.Context, name, path string, m configmap.Mapper) (fs.Fs, e
 	return f, nil
 }
 
-func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
+func (f *Fs) newBaseObject(ctx context.Context, remote string, info *drive.File) (o baseObject, err error) {
 	modifiedDate := info.ModifiedTime
 	if f.opt.UseCreatedDate {
 		modifiedDate = info.CreatedTime
@@ -1355,7 +1516,7 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 	if f.opt.SizeAsQuota {
 		size = info.QuotaBytesUsed
 	}
-	return baseObject{
+	o = baseObject{
 		fs:           f,
 		remote:       remote,
 		id:           info.Id,
@@ -1364,10 +1525,15 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 		bytes:        size,
 		parents:      info.Parents,
 	}
+	err = nil
+	if fs.GetConfig(ctx).Metadata {
+		err = o.parseMetadata(ctx, info)
+	}
+	return o, err
 }
 
 // getFileFields gets the fields for a normal file Get or List
-func (f *Fs) getFileFields() (fields googleapi.Field) {
+func (f *Fs) getFileFields(ctx context.Context) (fields googleapi.Field) {
 	fields = partialFields
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
@@ -1381,40 +1547,53 @@ func (f *Fs) getFileFields() (fields googleapi.Field) {
 	if f.opt.SizeAsQuota {
 		fields += ",quotaBytesUsed"
 	}
+	if fs.GetConfig(ctx).Metadata {
+		fields += "," + metadataFields
+	}
 	return fields
 }
 
 // newRegularObject creates an fs.Object for a normal drive.File
-func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
+func (f *Fs) newRegularObject(ctx context.Context, remote string, info *drive.File) (obj fs.Object, err error) {
 	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
 	if f.opt.SkipChecksumGphotos {
 		for _, space := range info.Spaces {
 			if space == "photos" {
 				info.Md5Checksum = ""
+				info.Sha1Checksum = ""
+				info.Sha256Checksum = ""
 				break
 			}
 		}
 	}
 	o := &Object{
-		baseObject: f.newBaseObject(remote, info),
 		url:        fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
 		md5sum:     strings.ToLower(info.Md5Checksum),
+		sha1sum:    strings.ToLower(info.Sha1Checksum),
+		sha256sum:  strings.ToLower(info.Sha256Checksum),
 		v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
 	}
+	o.baseObject, err = f.newBaseObject(ctx, remote, info)
+	if err != nil {
+		return nil, err
+	}
 	if info.ResourceKey != "" {
 		o.resourceKey = &info.ResourceKey
 	}
-	return o
+	return o, nil
 }
 
 // newDocumentObject creates an fs.Object for a google docs drive.File
-func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
+func (f *Fs) newDocumentObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
 	mediaType, _, err := mime.ParseMediaType(exportMimeType)
 	if err != nil {
 		return nil, err
 	}
 	url := info.ExportLinks[mediaType]
-	baseObject := f.newBaseObject(remote+extension, info)
+	baseObject, err := f.newBaseObject(ctx, remote+extension, info)
+	if err != nil {
+		return nil, err
+	}
 	baseObject.bytes = -1
 	baseObject.mimeType = exportMimeType
 	return &documentObject{
@@ -1426,7 +1605,7 @@ func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, expor
 }
 
 // newLinkObject creates an fs.Object that represents a link a google docs drive.File
-func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
+func (f *Fs) newLinkObject(ctx context.Context, remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
 	t := linkTemplate(exportMimeType)
 	if t == nil {
 		return nil, fmt.Errorf("unsupported link type %s", exportMimeType)
@@ -1445,7 +1624,10 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
 		return nil, fmt.Errorf("executing template failed: %w", err)
 	}
 
-	baseObject := f.newBaseObject(remote+extension, info)
+	baseObject, err := f.newBaseObject(ctx, remote+extension, info)
+	if err != nil {
+		return nil, err
+	}
 	baseObject.bytes = int64(buf.Len())
 	baseObject.mimeType = exportMimeType
 	return &linkObject{
@@ -1461,7 +1643,7 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
 func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *drive.File) (fs.Object, error) {
 	// If item has MD5 sum it is a file stored on drive
 	if info.Md5Checksum != "" {
-		return f.newRegularObject(remote, info), nil
+		return f.newRegularObject(ctx, remote, info)
 	}
 
 	extension, exportName, exportMimeType, isDocument := f.findExportFormat(ctx, info)
@@ -1492,13 +1674,15 @@ func (f *Fs) newObjectWithExportInfo(
 	case info.MimeType == shortcutMimeTypeDangling:
 		// Pretend a dangling shortcut is a regular object
 		// It will error if used, but appear in listings so it can be deleted
-		return f.newRegularObject(remote, info), nil
+		return f.newRegularObject(ctx, remote, info)
 	case info.Md5Checksum != "":
 		// If item has MD5 sum it is a file stored on drive
-		return f.newRegularObject(remote, info), nil
+		return f.newRegularObject(ctx, remote, info)
 	case f.opt.SkipGdocs:
 		fs.Debugf(remote, "Skipping google document type %q", info.MimeType)
 		return nil, fs.ErrorObjectNotFound
+	case f.opt.ShowAllGdocs:
+		return f.newDocumentObject(ctx, remote, info, "", info.MimeType)
 	default:
 		// If item MimeType is in the ExportFormats then it is a google doc
 		if !isDocument {
@@ -1510,9 +1694,9 @@ func (f *Fs) newObjectWithExportInfo(
 			return nil, fs.ErrorObjectNotFound
 		}
 		if isLinkMimeType(exportMimeType) {
-			return f.newLinkObject(remote, info, extension, exportMimeType)
+			return f.newLinkObject(ctx, remote, info, extension, exportMimeType)
 		}
-		return f.newDocumentObject(remote, info, extension, exportMimeType)
+		return f.newDocumentObject(ctx, remote, info, extension, exportMimeType)
 	}
 }
 
@@ -1563,26 +1747,72 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	return pathIDOut, found, err
 }
 
-// CreateDir makes a directory with pathID as parent and name leaf
-func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+// createDir makes a directory with pathID as parent and name leaf with optional metadata
+func (f *Fs) createDir(ctx context.Context, pathID, leaf string, metadata fs.Metadata) (info *drive.File, err error) {
 	leaf = f.opt.Enc.FromStandardName(leaf)
-	// fmt.Println("Making", path)
-	// Define the metadata for the directory we are going to create.
 	pathID = actualID(pathID)
 	createInfo := &drive.File{
-		Name:        leaf,
-		Description: leaf,
-		MimeType:    driveFolderType,
-		Parents:     []string{pathID},
+		Name:     leaf,
+		MimeType: driveFolderType,
+		Parents:  []string{pathID},
+	}
+	var updateMetadata updateMetadataFn
+	if len(metadata) > 0 {
+		updateMetadata, err = f.updateMetadata(ctx, createInfo, metadata, true)
+		if err != nil {
+			return nil, fmt.Errorf("create dir: failed to update metadata: %w", err)
+		}
 	}
-	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Create(createInfo).
-			Fields("id").
+			Fields(f.getFileFields(ctx)).
 			SupportsAllDrives(true).
 			Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
 	})
+	if err != nil {
+		return nil, err
+	}
+	if updateMetadata != nil {
+		err = updateMetadata(ctx, info)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return info, nil
+}
+
+// updateDir updates an existing a directory with the metadata passed in
+func (f *Fs) updateDir(ctx context.Context, dirID string, metadata fs.Metadata) (info *drive.File, err error) {
+	if len(metadata) == 0 {
+		return f.getFile(ctx, dirID, f.getFileFields(ctx))
+	}
+	dirID = actualID(dirID)
+	updateInfo := &drive.File{}
+	updateMetadata, err := f.updateMetadata(ctx, updateInfo, metadata, true)
+	if err != nil {
+		return nil, fmt.Errorf("update dir: failed to update metadata from source object: %w", err)
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		info, err = f.svc.Files.Update(dirID, updateInfo).
+			Fields(f.getFileFields(ctx)).
+			SupportsAllDrives(true).
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+	err = updateMetadata(ctx, info)
+	if err != nil {
+		return nil, err
+	}
+	return info, nil
+}
+
+// CreateDir makes a directory with pathID as parent and name leaf
+func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+	info, err := f.createDir(ctx, pathID, leaf, nil)
 	if err != nil {
 		return "", err
 	}
@@ -1693,7 +1923,7 @@ func (f *Fs) findExportFormatByMimeType(ctx context.Context, itemMimeType string
 	return "", "", isDocument
 }
 
-// findExportFormatByMimeType works out the optimum export settings
+// findExportFormat works out the optimum export settings
 // for the given drive.File.
 //
 // Look through the exportExtensions and find the first format that can be
@@ -1891,7 +2121,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		// drive where (A in parents) or (B in parents) returns nothing
 		// sometimes. See #3114, #4289 and
 		// https://issuetracker.google.com/issues/149522397
-		if len(dirs) > 1 && !foundItems {
+		if f.opt.FastListBugFix && len(dirs) > 1 && !foundItems {
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
 				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
@@ -1989,13 +2219,13 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		case in <- job:
 		default:
 			overflow = append(overflow, job)
-			wg.Add(-1)
+			wg.Done()
 		}
 	}
 
 	// Send the entry to the caller, queueing any directories as new jobs
 	cb := func(entry fs.DirEntry) error {
-		if d, isDir := entry.(*fs.Dir); isDir {
+		if d, isDir := entry.(fs.Directory); isDir {
 			job := listREntry{actualID(d.ID()), d.Remote()}
 			sendJob(job)
 		}
@@ -2140,7 +2370,7 @@ func (f *Fs) resolveShortcut(ctx context.Context, item *drive.File) (newItem *dr
 		fs.Errorf(nil, "Expecting shortcutDetails in %v", item)
 		return item, nil
 	}
-	newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.fileFields)
+	newItem, err = f.getFile(ctx, item.ShortcutDetails.TargetId, f.getFileFields(ctx))
 	if err != nil {
 		var gerr *googleapi.Error
 		if errors.As(err, &gerr) && gerr.Code == 404 {
@@ -2172,11 +2402,11 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *drive.File
 		if item.ResourceKey != "" {
 			f.dirResourceKeys.Store(item.Id, item.ResourceKey)
 		}
-		when, _ := time.Parse(timeFormatIn, item.ModifiedTime)
-		d := fs.NewDir(remote, when).SetID(item.Id)
-		if len(item.Parents) > 0 {
-			d.SetParentID(item.Parents[0])
+		baseObject, err := f.newBaseObject(ctx, remote, item)
+		if err != nil {
+			return nil, err
 		}
+		d := &Directory{baseObject: baseObject}
 		return d, nil
 	case f.opt.AuthOwnerOnly && !isAuthOwned(item):
 		// ignore object
@@ -2204,7 +2434,6 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 	// Define the metadata for the file we are going to create.
 	createInfo := &drive.File{
 		Name:         leaf,
-		Description:  leaf,
 		Parents:      []string{directoryID},
 		ModifiedTime: modTime.Format(timeFormatOut),
 	}
@@ -2272,6 +2501,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	} else {
 		createInfo.MimeType = fs.MimeTypeFromName(remote)
 	}
+	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, options, createInfo, false)
+	if err != nil {
+		return nil, err
+	}
 
 	var info *drive.File
 	if size >= 0 && size < int64(f.opt.UploadCutoff) {
@@ -2296,6 +2529,10 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 			return nil, err
 		}
 	}
+	err = updateMetadata(ctx, info)
+	if err != nil {
+		return nil, err
+	}
 	return f.newObjectWithInfo(ctx, remote, info)
 }
 
@@ -2361,6 +2598,59 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return err
 }
 
+// MkdirMetadata makes the directory passed in as dir.
+//
+// It shouldn't return an error if it already exists.
+//
+// If the metadata is not nil it is set.
+//
+// It returns the directory that was created.
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	var info *drive.File
+	dirID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err == fs.ErrorDirNotFound {
+		// Directory does not exist so create it
+		var leaf, parentID string
+		leaf, parentID, err = f.dirCache.FindPath(ctx, dir, true)
+		if err != nil {
+			return nil, err
+		}
+		info, err = f.createDir(ctx, parentID, leaf, metadata)
+	} else if err == nil {
+		// Directory exists and needs updating
+		info, err = f.updateDir(ctx, dirID, metadata)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert the info into a directory entry
+	entry, err := f.itemToDirEntry(ctx, dir, info)
+	if err != nil {
+		return nil, err
+	}
+	dirEntry, ok := entry.(fs.Directory)
+	if !ok {
+		return nil, fmt.Errorf("internal error: expecting %T to be an fs.Directory", entry)
+	}
+
+	return dirEntry, nil
+}
+
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	dirID, err := f.dirCache.FindDir(ctx, dir, false)
+	if err != nil {
+		return err
+	}
+	o := baseObject{
+		fs:     f,
+		remote: dir,
+		id:     dirID,
+	}
+	return o.SetModTime(ctx, modTime)
+}
+
 // delete a file or directory unconditionally by ID
 func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 	return f.pacer.Call(func() (bool, error) {
@@ -2504,6 +2794,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		createInfo.Description = ""
 	}
 
+	// Adjust metadata if required
+	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, fs.MetadataAsOpenOptions(ctx), createInfo, false)
+	if err != nil {
+		return nil, err
+	}
+
 	// get the ID of the thing to copy
 	// copy the contents if CopyShortcutContent
 	// else copy the shortcut only
@@ -2517,7 +2813,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		copy := f.svc.Files.Copy(id, createInfo).
-			Fields(partialFields).
+			Fields(f.getFileFields(ctx)).
 			SupportsAllDrives(true).
 			KeepRevisionForever(f.opt.KeepRevisionForever)
 		srcObj.addResourceKey(copy.Header())
@@ -2537,7 +2833,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	// FIXME remove this when google fixes the problem!
 	if isDoc {
 		// A short sleep is needed here in order to make the
-		// change effective, without it is is ignored. This is
+		// change effective, without it is ignored. This is
 		// probably some eventual consistency nastiness.
 		sleepTime := 2 * time.Second
 		fs.Debugf(f, "Sleeping for %v before setting the modtime to work around drive bug - see #4517", sleepTime)
@@ -2553,6 +2849,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			fs.Errorf(existingObject, "Failed to remove existing object after copy: %v", err)
 		}
 	}
+	// Finalise metadata
+	err = updateMetadata(ctx, info)
+	if err != nil {
+		return nil, err
+	}
 	return newObject, nil
 }
 
@@ -2726,13 +3027,19 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	dstParents := strings.Join(dstInfo.Parents, ",")
 	dstInfo.Parents = nil
 
+	// Adjust metadata if required
+	updateMetadata, err := f.fetchAndUpdateMetadata(ctx, src, fs.MetadataAsOpenOptions(ctx), dstInfo, true)
+	if err != nil {
+		return nil, err
+	}
+
 	// Do the move
 	var info *drive.File
 	err = f.pacer.Call(func() (bool, error) {
 		info, err = f.svc.Files.Update(shortcutID(srcObj.id), dstInfo).
 			RemoveParents(srcParentID).
 			AddParents(dstParents).
-			Fields(partialFields).
+			Fields(f.getFileFields(ctx)).
 			SupportsAllDrives(true).
 			Context(ctx).Do()
 		return f.shouldRetry(ctx, err)
@@ -2741,6 +3048,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
+	// Finalise metadata
+	err = updateMetadata(ctx, info)
+	if err != nil {
+		return nil, err
+	}
 	return f.newObjectWithInfo(ctx, remote, info)
 }
 
@@ -2984,7 +3296,7 @@ func (f *Fs) DirCacheFlush() {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.MD5)
+	return hash.NewHashSet(hash.MD5, hash.SHA1, hash.SHA256)
 }
 
 func (f *Fs) changeChunkSize(chunkSizeString string) (err error) {
@@ -3213,7 +3525,7 @@ func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTras
 
 // copy file with id to dest
 func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
-	info, err := f.getFile(ctx, id, f.fileFields)
+	info, err := f.getFile(ctx, id, f.getFileFields(ctx))
 	if err != nil {
 		return fmt.Errorf("couldn't find id: %w", err)
 	}
@@ -3246,6 +3558,50 @@ func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
 	return nil
 }
 
+func (f *Fs) query(ctx context.Context, query string) (entries []*drive.File, err error) {
+	list := f.svc.Files.List()
+	if query != "" {
+		list.Q(query)
+	}
+
+	if f.opt.ListChunk > 0 {
+		list.PageSize(f.opt.ListChunk)
+	}
+	list.SupportsAllDrives(true)
+	list.IncludeItemsFromAllDrives(true)
+	if f.isTeamDrive && !f.opt.SharedWithMe {
+		list.DriveId(f.opt.TeamDriveID)
+		list.Corpora("drive")
+	}
+	// If using appDataFolder then need to add Spaces
+	if f.rootFolderID == "appDataFolder" {
+		list.Spaces("appDataFolder")
+	}
+
+	fields := fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", f.getFileFields(ctx))
+
+	var results []*drive.File
+	for {
+		var files *drive.FileList
+		err = f.pacer.Call(func() (bool, error) {
+			files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
+			return f.shouldRetry(ctx, err)
+		})
+		if err != nil {
+			return nil, fmt.Errorf("failed to execute query: %w", err)
+		}
+		if files.IncompleteSearch {
+			fs.Errorf(f, "search result INCOMPLETE")
+		}
+		results = append(results, files.Files...)
+		if files.NextPageToken == "" {
+			break
+		}
+		list.PageToken(files.NextPageToken)
+	}
+	return results, nil
+}
+
 var commandHelp = []fs.CommandHelp{{
 	Name:  "get",
 	Short: "Get command for fetching the drive config parameters",
@@ -3396,6 +3752,47 @@ Use the --interactive/-i or --dry-run flag to see what would be copied before co
 }, {
 	Name:  "importformats",
 	Short: "Dump the import formats for debug purposes",
+}, {
+	Name:  "query",
+	Short: "List files using Google Drive query language",
+	Long: `This command lists files based on a query
+
+Usage:
+
+    rclone backend query drive: query
+    
+The query syntax is documented at [Google Drive Search query terms and 
+operators](https://developers.google.com/drive/api/guides/ref-search-terms).
+
+For example:
+
+	rclone backend query drive: "'0ABc9DEFGHIJKLMNop0QRatUVW3X' in parents and name contains 'foo'"
+
+If the query contains literal ' or \ characters, these need to be escaped with
+\ characters. "'" becomes "\'" and "\" becomes "\\\", for example to match a 
+file named "foo ' \.txt":
+
+	rclone backend query drive: "name = 'foo \' \\\.txt'"
+
+The result is a JSON array of matches, for example:
+
+    [
+	{
+		"createdTime": "2017-06-29T19:58:28.537Z",
+		"id": "0AxBe_CDEF4zkGHI4d0FjYko2QkD",
+		"md5Checksum": "68518d16be0c6fbfab918be61d658032",
+		"mimeType": "text/plain",
+		"modifiedTime": "2024-02-02T10:40:02.874Z",
+		"name": "foo ' \\.txt",
+		"parents": [
+			"0BxAe_BCDE4zkFGZpcWJGek0xbzC"
+		],
+		"resourceKey": "0-ABCDEFGHIXJQpIGqBJq3MC",
+		"sha1Checksum": "8f284fa768bfb4e45d076a579ab3905ab6bfa893",
+		"size": "311",
+		"webViewLink": "https://drive.google.com/file/d/0AxBe_CDEF4zkGHI4d0FjYko2QkD/view?usp=drivesdk\u0026resourcekey=0-ABCDEFGHIXJQpIGqBJq3MC"
+	}
+    ]`,
 }}
 
 // Command the backend to run a named command
@@ -3513,6 +3910,17 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 		return f.exportFormats(ctx), nil
 	case "importformats":
 		return f.importFormats(ctx), nil
+	case "query":
+		if len(arg) == 1 {
+			query := arg[0]
+			var results, err = f.query(ctx, query)
+			if err != nil {
+				return nil, fmt.Errorf("failed to execute query: %q, error: %w", query, err)
+			}
+			return results, nil
+		} else {
+			return nil, errors.New("need a query argument")
+		}
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}
@@ -3545,13 +3953,19 @@ func (o *baseObject) Remote() string {
 
 // Hash returns the Md5sum of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.MD5 {
-		return "", hash.ErrUnsupported
+	if t == hash.MD5 {
+		return o.md5sum, nil
 	}
-	return o.md5sum, nil
+	if t == hash.SHA1 {
+		return o.sha1sum, nil
+	}
+	if t == hash.SHA256 {
+		return o.sha256sum, nil
+	}
+	return "", hash.ErrUnsupported
 }
 func (o *baseObject) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.MD5 {
+	if t != hash.MD5 && t != hash.SHA1 && t != hash.SHA256 {
 		return "", hash.ErrUnsupported
 	}
 	return "", nil
@@ -3887,10 +4301,20 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		MimeType:     srcMimeType,
 		ModifiedTime: src.ModTime(ctx).Format(timeFormatOut),
 	}
+
+	updateMetadata, err := o.fs.fetchAndUpdateMetadata(ctx, src, options, updateInfo, true)
+	if err != nil {
+		return err
+	}
+
 	info, err := o.baseObject.update(ctx, updateInfo, srcMimeType, in, src)
 	if err != nil {
 		return err
 	}
+	err = updateMetadata(ctx, info)
+	if err != nil {
+		return err
+	}
 	newO, err := o.fs.newObjectWithInfo(ctx, o.remote, info)
 	if err != nil {
 		return err
@@ -3976,6 +4400,26 @@ func (o *baseObject) ParentID() string {
 	return ""
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *baseObject) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	if o.metadata != nil {
+		return *o.metadata, nil
+	}
+	fs.Debugf(o, "Fetching metadata")
+	id := actualID(o.id)
+	info, err := o.fs.getFile(ctx, id, o.fs.getFileFields(ctx))
+	if err != nil {
+		return nil, err
+	}
+	err = o.parseMetadata(ctx, info)
+	if err != nil {
+		return nil, err
+	}
+	return *o.metadata, nil
+}
+
 func (o *documentObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
@@ -3983,6 +4427,37 @@ func (o *linkObject) ext() string {
 	return o.baseObject.remote[len(o.baseObject.remote)-o.extLen:]
 }
 
+// Items returns the count of items in this directory or this
+// directory and subdirectories if known, -1 for unknown
+func (d *Directory) Items() int64 {
+	return -1
+}
+
+// SetMetadata sets metadata for a Directory
+//
+// It should return fs.ErrorNotImplemented if it can't set metadata
+func (d *Directory) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
+	info, err := d.fs.updateDir(ctx, d.id, metadata)
+	if err != nil {
+		return fmt.Errorf("failed to update directory info: %w", err)
+	}
+	// Update directory from info returned
+	baseObject, err := d.fs.newBaseObject(ctx, d.remote, info)
+	if err != nil {
+		return fmt.Errorf("failed to process directory info: %w", err)
+	}
+	d.baseObject = baseObject
+	return err
+}
+
+// Hash does nothing on a directory
+//
+// This method is implemented with the incorrect type signature to
+// stop the Directory type asserting to fs.Object or fs.ObjectInfo
+func (d *Directory) Hash() {
+	// Does nothing
+}
+
 // templates for document link files
 const (
 	urlTemplate = `[InternetShortcut]{{"\r"}}
@@ -4032,11 +4507,14 @@ var (
 	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirSetModTimer  = (*Fs)(nil)
+	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 	_ fs.ParentIDer      = (*Object)(nil)
+	_ fs.Metadataer      = (*Object)(nil)
 	_ fs.Object          = (*documentObject)(nil)
 	_ fs.MimeTyper       = (*documentObject)(nil)
 	_ fs.IDer            = (*documentObject)(nil)
@@ -4045,4 +4523,8 @@ var (
 	_ fs.MimeTyper       = (*linkObject)(nil)
 	_ fs.IDer            = (*linkObject)(nil)
 	_ fs.ParentIDer      = (*linkObject)(nil)
+	_ fs.Directory       = (*Directory)(nil)
+	_ fs.SetModTimer     = (*Directory)(nil)
+	_ fs.SetMetadataer   = (*Directory)(nil)
+	_ fs.ParentIDer      = (*Directory)(nil)
 )

backend/drive/drive_internal_test.go

@@ -524,12 +524,49 @@ func (f *Fs) InternalTestCopyID(t *testing.T) {
 	})
 }
 
+// TestIntegration/FsMkdir/FsPutFiles/Internal/Query
+func (f *Fs) InternalTestQuery(t *testing.T) {
+	ctx := context.Background()
+	var err error
+	t.Run("BadQuery", func(t *testing.T) {
+		_, err = f.query(ctx, "this is a bad query")
+		require.Error(t, err)
+		assert.Contains(t, err.Error(), "failed to execute query")
+	})
+
+	t.Run("NoMatch", func(t *testing.T) {
+		results, err := f.query(ctx, fmt.Sprintf("name='%s' and name!='%s'", existingSubDir, existingSubDir))
+		require.NoError(t, err)
+		assert.Len(t, results, 0)
+	})
+
+	t.Run("GoodQuery", func(t *testing.T) {
+		pathSegments := strings.Split(existingFile, "/")
+		var parent string
+		for _, item := range pathSegments {
+			// the file name contains ' characters which must be escaped
+			escapedItem := f.opt.Enc.FromStandardName(item)
+			escapedItem = strings.ReplaceAll(escapedItem, `\`, `\\`)
+			escapedItem = strings.ReplaceAll(escapedItem, `'`, `\'`)
+
+			results, err := f.query(ctx, fmt.Sprintf("%strashed=false and name='%s'", parent, escapedItem))
+			require.NoError(t, err)
+			require.True(t, len(results) > 0)
+			for _, result := range results {
+				assert.True(t, len(result.Id) > 0)
+				assert.Equal(t, result.Name, item)
+			}
+			parent = fmt.Sprintf("'%s' in parents and ", results[0].Id)
+		}
+	})
+}
+
 // TestIntegration/FsMkdir/FsPutFiles/Internal/AgeQuery
 func (f *Fs) InternalTestAgeQuery(t *testing.T) {
 	// Check set up for filtering
 	assert.True(t, f.Features().FilterAware)
 
-	opt := &filter.Opt{}
+	opt := &filter.Options{}
 	err := opt.MaxAge.Set("1h")
 	assert.NoError(t, err)
 	flt, err := filter.NewFilter(opt)
@@ -611,6 +648,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 	t.Run("Shortcuts", f.InternalTestShortcuts)
 	t.Run("UnTrash", f.InternalTestUnTrash)
 	t.Run("CopyID", f.InternalTestCopyID)
+	t.Run("Query", f.InternalTestQuery)
 	t.Run("AgeQuery", f.InternalTestAgeQuery)
 	t.Run("ShouldRetry", f.InternalTestShouldRetry)
 }

backend/drive/metadata.go

@@ -0,0 +1,638 @@
+package drive
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/errcount"
+	"golang.org/x/sync/errgroup"
+	drive "google.golang.org/api/drive/v3"
+	"google.golang.org/api/googleapi"
+)
+
+// system metadata keys which this backend owns
+var systemMetadataInfo = map[string]fs.MetadataHelp{
+	"content-type": {
+		Help:    "The MIME type of the file.",
+		Type:    "string",
+		Example: "text/plain",
+	},
+	"mtime": {
+		Help:    "Time of last modification with mS accuracy.",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999Z07:00",
+	},
+	"btime": {
+		Help:    "Time of file birth (creation) with mS accuracy. Note that this is only writable on fresh uploads - it can't be written for updates.",
+		Type:    "RFC 3339",
+		Example: "2006-01-02T15:04:05.999Z07:00",
+	},
+	"copy-requires-writer-permission": {
+		Help:    "Whether the options to copy, print, or download this file, should be disabled for readers and commenters.",
+		Type:    "boolean",
+		Example: "true",
+	},
+	"writers-can-share": {
+		Help:    "Whether users with only writer permission can modify the file's permissions. Not populated and ignored when setting for items in shared drives.",
+		Type:    "boolean",
+		Example: "false",
+	},
+	"viewed-by-me": {
+		Help:     "Whether the file has been viewed by this user.",
+		Type:     "boolean",
+		Example:  "true",
+		ReadOnly: true,
+	},
+	"owner": {
+		Help:    "The owner of the file. Usually an email address. Enable with --drive-metadata-owner.",
+		Type:    "string",
+		Example: "user@example.com",
+	},
+	"permissions": {
+		Help:    "Permissions in a JSON dump of Google drive format. On shared drives these will only be present if they aren't inherited. Enable with --drive-metadata-permissions.",
+		Type:    "JSON",
+		Example: "{}",
+	},
+	"folder-color-rgb": {
+		Help:    "The color for a folder or a shortcut to a folder as an RGB hex string.",
+		Type:    "string",
+		Example: "881133",
+	},
+	"description": {
+		Help:    "A short description of the file.",
+		Type:    "string",
+		Example: "Contract for signing",
+	},
+	"starred": {
+		Help:    "Whether the user has starred the file.",
+		Type:    "boolean",
+		Example: "false",
+	},
+	"labels": {
+		Help:    "Labels attached to this file in a JSON dump of Googled drive format. Enable with --drive-metadata-labels.",
+		Type:    "JSON",
+		Example: "[]",
+	},
+}
+
+// Extra fields we need to fetch to implement the system metadata above
+var metadataFields = googleapi.Field(strings.Join([]string{
+	"copyRequiresWriterPermission",
+	"description",
+	"folderColorRgb",
+	"hasAugmentedPermissions",
+	"owners",
+	"permissionIds",
+	"permissions",
+	"properties",
+	"starred",
+	"viewedByMe",
+	"viewedByMeTime",
+	"writersCanShare",
+}, ","))
+
+// Fields we need to read from permissions
+var permissionsFields = googleapi.Field(strings.Join([]string{
+	"*",
+	"permissionDetails/*",
+}, ","))
+
+// getPermission returns permissions for the fileID and permissionID passed in
+func (f *Fs) getPermission(ctx context.Context, fileID, permissionID string, useCache bool) (perm *drive.Permission, inherited bool, err error) {
+	f.permissionsMu.Lock()
+	defer f.permissionsMu.Unlock()
+	if useCache {
+		perm = f.permissions[permissionID]
+		if perm != nil {
+			return perm, false, nil
+		}
+	}
+	fs.Debugf(f, "Fetching permission %q", permissionID)
+	err = f.pacer.Call(func() (bool, error) {
+		perm, err = f.svc.Permissions.Get(fileID, permissionID).
+			Fields(permissionsFields).
+			SupportsAllDrives(true).
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, false, err
+	}
+
+	inherited = len(perm.PermissionDetails) > 0 && perm.PermissionDetails[0].Inherited
+
+	cleanPermission(perm)
+
+	// cache the permission
+	f.permissions[permissionID] = perm
+
+	return perm, inherited, err
+}
+
+// Set the permissions on the info
+func (f *Fs) setPermissions(ctx context.Context, info *drive.File, permissions []*drive.Permission) (err error) {
+	errs := errcount.New()
+	for _, perm := range permissions {
+		if perm.Role == "owner" {
+			// ignore owner permissions - these are set with owner
+			continue
+		}
+		cleanPermissionForWrite(perm)
+		err := f.pacer.Call(func() (bool, error) {
+			_, err := f.svc.Permissions.Create(info.Id, perm).
+				SupportsAllDrives(true).
+				SendNotificationEmail(false).
+				Context(ctx).Do()
+			return f.shouldRetry(ctx, err)
+		})
+		if err != nil {
+			fs.Errorf(f, "Failed to set permission %s for %q: %v", perm.Role, perm.EmailAddress, err)
+			errs.Add(err)
+		}
+	}
+	err = errs.Err("failed to set permission")
+	if err != nil {
+		err = fserrors.NoRetryError(err)
+	}
+	return err
+}
+
+// Clean attributes from permissions which we can't write
+func cleanPermissionForWrite(perm *drive.Permission) {
+	perm.Deleted = false
+	perm.DisplayName = ""
+	perm.Id = ""
+	perm.Kind = ""
+	perm.PermissionDetails = nil
+	perm.TeamDrivePermissionDetails = nil
+}
+
+// Clean and cache the permission if not already cached
+func (f *Fs) cleanAndCachePermission(perm *drive.Permission) {
+	f.permissionsMu.Lock()
+	defer f.permissionsMu.Unlock()
+	cleanPermission(perm)
+	if _, found := f.permissions[perm.Id]; !found {
+		f.permissions[perm.Id] = perm
+	}
+}
+
+// Clean fields we don't need to keep from the permission
+func cleanPermission(perm *drive.Permission) {
+	// DisplayName: Output only. The "pretty" name of the value of the
+	// permission. The following is a list of examples for each type of
+	// permission: * `user` - User's full name, as defined for their Google
+	// account, such as "Joe Smith." * `group` - Name of the Google Group,
+	// such as "The Company Administrators." * `domain` - String domain
+	// name, such as "thecompany.com." * `anyone` - No `displayName` is
+	// present.
+	perm.DisplayName = ""
+
+	// Kind: Output only. Identifies what kind of resource this is. Value:
+	// the fixed string "drive#permission".
+	perm.Kind = ""
+
+	// PermissionDetails: Output only. Details of whether the permissions on
+	// this shared drive item are inherited or directly on this item. This
+	// is an output-only field which is present only for shared drive items.
+	perm.PermissionDetails = nil
+
+	// PhotoLink: Output only. A link to the user's profile photo, if
+	// available.
+	perm.PhotoLink = ""
+
+	// TeamDrivePermissionDetails: Output only. Deprecated: Output only. Use
+	// `permissionDetails` instead.
+	perm.TeamDrivePermissionDetails = nil
+}
+
+// Fields we need to read from labels
+var labelsFields = googleapi.Field(strings.Join([]string{
+	"*",
+}, ","))
+
+// getLabels returns labels for the fileID passed in
+func (f *Fs) getLabels(ctx context.Context, fileID string) (labels []*drive.Label, err error) {
+	fs.Debugf(f, "Fetching labels for %q", fileID)
+	listLabels := f.svc.Files.ListLabels(fileID).
+		Fields(labelsFields).
+		Context(ctx)
+	for {
+		var info *drive.LabelList
+		err = f.pacer.Call(func() (bool, error) {
+			info, err = listLabels.Do()
+			return f.shouldRetry(ctx, err)
+		})
+		if err != nil {
+			return nil, err
+		}
+		labels = append(labels, info.Labels...)
+		if info.NextPageToken == "" {
+			break
+		}
+		listLabels.PageToken(info.NextPageToken)
+	}
+	for _, label := range labels {
+		cleanLabel(label)
+	}
+	return labels, nil
+}
+
+// Set the labels on the info
+func (f *Fs) setLabels(ctx context.Context, info *drive.File, labels []*drive.Label) (err error) {
+	if len(labels) == 0 {
+		return nil
+	}
+	req := drive.ModifyLabelsRequest{}
+	for _, label := range labels {
+		req.LabelModifications = append(req.LabelModifications, &drive.LabelModification{
+			FieldModifications: labelFieldsToFieldModifications(label.Fields),
+			LabelId:            label.Id,
+		})
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.svc.Files.ModifyLabels(info.Id, &req).
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set labels: %w", err)
+	}
+	return nil
+}
+
+// Convert label fields into something which can set the fields
+func labelFieldsToFieldModifications(fields map[string]drive.LabelField) (out []*drive.LabelFieldModification) {
+	for id, field := range fields {
+		var emails []string
+		for _, user := range field.User {
+			emails = append(emails, user.EmailAddress)
+		}
+		out = append(out, &drive.LabelFieldModification{
+			// FieldId: The ID of the field to be modified.
+			FieldId: id,
+
+			// SetDateValues: Replaces the value of a dateString Field with these
+			// new values. The string must be in the RFC 3339 full-date format:
+			// YYYY-MM-DD.
+			SetDateValues: field.DateString,
+
+			// SetIntegerValues: Replaces the value of an `integer` field with these
+			// new values.
+			SetIntegerValues: field.Integer,
+
+			// SetSelectionValues: Replaces a `selection` field with these new
+			// values.
+			SetSelectionValues: field.Selection,
+
+			// SetTextValues: Sets the value of a `text` field.
+			SetTextValues: field.Text,
+
+			// SetUserValues: Replaces a `user` field with these new values. The
+			// values must be valid email addresses.
+			SetUserValues: emails,
+		})
+	}
+	return out
+}
+
+// Clean fields we don't need to keep from the label
+func cleanLabel(label *drive.Label) {
+	// Kind: This is always drive#label
+	label.Kind = ""
+
+	for name, field := range label.Fields {
+		// Kind: This is always drive#labelField.
+		field.Kind = ""
+
+		// Note the fields are copies so we need to write them
+		// back to the map
+		label.Fields[name] = field
+	}
+}
+
+// Parse the metadata from drive item
+//
+// It should return nil if there is no Metadata
+func (o *baseObject) parseMetadata(ctx context.Context, info *drive.File) (err error) {
+	metadata := make(fs.Metadata, 16)
+
+	// Dump user metadata first as it overrides system metadata
+	for k, v := range info.Properties {
+		metadata[k] = v
+	}
+
+	// System metadata
+	metadata["copy-requires-writer-permission"] = fmt.Sprint(info.CopyRequiresWriterPermission)
+	metadata["writers-can-share"] = fmt.Sprint(info.WritersCanShare)
+	metadata["viewed-by-me"] = fmt.Sprint(info.ViewedByMe)
+	metadata["content-type"] = info.MimeType
+
+	// Owners: Output only. The owner of this file. Only certain legacy
+	// files may have more than one owner. This field isn't populated for
+	// items in shared drives.
+	if o.fs.opt.MetadataOwner.IsSet(rwRead) && len(info.Owners) > 0 {
+		user := info.Owners[0]
+		if len(info.Owners) > 1 {
+			fs.Logf(o, "Ignoring more than 1 owner")
+		}
+		if user != nil {
+			id := user.EmailAddress
+			if id == "" {
+				id = user.DisplayName
+			}
+			metadata["owner"] = id
+		}
+	}
+
+	if o.fs.opt.MetadataPermissions.IsSet(rwRead) {
+		// We only write permissions out if they are not inherited.
+		//
+		// On My Drives permissions seem to be attached to every item
+		// so they will always be written out.
+		//
+		// On Shared Drives only non-inherited permissions will be
+		// written out.
+
+		// To read the inherited permissions flag will mean we need to
+		// read the permissions for each object and the cache will be
+		// useless. However shared drives don't return permissions
+		// only permissionIds so will need to fetch them for each
+		// object. We use HasAugmentedPermissions to see if there are
+		// special permissions before fetching them to save transactions.
+
+		// HasAugmentedPermissions: Output only. Whether there are permissions
+		// directly on this file. This field is only populated for items in
+		// shared drives.
+		if o.fs.isTeamDrive && !info.HasAugmentedPermissions {
+			// Don't process permissions if there aren't any specifically set
+			fs.Debugf(o, "Ignoring %d permissions and %d permissionIds as is shared drive with hasAugmentedPermissions false", len(info.Permissions), len(info.PermissionIds))
+			info.Permissions = nil
+			info.PermissionIds = nil
+		}
+
+		// PermissionIds: Output only. List of permission IDs for users with
+		// access to this file.
+		//
+		// Only process these if we have no Permissions
+		if len(info.PermissionIds) > 0 && len(info.Permissions) == 0 {
+			info.Permissions = make([]*drive.Permission, 0, len(info.PermissionIds))
+			g, gCtx := errgroup.WithContext(ctx)
+			g.SetLimit(o.fs.ci.Checkers)
+			var mu sync.Mutex // protect the info.Permissions from concurrent writes
+			for _, permissionID := range info.PermissionIds {
+				permissionID := permissionID
+				g.Go(func() error {
+					// must fetch the team drive ones individually to check the inherited flag
+					perm, inherited, err := o.fs.getPermission(gCtx, actualID(info.Id), permissionID, !o.fs.isTeamDrive)
+					if err != nil {
+						return fmt.Errorf("failed to read permission: %w", err)
+					}
+					// Don't write inherited permissions out
+					if inherited {
+						return nil
+					}
+					// Don't write owner role out - these are covered by the owner metadata
+					if perm.Role == "owner" {
+						return nil
+					}
+					mu.Lock()
+					info.Permissions = append(info.Permissions, perm)
+					mu.Unlock()
+					return nil
+				})
+			}
+			err = g.Wait()
+			if err != nil {
+				return err
+			}
+		} else {
+			// Clean the fetched permissions
+			for _, perm := range info.Permissions {
+				o.fs.cleanAndCachePermission(perm)
+			}
+		}
+
+		// Permissions: Output only. The full list of permissions for the file.
+		// This is only available if the requesting user can share the file. Not
+		// populated for items in shared drives.
+		if len(info.Permissions) > 0 {
+			buf, err := json.Marshal(info.Permissions)
+			if err != nil {
+				return fmt.Errorf("failed to marshal permissions: %w", err)
+			}
+			metadata["permissions"] = string(buf)
+		}
+
+		// Permission propagation
+		// https://developers.google.com/drive/api/guides/manage-sharing#permission-propagation
+		// Leads me to believe that in non shared drives, permissions
+		// are added to each item when you set permissions for a
+		// folder whereas in shared drives they are inherited and
+		// placed on the item directly.
+	}
+
+	if info.FolderColorRgb != "" {
+		metadata["folder-color-rgb"] = info.FolderColorRgb
+	}
+	if info.Description != "" {
+		metadata["description"] = info.Description
+	}
+	metadata["starred"] = fmt.Sprint(info.Starred)
+	metadata["btime"] = info.CreatedTime
+	metadata["mtime"] = info.ModifiedTime
+
+	if o.fs.opt.MetadataLabels.IsSet(rwRead) {
+		// FIXME would be really nice if we knew if files had labels
+		// before listing but we need to know all possible label IDs
+		// to get it in the listing.
+
+		labels, err := o.fs.getLabels(ctx, actualID(info.Id))
+		if err != nil {
+			return fmt.Errorf("failed to fetch labels: %w", err)
+		}
+		buf, err := json.Marshal(labels)
+		if err != nil {
+			return fmt.Errorf("failed to marshal labels: %w", err)
+		}
+		metadata["labels"] = string(buf)
+	}
+
+	o.metadata = &metadata
+	return nil
+}
+
+// Set the owner on the info
+func (f *Fs) setOwner(ctx context.Context, info *drive.File, owner string) (err error) {
+	perm := drive.Permission{
+		Role:         "owner",
+		EmailAddress: owner,
+		// Type: The type of the grantee. Valid values are: * `user` * `group` *
+		// `domain` * `anyone` When creating a permission, if `type` is `user`
+		// or `group`, you must provide an `emailAddress` for the user or group.
+		// When `type` is `domain`, you must provide a `domain`. There isn't
+		// extra information required for an `anyone` type.
+		Type: "user",
+	}
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.svc.Permissions.Create(info.Id, &perm).
+			SupportsAllDrives(true).
+			TransferOwnership(true).
+			// SendNotificationEmail(false). - required apparently!
+			Context(ctx).Do()
+		return f.shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return fmt.Errorf("failed to set owner: %w", err)
+	}
+	return nil
+}
+
+// Call back to set metadata that can't be set on the upload/update
+//
+// The *drive.File passed in holds the current state of the drive.File
+// and this should update it with any modifications.
+type updateMetadataFn func(context.Context, *drive.File) error
+
+// read the metadata from meta and write it into updateInfo
+//
+// update should be true if this is being used to create metadata for
+// an update/PATCH call as the rules on what can be updated are
+// slightly different there.
+//
+// It returns a callback which should be called to finish the updates
+// after the data is uploaded.
+func (f *Fs) updateMetadata(ctx context.Context, updateInfo *drive.File, meta fs.Metadata, update bool) (callback updateMetadataFn, err error) {
+	callbackFns := []updateMetadataFn{}
+	callback = func(ctx context.Context, info *drive.File) error {
+		for _, fn := range callbackFns {
+			err := fn(ctx, info)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	// merge metadata into request and user metadata
+	for k, v := range meta {
+		k, v := k, v
+		// parse a boolean from v and write into out
+		parseBool := func(out *bool) error {
+			b, err := strconv.ParseBool(v)
+			if err != nil {
+				return fmt.Errorf("can't parse metadata %q = %q: %w", k, v, err)
+			}
+			*out = b
+			return nil
+		}
+		switch k {
+		case "copy-requires-writer-permission":
+			if err := parseBool(&updateInfo.CopyRequiresWriterPermission); err != nil {
+				return nil, err
+			}
+		case "writers-can-share":
+			if !f.isTeamDrive {
+				if err := parseBool(&updateInfo.WritersCanShare); err != nil {
+					return nil, err
+				}
+			} else {
+				fs.Debugf(f, "Ignoring %s=%s as can't set on shared drives", k, v)
+			}
+		case "viewed-by-me":
+			// Can't write this
+		case "content-type":
+			updateInfo.MimeType = v
+		case "owner":
+			if !f.opt.MetadataOwner.IsSet(rwWrite) {
+				continue
+			}
+			// Can't set Owner on upload so need to set afterwards
+			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
+				err := f.setOwner(ctx, info, v)
+				if err != nil && f.opt.MetadataOwner.IsSet(rwFailOK) {
+					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
+					return nil
+				}
+				return err
+			})
+		case "permissions":
+			if !f.opt.MetadataPermissions.IsSet(rwWrite) {
+				continue
+			}
+			var perms []*drive.Permission
+			err := json.Unmarshal([]byte(v), &perms)
+			if err != nil {
+				return nil, fmt.Errorf("failed to unmarshal permissions: %w", err)
+			}
+			// Can't set Permissions on upload so need to set afterwards
+			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
+				err := f.setPermissions(ctx, info, perms)
+				if err != nil && f.opt.MetadataPermissions.IsSet(rwFailOK) {
+					// We've already logged the permissions errors individually here
+					fs.Debugf(f, "Ignoring error as failok is set: %v", err)
+					return nil
+				}
+				return err
+			})
+		case "labels":
+			if !f.opt.MetadataLabels.IsSet(rwWrite) {
+				continue
+			}
+			var labels []*drive.Label
+			err := json.Unmarshal([]byte(v), &labels)
+			if err != nil {
+				return nil, fmt.Errorf("failed to unmarshal labels: %w", err)
+			}
+			// Can't set Labels on upload so need to set afterwards
+			callbackFns = append(callbackFns, func(ctx context.Context, info *drive.File) error {
+				err := f.setLabels(ctx, info, labels)
+				if err != nil && f.opt.MetadataLabels.IsSet(rwFailOK) {
+					fs.Errorf(f, "Ignoring error as failok is set: %v", err)
+					return nil
+				}
+				return err
+			})
+		case "folder-color-rgb":
+			updateInfo.FolderColorRgb = v
+		case "description":
+			updateInfo.Description = v
+		case "starred":
+			if err := parseBool(&updateInfo.Starred); err != nil {
+				return nil, err
+			}
+		case "btime":
+			if update {
+				fs.Debugf(f, "Skipping btime metadata as can't update it on an existing file: %v", v)
+			} else {
+				updateInfo.CreatedTime = v
+			}
+		case "mtime":
+			updateInfo.ModifiedTime = v
+		default:
+			if updateInfo.Properties == nil {
+				updateInfo.Properties = make(map[string]string, 1)
+			}
+			updateInfo.Properties[k] = v
+		}
+	}
+	return callback, nil
+}
+
+// Fetch metadata and update updateInfo if --metadata is in use
+func (f *Fs) fetchAndUpdateMetadata(ctx context.Context, src fs.ObjectInfo, options []fs.OpenOption, updateInfo *drive.File, update bool) (callback updateMetadataFn, err error) {
+	meta, err := fs.GetMetadataOptions(ctx, f, src, options)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read metadata from source object: %w", err)
+	}
+	callback, err = f.updateMetadata(ctx, updateInfo, meta, update)
+	if err != nil {
+		return nil, fmt.Errorf("failed to update metadata from source object: %w", err)
+	}
+	return callback, nil
+}

backend/dropbox/batcher.go

@@ -8,121 +8,19 @@ package dropbox
 
 import (
 	"context"
-	"errors"
 	"fmt"
-	"sync"
-	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
-	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
-	"github.com/rclone/rclone/lib/atexit"
 )
 
-const (
-	maxBatchSize          = 1000                   // max size the batch can be
-	defaultTimeoutSync    = 500 * time.Millisecond // kick off the batch if nothing added for this long (sync)
-	defaultTimeoutAsync   = 10 * time.Second       // kick off the batch if nothing added for this long (ssync)
-	defaultBatchSizeAsync = 100                    // default batch size if async
-)
-
-// batcher holds info about the current items waiting for upload
-type batcher struct {
-	f        *Fs                 // Fs this batch is part of
-	mode     string              // configured batch mode
-	size     int                 // maximum size for batch
-	timeout  time.Duration       // idle timeout for batch
-	async    bool                // whether we are using async batching
-	in       chan batcherRequest // incoming items to batch
-	closed   chan struct{}       // close to indicate batcher shut down
-	atexit   atexit.FnHandle     // atexit handle
-	shutOnce sync.Once           // make sure we shutdown once only
-	wg       sync.WaitGroup      // wait for shutdown
-}
-
-// batcherRequest holds an incoming request with a place for a reply
-type batcherRequest struct {
-	commitInfo *files.UploadSessionFinishArg
-	result     chan<- batcherResponse
-}
-
-// Return true if batcherRequest is the quit request
-func (br *batcherRequest) isQuit() bool {
-	return br.commitInfo == nil
-}
-
-// Send this to get the engine to quit
-var quitRequest = batcherRequest{}
-
-// batcherResponse holds a response to be delivered to clients waiting
-// for a batch to complete.
-type batcherResponse struct {
-	err   error
-	entry *files.FileMetadata
-}
-
-// newBatcher creates a new batcher structure
-func newBatcher(ctx context.Context, f *Fs, mode string, size int, timeout time.Duration) (*batcher, error) {
-	// fs.Debugf(f, "Creating batcher with mode %q, size %d, timeout %v", mode, size, timeout)
-	if size > maxBatchSize || size < 0 {
-		return nil, fmt.Errorf("dropbox: batch size must be < %d and >= 0 - it is currently %d", maxBatchSize, size)
-	}
-
-	async := false
-
-	switch mode {
-	case "sync":
-		if size <= 0 {
-			ci := fs.GetConfig(ctx)
-			size = ci.Transfers
-		}
-		if timeout <= 0 {
-			timeout = defaultTimeoutSync
-		}
-	case "async":
-		if size <= 0 {
-			size = defaultBatchSizeAsync
-		}
-		if timeout <= 0 {
-			timeout = defaultTimeoutAsync
-		}
-		async = true
-	case "off":
-		size = 0
-	default:
-		return nil, fmt.Errorf("dropbox: batch mode must be sync|async|off not %q", mode)
-	}
-
-	b := &batcher{
-		f:       f,
-		mode:    mode,
-		size:    size,
-		timeout: timeout,
-		async:   async,
-		in:      make(chan batcherRequest, size),
-		closed:  make(chan struct{}),
-	}
-	if b.Batching() {
-		b.atexit = atexit.Register(b.Shutdown)
-		b.wg.Add(1)
-		go b.commitLoop(context.Background())
-	}
-	return b, nil
-
-}
-
-// Batching returns true if batching is active
-func (b *batcher) Batching() bool {
-	return b.size > 0
-}
-
 // finishBatch commits the batch, returning a batch status to poll or maybe complete
-func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
+func (f *Fs) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
 	var arg = &files.UploadSessionFinishBatchArg{
 		Entries: items,
 	}
-	err = b.f.pacer.Call(func() (bool, error) {
-		complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
+	err = f.pacer.Call(func() (bool, error) {
+		complete, err = f.srv.UploadSessionFinishBatchV2(arg)
 		// If error is insufficient space then don't retry
 		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
 			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
@@ -139,23 +37,10 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
 	return complete, nil
 }
 
-// commit a batch
-func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []chan<- batcherResponse) (err error) {
-	// If commit fails then signal clients if sync
-	var signalled = b.async
-	defer func() {
-		if err != nil && !signalled {
-			// Signal to clients that there was an error
-			for _, result := range results {
-				result <- batcherResponse{err: err}
-			}
-		}
-	}()
-	desc := fmt.Sprintf("%s batch length %d starting with: %s", b.mode, len(items), items[0].Commit.Path)
-	fs.Debugf(b.f, "Committing %s", desc)
-
+// Called by the batcher to commit a batch
+func (f *Fs) commitBatch(ctx context.Context, items []*files.UploadSessionFinishArg, results []*files.FileMetadata, errors []error) (err error) {
 	// finalise the batch getting either a result or a job id to poll
-	complete, err := b.finishBatch(ctx, items)
+	complete, err := f.finishBatch(ctx, items)
 	if err != nil {
 		return err
 	}
@@ -166,19 +51,13 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
 		return fmt.Errorf("expecting %d items in batch but got %d", len(results), len(entries))
 	}
 
-	// Report results to clients
-	var (
-		errorTag   = ""
-		errorCount = 0
-	)
+	// Format results for return
 	for i := range results {
 		item := entries[i]
-		resp := batcherResponse{}
 		if item.Tag == "success" {
-			resp.entry = item.Success
+			results[i] = item.Success
 		} else {
-			errorCount++
-			errorTag = item.Tag
+			errorTag := item.Tag
 			if item.Failure != nil {
 				errorTag = item.Failure.Tag
 				if item.Failure.LookupFailed != nil {
@@ -191,112 +70,9 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
 					errorTag += "/" + item.Failure.PropertiesError.Tag
 				}
 			}
-			resp.err = fmt.Errorf("batch upload failed: %s", errorTag)
-		}
-		if !b.async {
-			results[i] <- resp
+			errors[i] = fmt.Errorf("upload failed: %s", errorTag)
 		}
 	}
-	// Show signalled so no need to report error to clients from now on
-	signalled = true
 
-	// Report an error if any failed in the batch
-	if errorTag != "" {
-		return fmt.Errorf("batch had %d errors: last error: %s", errorCount, errorTag)
-	}
-
-	fs.Debugf(b.f, "Committed %s", desc)
 	return nil
 }
-
-// commitLoop runs the commit engine in the background
-func (b *batcher) commitLoop(ctx context.Context) {
-	var (
-		items     []*files.UploadSessionFinishArg // current batch of uncommitted files
-		results   []chan<- batcherResponse        // current batch of clients awaiting results
-		idleTimer = time.NewTimer(b.timeout)
-		commit    = func() {
-			err := b.commitBatch(ctx, items, results)
-			if err != nil {
-				fs.Errorf(b.f, "%s batch commit: failed to commit batch length %d: %v", b.mode, len(items), err)
-			}
-			items, results = nil, nil
-		}
-	)
-	defer b.wg.Done()
-	defer idleTimer.Stop()
-	idleTimer.Stop()
-
-outer:
-	for {
-		select {
-		case req := <-b.in:
-			if req.isQuit() {
-				break outer
-			}
-			items = append(items, req.commitInfo)
-			results = append(results, req.result)
-			idleTimer.Stop()
-			if len(items) >= b.size {
-				commit()
-			} else {
-				idleTimer.Reset(b.timeout)
-			}
-		case <-idleTimer.C:
-			if len(items) > 0 {
-				fs.Debugf(b.f, "Batch idle for %v so committing", b.timeout)
-				commit()
-			}
-		}
-
-	}
-	// commit any remaining items
-	if len(items) > 0 {
-		commit()
-	}
-}
-
-// Shutdown finishes any pending batches then shuts everything down
-//
-// Can be called from atexit handler
-func (b *batcher) Shutdown() {
-	if !b.Batching() {
-		return
-	}
-	b.shutOnce.Do(func() {
-		atexit.Unregister(b.atexit)
-		fs.Infof(b.f, "Committing uploads - please wait...")
-		// show that batcher is shutting down
-		close(b.closed)
-		// quit the commitLoop by sending a quitRequest message
-		//
-		// Note that we don't close b.in because that will
-		// cause write to closed channel in Commit when we are
-		// exiting due to a signal.
-		b.in <- quitRequest
-		b.wg.Wait()
-	})
-}
-
-// Commit commits the file using a batch call, first adding it to the
-// batch and then waiting for the batch to complete in a synchronous
-// way if async is not set.
-func (b *batcher) Commit(ctx context.Context, commitInfo *files.UploadSessionFinishArg) (entry *files.FileMetadata, err error) {
-	select {
-	case <-b.closed:
-		return nil, fserrors.FatalError(errors.New("batcher is shutting down"))
-	default:
-	}
-	fs.Debugf(b.f, "Adding %q to batch", commitInfo.Commit.Path)
-	resp := make(chan batcherResponse, 1)
-	b.in <- batcherRequest{
-		commitInfo: commitInfo,
-		result:     resp,
-	}
-	// If running async then don't wait for the result
-	if b.async {
-		return nil, nil
-	}
-	result := <-resp
-	return result.entry, result.err
-}

backend/dropbox/dropbox.go

@@ -47,6 +47,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
@@ -121,6 +122,14 @@ var (
 
 	// Errors
 	errNotSupportedInSharedMode = fserrors.NoRetryError(errors.New("not supported in shared files mode"))
+
+	// Configure the batcher
+	defaultBatcherOptions = batcher.Options{
+		MaxBatchSize:          1000,
+		DefaultTimeoutSync:    500 * time.Millisecond,
+		DefaultTimeoutAsync:   10 * time.Second,
+		DefaultBatchSizeAsync: 100,
+	}
 )
 
 // Gets an oauth config with the right scopes
@@ -152,7 +161,7 @@ func init() {
 				},
 			})
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: append(append(oauthutil.SharedOptions, []fs.Option{{
 			Name: "chunk_size",
 			Help: fmt.Sprintf(`Upload chunk size (< %v).
 
@@ -207,71 +216,12 @@ are supported.
 
 Note that we don't unmount the shared folder afterwards so the 
 --dropbox-shared-folders can be omitted after the first use of a particular 
-shared folder.`,
+shared folder.
+
+See also --dropbox-root-namespace for an alternative way to work with shared
+folders.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "batch_mode",
-			Help: `Upload file batching sync|async|off.
-
-This sets the batch mode used by rclone.
-
-For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)
-
-This has 3 possible values
-
-- off - no batching
-- sync - batch uploads and check completion (default)
-- async - batch upload and don't check completion
-
-Rclone will close any outstanding batches when it exits which may make
-a delay on quit.
-`,
-			Default:  "sync",
-			Advanced: true,
-		}, {
-			Name: "batch_size",
-			Help: `Max number of files in upload batch.
-
-This sets the batch size of files to upload. It has to be less than 1000.
-
-By default this is 0 which means rclone which calculate the batch size
-depending on the setting of batch_mode.
-
-- batch_mode: async - default batch_size is 100
-- batch_mode: sync - default batch_size is the same as --transfers
-- batch_mode: off - not in use
-
-Rclone will close any outstanding batches when it exits which may make
-a delay on quit.
-
-Setting this is a great idea if you are uploading lots of small files
-as it will make them a lot quicker. You can use --transfers 32 to
-maximise throughput.
-`,
-			Default:  0,
-			Advanced: true,
-		}, {
-			Name: "batch_timeout",
-			Help: `Max time to allow an idle upload batch before uploading.
-
-If an upload batch is idle for more than this long then it will be
-uploaded.
-
-The default for this is 0 which means rclone will choose a sensible
-default based on the batch_mode in use.
-
-- batch_mode: async - default batch_timeout is 10s
-- batch_mode: sync - default batch_timeout is 500ms
-- batch_mode: off - not in use
-`,
-			Default:  fs.Duration(0),
-			Advanced: true,
-		}, {
-			Name:     "batch_commit_timeout",
-			Help:     `Max time to wait for a batch to finish committing`,
-			Default:  fs.Duration(10 * time.Minute),
-			Advanced: true,
 		}, {
 			Name:     "pacer_min_sleep",
 			Default:  defaultMinSleep,
@@ -290,23 +240,28 @@ default based on the batch_mode in use.
 				encoder.EncodeDel |
 				encoder.EncodeRightSpace |
 				encoder.EncodeInvalidUtf8,
-		}}...),
+		}, {
+			Name:     "root_namespace",
+			Help:     "Specify a different Dropbox namespace ID to use as the root for all paths.",
+			Default:  "",
+			Advanced: true,
+		}}...), defaultBatcherOptions.FsOptions("For full info see [the main docs](https://rclone.org/dropbox/#batch-mode)\n\n")...),
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize          fs.SizeSuffix        `config:"chunk_size"`
-	Impersonate        string               `config:"impersonate"`
-	SharedFiles        bool                 `config:"shared_files"`
-	SharedFolders      bool                 `config:"shared_folders"`
-	BatchMode          string               `config:"batch_mode"`
-	BatchSize          int                  `config:"batch_size"`
-	BatchTimeout       fs.Duration          `config:"batch_timeout"`
-	BatchCommitTimeout fs.Duration          `config:"batch_commit_timeout"`
-	AsyncBatch         bool                 `config:"async_batch"`
-	PacerMinSleep      fs.Duration          `config:"pacer_min_sleep"`
-	Enc                encoder.MultiEncoder `config:"encoding"`
+	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
+	Impersonate   string               `config:"impersonate"`
+	SharedFiles   bool                 `config:"shared_files"`
+	SharedFolders bool                 `config:"shared_folders"`
+	BatchMode     string               `config:"batch_mode"`
+	BatchSize     int                  `config:"batch_size"`
+	BatchTimeout  fs.Duration          `config:"batch_timeout"`
+	AsyncBatch    bool                 `config:"async_batch"`
+	PacerMinSleep fs.Duration          `config:"pacer_min_sleep"`
+	Enc           encoder.MultiEncoder `config:"encoding"`
+	RootNsid      string               `config:"root_namespace"`
 }
 
 // Fs represents a remote dropbox server
@@ -325,7 +280,7 @@ type Fs struct {
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
 	pacer          *fs.Pacer      // To pace the API calls
 	ns             string         // The namespace we are using or "" for none
-	batcher        *batcher       // batch builder
+	batcher        *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata]
 }
 
 // Object describes a dropbox object
@@ -431,7 +386,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	oldToken = strings.TrimSpace(oldToken)
 	if ok && oldToken != "" && oldToken[0] != '{' {
 		fs.Infof(name, "Converting token to new format")
-		newToken := fmt.Sprintf(`{"access_token":"%s","token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
+		newToken := fmt.Sprintf(`{"access_token":%q,"token_type":"bearer","expiry":"0001-01-01T00:00:00Z"}`, oldToken)
 		err := config.SetValueAndSave(name, config.ConfigToken, newToken)
 		if err != nil {
 			return nil, fmt.Errorf("NewFS convert token: %w", err)
@@ -451,7 +406,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		ci:    ci,
 		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(opt.PacerMinSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
-	f.batcher, err = newBatcher(ctx, f, f.opt.BatchMode, f.opt.BatchSize, time.Duration(f.opt.BatchTimeout))
+	batcherOptions := defaultBatcherOptions
+	batcherOptions.Mode = f.opt.BatchMode
+	batcherOptions.Size = f.opt.BatchSize
+	batcherOptions.Timeout = time.Duration(f.opt.BatchTimeout)
+	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
 	if err != nil {
 		return nil, err
 	}
@@ -478,15 +437,15 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		members := []*team.UserSelectorArg{&user}
 		args := team.NewMembersGetInfoArgs(members)
 
-		memberIds, err := f.team.MembersGetInfo(args)
+		memberIDs, err := f.team.MembersGetInfo(args)
 		if err != nil {
 			return nil, fmt.Errorf("invalid dropbox team member: %q: %w", opt.Impersonate, err)
 		}
-		if len(memberIds) == 0 || memberIds[0].MemberInfo == nil || memberIds[0].MemberInfo.Profile == nil {
+		if len(memberIDs) == 0 || memberIDs[0].MemberInfo == nil || memberIDs[0].MemberInfo.Profile == nil {
 			return nil, fmt.Errorf("dropbox team member not found: %q", opt.Impersonate)
 		}
 
-		cfg.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
+		cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId
 	}
 
 	f.srv = files.New(cfg)
@@ -552,8 +511,11 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 
 	f.features.Fill(ctx, f)
 
-	// If root starts with / then use the actual root
-	if strings.HasPrefix(root, "/") {
+	if f.opt.RootNsid != "" {
+		f.ns = f.opt.RootNsid
+		fs.Debugf(f, "Overriding root namespace to %q", f.ns)
+	} else if strings.HasPrefix(root, "/") {
+		// If root starts with / then use the actual root
 		var acc *users.FullAccount
 		err = f.pacer.Call(func() (bool, error) {
 			acc, err = f.users.GetCurrentAccount()
@@ -694,7 +656,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
-// listSharedFoldersApi lists all available shared folders mounted and not mounted
+// listSharedFolders lists all available shared folders mounted and not mounted
 // we'll need the id later so we have to return them in original format
 func (f *Fs) listSharedFolders(ctx context.Context) (entries fs.DirEntries, err error) {
 	started := false
@@ -996,6 +958,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	if root == "/" {
 		return errors.New("can't remove root directory")
 	}
+	encRoot := f.opt.Enc.FromStandardPath(root)
 
 	if check {
 		// check directory exists
@@ -1004,10 +967,9 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 			return fmt.Errorf("Rmdir: %w", err)
 		}
 
-		root = f.opt.Enc.FromStandardPath(root)
 		// check directory empty
 		arg := files.ListFolderArg{
-			Path:      root,
+			Path:      encRoot,
 			Recursive: false,
 		}
 		if root == "/" {
@@ -1028,7 +990,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 
 	// remove it
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: root})
+		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: encRoot})
 		return shouldRetry(ctx, err)
 	})
 	return err
@@ -1281,18 +1243,21 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return nil, err
 	}
 	var total uint64
+	used := q.Used
 	if q.Allocation != nil {
 		if q.Allocation.Individual != nil {
 			total += q.Allocation.Individual.Allocated
 		}
 		if q.Allocation.Team != nil {
 			total += q.Allocation.Team.Allocated
+			// Override used with Team.Used as this includes q.Used already
+			used = q.Allocation.Team.Used
 		}
 	}
 	usage = &fs.Usage{
-		Total: fs.NewUsageValue(int64(total)),          // quota of bytes that can be used
-		Used:  fs.NewUsageValue(int64(q.Used)),         // bytes in use
-		Free:  fs.NewUsageValue(int64(total - q.Used)), // bytes which can be uploaded before reaching the quota
+		Total: fs.NewUsageValue(int64(total)),        // quota of bytes that can be used
+		Used:  fs.NewUsageValue(int64(used)),         // bytes in use
+		Free:  fs.NewUsageValue(int64(total - used)), // bytes which can be uploaded before reaching the quota
 	}
 	return usage, nil
 }
@@ -1722,7 +1687,7 @@ func (o *Object) uploadChunked(ctx context.Context, in0 io.Reader, commitInfo *f
 	// If we are batching then we should have written all the data now
 	// store the commit info now for a batch commit
 	if o.fs.batcher.Batching() {
-		return o.fs.batcher.Commit(ctx, args)
+		return o.fs.batcher.Commit(ctx, o.remote, args)
 	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/fichier/api.go

@@ -28,14 +28,14 @@ var retryErrorCodes = []int{
 	509, // Bandwidth Limit Exceeded
 }
 
-var errorRegex = regexp.MustCompile(`#\d{1,3}`)
+var errorRegex = regexp.MustCompile(`#(\d{1,3})`)
 
 func parseFichierError(err error) int {
 	matches := errorRegex.FindStringSubmatch(err.Error())
 	if len(matches) == 0 {
 		return 0
 	}
-	code, err := strconv.Atoi(matches[0])
+	code, err := strconv.Atoi(matches[1])
 	if err != nil {
 		fs.Debugf(nil, "failed parsing fichier error: %v", err)
 		return 0
@@ -61,7 +61,7 @@ func shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, err
 			return false, err // No such user
 		case 186:
 			return false, err // IP blocked?
-		case 374:
+		case 374, 412: // Flood detected seems to be #412 now
 			fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
 			time.Sleep(30 * time.Second)
 		default:

backend/fichier/fichier.go

@@ -441,23 +441,28 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Debugf(src, "Can't move - not same remote type")
 		return nil, fs.ErrorCantMove
 	}
+	srcFs := srcObj.fs
 
 	// Find current directory ID
-	_, currentDirectoryID, err := f.dirCache.FindPath(ctx, remote, false)
+	srcLeaf, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcObj.remote, false)
 	if err != nil {
 		return nil, err
 	}
 
 	// Create temporary object
-	dstObj, leaf, directoryID, err := f.createObject(ctx, remote)
+	dstObj, dstLeaf, dstDirectoryID, err := f.createObject(ctx, remote)
 	if err != nil {
 		return nil, err
 	}
 
 	// If it is in the correct directory, just rename it
 	var url string
-	if currentDirectoryID == directoryID {
-		resp, err := f.renameFile(ctx, srcObj.file.URL, leaf)
+	if srcDirectoryID == dstDirectoryID {
+		// No rename needed
+		if srcLeaf == dstLeaf {
+			return src, nil
+		}
+		resp, err := f.renameFile(ctx, srcObj.file.URL, dstLeaf)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't rename file: %w", err)
 		}
@@ -466,11 +471,16 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		}
 		url = resp.URLs[0].URL
 	} else {
-		folderID, err := strconv.Atoi(directoryID)
+		dstFolderID, err := strconv.Atoi(dstDirectoryID)
 		if err != nil {
 			return nil, err
 		}
-		resp, err := f.moveFile(ctx, srcObj.file.URL, folderID, leaf)
+		rename := dstLeaf
+		// No rename needed
+		if srcLeaf == dstLeaf {
+			rename = ""
+		}
+		resp, err := f.moveFile(ctx, srcObj.file.URL, dstFolderID, rename)
 		if err != nil {
 			return nil, fmt.Errorf("couldn't move file: %w", err)
 		}

backend/filefabric/filefabric.go

@@ -158,9 +158,9 @@ type Fs struct {
 	tokenMu         sync.Mutex         // hold when reading the token
 	token           string             // current access token
 	tokenExpiry     time.Time          // time the current token expires
-	tokenExpired    int32              // read and written with atomic
-	canCopyWithName bool               // set if detected that can use fi_name in copy
-	precision       time.Duration      // precision reported
+	tokenExpired    atomic.Int32
+	canCopyWithName bool          // set if detected that can use fi_name in copy
+	precision       time.Duration // precision reported
 }
 
 // Object describes a filefabric object
@@ -243,7 +243,7 @@ func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error, st
 		err = status // return the error from the RPC
 		code := status.GetCode()
 		if code == "login_token_expired" {
-			atomic.AddInt32(&f.tokenExpired, 1)
+			f.tokenExpired.Add(1)
 		} else {
 			for _, retryCode := range retryStatusCodes {
 				if code == retryCode.code {
@@ -323,12 +323,12 @@ func (f *Fs) getToken(ctx context.Context) (token string, err error) {
 	var refreshed = false
 	defer func() {
 		if refreshed {
-			atomic.StoreInt32(&f.tokenExpired, 0)
+			f.tokenExpired.Store(0)
 		}
 		f.tokenMu.Unlock()
 	}()
 
-	expired := atomic.LoadInt32(&f.tokenExpired) != 0
+	expired := f.tokenExpired.Load() != 0
 	if expired {
 		fs.Debugf(f, "Token invalid - refreshing")
 	}

backend/filescom/filescom.go

@@ -0,0 +1,901 @@
+// Package filescom provides an interface to the Files.com
+// object storage system.
+package filescom
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+	"strings"
+	"time"
+
+	files_sdk "github.com/Files-com/files-sdk-go/v3"
+	"github.com/Files-com/files-sdk-go/v3/bundle"
+	"github.com/Files-com/files-sdk-go/v3/file"
+	file_migration "github.com/Files-com/files-sdk-go/v3/filemigration"
+	"github.com/Files-com/files-sdk-go/v3/folder"
+	"github.com/Files-com/files-sdk-go/v3/session"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+)
+
+/*
+Run of rclone info
+stringNeedsEscaping = []rune{
+        '/', '\x00'
+}
+maxFileLength = 512 // for 1 byte unicode characters
+maxFileLength = 512 // for 2 byte unicode characters
+maxFileLength = 512 // for 3 byte unicode characters
+maxFileLength = 512 // for 4 byte unicode characters
+canWriteUnnormalized = true
+canReadUnnormalized   = true
+canReadRenormalized   = true
+canStream = true
+*/
+
+const (
+	minSleep      = 10 * time.Millisecond
+	maxSleep      = 2 * time.Second
+	decayConstant = 2 // bigger for slower decay, exponential
+
+	folderNotEmpty = "processing-failure/folder-not-empty"
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "filescom",
+		Description: "Files.com",
+		NewFs:       NewFs,
+		Options: []fs.Option{
+			{
+				Name: "site",
+				Help: "Your site subdomain (e.g. mysite) or custom domain (e.g. myfiles.customdomain.com).",
+			}, {
+				Name: "username",
+				Help: "The username used to authenticate with Files.com.",
+			}, {
+				Name:       "password",
+				Help:       "The password used to authenticate with Files.com.",
+				IsPassword: true,
+			}, {
+				Name:      "api_key",
+				Help:      "The API key used to authenticate with Files.com.",
+				Advanced:  true,
+				Sensitive: true,
+			}, {
+				Name:     config.ConfigEncoding,
+				Help:     config.ConfigEncodingHelp,
+				Advanced: true,
+				Default: (encoder.Display |
+					encoder.EncodeBackSlash |
+					encoder.EncodeRightSpace |
+					encoder.EncodeRightCrLfHtVt |
+					encoder.EncodeInvalidUtf8),
+			}},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	Site     string               `config:"site"`
+	Username string               `config:"username"`
+	Password string               `config:"password"`
+	APIKey   string               `config:"api_key"`
+	Enc      encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents a remote files.com server
+type Fs struct {
+	name            string                 // name of this remote
+	root            string                 // the path we are working on
+	opt             Options                // parsed options
+	features        *fs.Features           // optional features
+	fileClient      *file.Client           // the connection to the file API
+	folderClient    *folder.Client         // the connection to the folder API
+	migrationClient *file_migration.Client // the connection to the file migration API
+	bundleClient    *bundle.Client         // the connection to the bundle API
+	pacer           *fs.Pacer              // pacer for API calls
+}
+
+// Object describes a files object
+//
+// Will definitely have info but maybe not meta
+type Object struct {
+	fs       *Fs       // what this object is part of
+	remote   string    // The remote path
+	size     int64     // size of the object
+	crc32    string    // CRC32 of the object content
+	md5      string    // MD5 of the object content
+	mimeType string    // Content-Type of the object
+	modTime  time.Time // modification time of the object
+}
+
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("files root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// Encode remote and turn it into an absolute path in the share
+func (f *Fs) absPath(remote string) string {
+	return f.opt.Enc.FromStandardPath(path.Join(f.root, remote))
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	429, // Too Many Requests.
+	500, // Internal Server Error
+	502, // Bad Gateway
+	503, // Service Unavailable
+	504, // Gateway Timeout
+	509, // Bandwidth Limit Exceeded
+}
+
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(ctx context.Context, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+
+	if apiErr, ok := err.(files_sdk.ResponseError); ok {
+		for _, e := range retryErrorCodes {
+			if apiErr.HttpCode == e {
+				fs.Debugf(nil, "Retrying API error %v", err)
+				return true, err
+			}
+		}
+	}
+
+	return fserrors.ShouldRetry(err), err
+}
+
+// readMetaDataForPath reads the metadata from the path
+func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *files_sdk.File, err error) {
+	params := files_sdk.FileFindParams{
+		Path: f.absPath(path),
+	}
+
+	var file files_sdk.File
+	err = f.pacer.Call(func() (bool, error) {
+		file, err = f.fileClient.Find(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return &file, nil
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	root = strings.Trim(root, "/")
+
+	config, err := newClientConfig(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+
+	f := &Fs{
+		name:            name,
+		root:            root,
+		opt:             *opt,
+		fileClient:      &file.Client{Config: config},
+		folderClient:    &folder.Client{Config: config},
+		migrationClient: &file_migration.Client{Config: config},
+		bundleClient:    &bundle.Client{Config: config},
+		pacer:           fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+	}
+	f.features = (&fs.Features{
+		CaseInsensitive:          true,
+		CanHaveEmptyDirectories:  true,
+		ReadMimeType:             true,
+		DirModTimeUpdatesOnWrite: true,
+	}).Fill(ctx, f)
+
+	if f.root != "" {
+		info, err := f.readMetaDataForPath(ctx, "")
+		if err == nil && !info.IsDir() {
+			f.root = path.Dir(f.root)
+			if f.root == "." {
+				f.root = ""
+			}
+			return f, fs.ErrorIsFile
+		}
+	}
+
+	return f, err
+}
+
+func newClientConfig(ctx context.Context, opt *Options) (config files_sdk.Config, err error) {
+	if opt.Site != "" {
+		if strings.Contains(opt.Site, ".") {
+			config.EndpointOverride = opt.Site
+		} else {
+			config.Subdomain = opt.Site
+		}
+
+		_, err = url.ParseRequestURI(config.Endpoint())
+		if err != nil {
+			err = fmt.Errorf("invalid domain or subdomain: %v", opt.Site)
+			return
+		}
+	}
+
+	config = config.Init().SetCustomClient(fshttp.NewClient(ctx))
+
+	if opt.APIKey != "" {
+		config.APIKey = opt.APIKey
+		return
+	}
+
+	if opt.Username == "" {
+		err = errors.New("username not found")
+		return
+	}
+	if opt.Password == "" {
+		err = errors.New("password not found")
+		return
+	}
+	opt.Password, err = obscure.Reveal(opt.Password)
+	if err != nil {
+		return
+	}
+
+	sessionClient := session.Client{Config: config}
+	params := files_sdk.SessionCreateParams{
+		Username: opt.Username,
+		Password: opt.Password,
+	}
+
+	thisSession, err := sessionClient.Create(params, files_sdk.WithContext(ctx))
+	if err != nil {
+		err = fmt.Errorf("couldn't create session: %w", err)
+		return
+	}
+
+	config.SessionId = thisSession.Id
+	return
+}
+
+// Return an Object from a path
+//
+// If it can't be found it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, file *files_sdk.File) (fs.Object, error) {
+	o := &Object{
+		fs:     f,
+		remote: remote,
+	}
+	var err error
+	if file != nil {
+		err = o.setMetaData(file)
+	} else {
+		err = o.readMetaData(ctx) // reads info and meta, returning an error
+	}
+	if err != nil {
+		return nil, err
+	}
+	return o, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	return f.newObjectWithInfo(ctx, remote, nil)
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	var it *folder.Iter
+	params := files_sdk.FolderListForParams{
+		Path: f.absPath(dir),
+	}
+
+	err = f.pacer.Call(func() (bool, error) {
+		it, err = f.folderClient.ListFor(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, fmt.Errorf("couldn't list files: %w", err)
+	}
+
+	for it.Next() {
+		item := ptr(it.File())
+		remote := f.opt.Enc.ToStandardPath(item.DisplayName)
+		remote = path.Join(dir, remote)
+		if remote == dir {
+			continue
+		}
+
+		if item.IsDir() {
+			d := fs.NewDir(remote, item.ModTime())
+			entries = append(entries, d)
+		} else {
+			o, err := f.newObjectWithInfo(ctx, remote, item)
+			if err != nil {
+				return nil, err
+			}
+			entries = append(entries, o)
+		}
+	}
+	err = it.Err()
+	if files_sdk.IsNotExist(err) {
+		return nil, fs.ErrorDirNotFound
+	}
+	return
+}
+
+// Creates from the parameters passed in a half finished Object which
+// must have setMetaData called on it
+//
+// Returns the object and error.
+//
+// Used to create new objects
+func (f *Fs) createObject(ctx context.Context, remote string) (o *Object, err error) {
+	// Create the directory for the object if it doesn't exist
+	err = f.mkParentDir(ctx, remote)
+	if err != nil {
+		return
+	}
+	// Temporary Object under construction
+	o = &Object{
+		fs:     f,
+		remote: remote,
+	}
+	return o, nil
+}
+
+// Put the object
+//
+// Copy the reader in to the new object which is returned.
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	// Temporary Object under construction
+	fs := &Object{
+		fs:     f,
+		remote: src.Remote(),
+	}
+	return fs, fs.Update(ctx, in, src, options...)
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
+func (f *Fs) mkdir(ctx context.Context, path string) error {
+	if path == "" || path == "." {
+		return nil
+	}
+
+	params := files_sdk.FolderCreateParams{
+		Path:         path,
+		MkdirParents: ptr(true),
+	}
+
+	err := f.pacer.Call(func() (bool, error) {
+		_, err := f.folderClient.Create(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if files_sdk.IsExist(err) {
+		return nil
+	}
+	return err
+}
+
+// Make the parent directory of remote
+func (f *Fs) mkParentDir(ctx context.Context, remote string) error {
+	return f.mkdir(ctx, path.Dir(f.absPath(remote)))
+}
+
+// Mkdir creates the container if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	return f.mkdir(ctx, f.absPath(dir))
+}
+
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	o := Object{
+		fs:     f,
+		remote: dir,
+	}
+	return o.SetModTime(ctx, modTime)
+}
+
+// purgeCheck removes the root directory, if check is set then it
+// refuses to do so if it has anything in
+func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
+	path := f.absPath(dir)
+	if path == "" || path == "." {
+		return errors.New("can't purge root directory")
+	}
+
+	params := files_sdk.FileDeleteParams{
+		Path:      path,
+		Recursive: ptr(!check),
+	}
+
+	err := f.pacer.Call(func() (bool, error) {
+		err := f.fileClient.Delete(params, files_sdk.WithContext(ctx))
+		// Allow for eventual consistency deletion of child objects.
+		if isFolderNotEmpty(err) {
+			return true, err
+		}
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		if files_sdk.IsNotExist(err) {
+			return fs.ErrorDirNotFound
+		} else if isFolderNotEmpty(err) {
+			return fs.ErrorDirectoryNotEmpty
+		}
+
+		return fmt.Errorf("rmdir failed: %w", err)
+	}
+	return nil
+}
+
+// Rmdir deletes the root folder
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	return f.purgeCheck(ctx, dir, true)
+}
+
+// Precision return the precision of this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Second
+}
+
+// Copy src to this remote using server-side copy operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dstObj fs.Object, err error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+	err = srcObj.readMetaData(ctx)
+	if err != nil {
+		return
+	}
+
+	srcPath := srcObj.fs.absPath(srcObj.remote)
+	dstPath := f.absPath(remote)
+	if strings.EqualFold(srcPath, dstPath) {
+		return nil, fmt.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
+	}
+
+	// Create temporary object
+	dstObj, err = f.createObject(ctx, remote)
+	if err != nil {
+		return
+	}
+
+	// Copy the object
+	params := files_sdk.FileCopyParams{
+		Path:        srcPath,
+		Destination: dstPath,
+		Overwrite:   ptr(true),
+	}
+
+	var action files_sdk.FileAction
+	err = f.pacer.Call(func() (bool, error) {
+		action, err = f.fileClient.Copy(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return
+	}
+
+	err = f.waitForAction(ctx, action, "copy")
+	if err != nil {
+		return
+	}
+
+	err = dstObj.SetModTime(ctx, srcObj.modTime)
+	return
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge(ctx context.Context, dir string) error {
+	return f.purgeCheck(ctx, dir, false)
+}
+
+// move a file or folder
+func (f *Fs) move(ctx context.Context, src *Fs, srcRemote string, dstRemote string) (info *files_sdk.File, err error) {
+	// Move the object
+	params := files_sdk.FileMoveParams{
+		Path:        src.absPath(srcRemote),
+		Destination: f.absPath(dstRemote),
+	}
+
+	var action files_sdk.FileAction
+	err = f.pacer.Call(func() (bool, error) {
+		action, err = f.fileClient.Move(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = f.waitForAction(ctx, action, "move")
+	if err != nil {
+		return nil, err
+	}
+
+	info, err = f.readMetaDataForPath(ctx, dstRemote)
+	return
+}
+
+func (f *Fs) waitForAction(ctx context.Context, action files_sdk.FileAction, operation string) (err error) {
+	var migration files_sdk.FileMigration
+	err = f.pacer.Call(func() (bool, error) {
+		migration, err = f.migrationClient.Wait(action, func(migration files_sdk.FileMigration) {
+			// noop
+		}, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if err == nil && migration.Status != "completed" {
+		return fmt.Errorf("%v did not complete successfully: %v", operation, migration.Status)
+	}
+	return
+}
+
+// Move src to this remote using server-side move operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't move - not same remote type")
+		return nil, fs.ErrorCantMove
+	}
+
+	// Create temporary object
+	dstObj, err := f.createObject(ctx, remote)
+	if err != nil {
+		return nil, err
+	}
+
+	// Do the move
+	info, err := f.move(ctx, srcObj.fs, srcObj.remote, dstObj.remote)
+	if err != nil {
+		return nil, err
+	}
+
+	err = dstObj.setMetaData(info)
+	if err != nil {
+		return nil, err
+	}
+	return dstObj, nil
+}
+
+// DirMove moves src, srcRemote to this remote at dstRemote
+// using server-side move operations.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantDirMove
+//
+// If destination exists then return fs.ErrorDirExists
+func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) (err error) {
+	srcFs, ok := src.(*Fs)
+	if !ok {
+		fs.Debugf(srcFs, "Can't move directory - not same remote type")
+		return fs.ErrorCantDirMove
+	}
+
+	// Check if destination exists
+	_, err = f.readMetaDataForPath(ctx, dstRemote)
+	if err == nil {
+		return fs.ErrorDirExists
+	}
+
+	// Create temporary object
+	dstObj, err := f.createObject(ctx, dstRemote)
+	if err != nil {
+		return
+	}
+
+	// Do the move
+	_, err = f.move(ctx, srcFs, srcRemote, dstObj.remote)
+	return
+}
+
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (url string, err error) {
+	params := files_sdk.BundleCreateParams{
+		Paths: []string{f.absPath(remote)},
+	}
+	if expire < fs.DurationOff {
+		params.ExpiresAt = ptr(time.Now().Add(time.Duration(expire)))
+	}
+
+	var bundle files_sdk.Bundle
+	err = f.pacer.Call(func() (bool, error) {
+		bundle, err = f.bundleClient.Create(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+
+	url = bundle.Url
+	return
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.NewHashSet(hash.CRC32, hash.MD5)
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns the MD5 of an object returning a lowercase hex string
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	switch t {
+	case hash.CRC32:
+		if o.crc32 == "" {
+			return "", nil
+		}
+		return fmt.Sprintf("%08s", o.crc32), nil
+	case hash.MD5:
+		return o.md5, nil
+	}
+	return "", hash.ErrUnsupported
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// setMetaData sets the metadata from info
+func (o *Object) setMetaData(file *files_sdk.File) error {
+	o.modTime = file.ModTime()
+
+	if !file.IsDir() {
+		o.size = file.Size
+		o.crc32 = file.Crc32
+		o.md5 = file.Md5
+		o.mimeType = file.MimeType
+	}
+
+	return nil
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// it also sets the info
+func (o *Object) readMetaData(ctx context.Context) (err error) {
+	file, err := o.fs.readMetaDataForPath(ctx, o.remote)
+	if err != nil {
+		if files_sdk.IsNotExist(err) {
+			return fs.ErrorObjectNotFound
+		}
+		return err
+	}
+	if file.IsDir() {
+		return fs.ErrorIsDir
+	}
+	return o.setMetaData(file)
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {
+	params := files_sdk.FileUpdateParams{
+		Path:          o.fs.absPath(o.remote),
+		ProvidedMtime: &modTime,
+	}
+
+	var file files_sdk.File
+	err = o.fs.pacer.Call(func() (bool, error) {
+		file, err = o.fs.fileClient.Update(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return err
+	}
+	return o.setMetaData(&file)
+}
+
+// Storable returns a boolean showing whether this object storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	// Offset and Count for range download
+	var offset, count int64
+	fs.FixRangeOption(options, o.size)
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			offset, count = x.Decode(o.size)
+			if count < 0 {
+				count = o.size - offset
+			}
+		case *fs.SeekOption:
+			offset = x.Offset
+			count = o.size - offset
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	params := files_sdk.FileDownloadParams{
+		Path: o.fs.absPath(o.remote),
+	}
+
+	headers := &http.Header{}
+	headers.Set("Range", fmt.Sprintf("bytes=%v-%v", offset, offset+count-1))
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, err = o.fs.fileClient.Download(
+			params,
+			files_sdk.WithContext(ctx),
+			files_sdk.RequestHeadersOption(headers),
+			files_sdk.ResponseBodyOption(func(closer io.ReadCloser) error {
+				in = closer
+				return err
+			}),
+		)
+		return shouldRetry(ctx, err)
+	})
+	return
+}
+
+// Returns a pointer to t - useful for returning pointers to constants
+func ptr[T any](t T) *T {
+	return &t
+}
+
+func isFolderNotEmpty(err error) bool {
+	var re files_sdk.ResponseError
+	ok := errors.As(err, &re)
+	return ok && re.Type == folderNotEmpty
+}
+
+// Update the object with the contents of the io.Reader, modTime and size
+//
+// If existing is set then it updates the object rather than creating a new one.
+//
+// The new object may have been created if an error is returned.
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	uploadOpts := []file.UploadOption{
+		file.UploadWithContext(ctx),
+		file.UploadWithReader(in),
+		file.UploadWithDestinationPath(o.fs.absPath(o.remote)),
+		file.UploadWithProvidedMtime(src.ModTime(ctx)),
+	}
+
+	err := o.fs.pacer.Call(func() (bool, error) {
+		err := o.fs.fileClient.Upload(uploadOpts...)
+		return shouldRetry(ctx, err)
+	})
+	if err != nil {
+		return err
+	}
+
+	return o.readMetaData(ctx)
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	params := files_sdk.FileDeleteParams{
+		Path: o.fs.absPath(o.remote),
+	}
+
+	return o.fs.pacer.Call(func() (bool, error) {
+		err := o.fs.fileClient.Delete(params, files_sdk.WithContext(ctx))
+		return shouldRetry(ctx, err)
+	})
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.mimeType
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs           = (*Fs)(nil)
+	_ fs.Purger       = (*Fs)(nil)
+	_ fs.PutStreamer  = (*Fs)(nil)
+	_ fs.Copier       = (*Fs)(nil)
+	_ fs.Mover        = (*Fs)(nil)
+	_ fs.DirMover     = (*Fs)(nil)
+	_ fs.PublicLinker = (*Fs)(nil)
+	_ fs.Object       = (*Object)(nil)
+	_ fs.MimeTyper    = (*Object)(nil)
+)

backend/filescom/filescom_test.go

@@ -0,0 +1,17 @@
+// Test Files filesystem interface
+package filescom_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/filescom"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFilesCom:",
+		NilObject:  (*filescom.Object)(nil),
+	})
+}

backend/ftp/ftp.go

@@ -15,7 +15,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/rclone/ftp"
+	"github.com/jlaffaye/ftp"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/config"
@@ -85,7 +85,7 @@ to an encrypted one. Cannot be used in combination with implicit FTPS.`,
 			Default: false,
 		}, {
 			Name: "concurrency",
-			Help: strings.Replace(`Maximum number of FTP simultaneous connections, 0 for unlimited.
+			Help: strings.ReplaceAll(`Maximum number of FTP simultaneous connections, 0 for unlimited.
 
 Note that setting this is very likely to cause deadlocks so it should
 be used with care.
@@ -99,7 +99,7 @@ maximum of |--checkers| and |--transfers|.
 So for |concurrency 3| you'd use |--checkers 2 --transfers 2
 --check-first| or |--checkers 1 --transfers 1|.
 
-`, "|", "`", -1),
+`, "|", "`"),
 			Default:  0,
 			Advanced: true,
 		}, {
@@ -249,7 +249,6 @@ type Fs struct {
 	pool     []*ftp.ServerConn
 	drain    *time.Timer // used to drain the pool when we stop using the connections
 	tokens   *pacer.TokenDispenser
-	tlsConf  *tls.Config
 	pacer    *fs.Pacer // pacer for FTP connections
 	fGetTime bool      // true if the ftp library accepts GetTime
 	fSetTime bool      // true if the ftp library accepts SetTime
@@ -362,10 +361,36 @@ func shouldRetry(ctx context.Context, err error) (bool, error) {
 	return fserrors.ShouldRetry(err), err
 }
 
+// Get a TLS config with a unique session cache.
+//
+// We can't share session caches between connections.
+//
+// See: https://github.com/rclone/rclone/issues/7234
+func (f *Fs) tlsConfig() *tls.Config {
+	var tlsConfig *tls.Config
+	if f.opt.TLS || f.opt.ExplicitTLS {
+		tlsConfig = &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
+		}
+		if f.opt.TLSCacheSize > 0 {
+			tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(f.opt.TLSCacheSize)
+		}
+		if f.opt.DisableTLS13 {
+			tlsConfig.MaxVersion = tls.VersionTLS12
+		}
+	}
+	return tlsConfig
+}
+
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	fs.Debugf(f, "Connecting to FTP server")
 
+	// tls.Config for this connection only. Will be used for data
+	// and control connections.
+	tlsConfig := f.tlsConfig()
+
 	// Make ftp library dial with fshttp dialer optionally using TLS
 	initialConnection := true
 	dial := func(network, address string) (conn net.Conn, err error) {
@@ -383,7 +408,7 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 			return nil, err
 		}
 		// Connect using cleartext only for non TLS
-		if f.tlsConf == nil {
+		if tlsConfig == nil {
 			return conn, nil
 		}
 		// Initial connection only needs to be cleartext for explicit TLS
@@ -392,7 +417,7 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 			return conn, nil
 		}
 		// Upgrade connection to TLS
-		tlsConn := tls.Client(conn, f.tlsConf)
+		tlsConn := tls.Client(conn, tlsConfig)
 		// Do the initial handshake - tls.Client doesn't do it for us
 		// If we do this then connections to proftpd/pureftpd lock up
 		// See: https://github.com/rclone/rclone/issues/6426
@@ -414,9 +439,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
 	if f.opt.TLS {
 		// Our dialer takes care of TLS but ftp library also needs tlsConf
 		// as a trigger for sending PSBZ and PROT options to server.
-		ftpConfig = append(ftpConfig, ftp.DialWithTLS(f.tlsConf))
+		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
 	} else if f.opt.ExplicitTLS {
-		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(f.tlsConf))
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -571,19 +596,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 	if opt.TLS && opt.ExplicitTLS {
 		return nil, errors.New("implicit TLS and explicit TLS are mutually incompatible, please revise your config")
 	}
-	var tlsConfig *tls.Config
-	if opt.TLS || opt.ExplicitTLS {
-		tlsConfig = &tls.Config{
-			ServerName:         opt.Host,
-			InsecureSkipVerify: opt.SkipVerifyTLSCert,
-		}
-		if opt.TLSCacheSize > 0 {
-			tlsConfig.ClientSessionCache = tls.NewLRUClientSessionCache(opt.TLSCacheSize)
-		}
-		if opt.DisableTLS13 {
-			tlsConfig.MaxVersion = tls.VersionTLS12
-		}
-	}
 	u := protocol + path.Join(dialAddr+"/", root)
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
@@ -596,7 +608,6 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs
 		pass:     pass,
 		dialAddr: dialAddr,
 		tokens:   pacer.NewTokenDispenser(opt.Concurrency),
-		tlsConf:  tlsConfig,
 		pacer:    fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
@@ -959,6 +970,8 @@ func (f *Fs) mkdir(ctx context.Context, abspath string) error {
 	f.putFtpConnection(&c, err)
 	if errX := textprotoError(err); errX != nil {
 		switch errX.Code {
+		case ftp.StatusRequestedFileActionOK: // some ftp servers apparently return 250 instead of 257
+			err = nil // see: https://forum.rclone.org/t/rclone-pop-up-an-i-o-error-when-creating-a-folder-in-a-mounted-ftp-drive/44368/
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
 		case 521: // dir already exists: error number according to RFC 959: issue #2363

backend/gofile/api/types.go

@@ -0,0 +1,311 @@
+// Package api has type definitions for gofile
+//
+// Converted from the API docs with help from https://mholt.github.io/json-to-go/
+package api
+
+import (
+	"fmt"
+	"time"
+)
+
+const (
+	// 2017-05-03T07:26:10-07:00
+	timeFormat = `"` + time.RFC3339 + `"`
+)
+
+// Time represents date and time information for the
+// gofile API, by using RFC3339
+type Time time.Time
+
+// MarshalJSON turns a Time into JSON (in UTC)
+func (t *Time) MarshalJSON() (out []byte, err error) {
+	timeString := (*time.Time)(t).Format(timeFormat)
+	return []byte(timeString), nil
+}
+
+// UnmarshalJSON turns JSON into a Time
+func (t *Time) UnmarshalJSON(data []byte) error {
+	newT, err := time.Parse(timeFormat, string(data))
+	if err != nil {
+		return err
+	}
+	*t = Time(newT)
+	return nil
+}
+
+// Error is returned from gofile when things go wrong
+type Error struct {
+	Status string `json:"status"`
+}
+
+// Error returns a string for the error and satisfies the error interface
+func (e Error) Error() string {
+	out := fmt.Sprintf("Error %q", e.Status)
+	return out
+}
+
+// IsError returns true if there is an error
+func (e Error) IsError() bool {
+	return e.Status != "ok"
+}
+
+// Err returns err if not nil, or e if IsError or nil
+func (e Error) Err(err error) error {
+	if err != nil {
+		return err
+	}
+	if e.IsError() {
+		return e
+	}
+	return nil
+}
+
+// Check Error satisfies the error interface
+var _ error = (*Error)(nil)
+
+// Types of things in Item
+const (
+	ItemTypeFolder = "folder"
+	ItemTypeFile   = "file"
+)
+
+// Item describes a folder or a file as returned by /contents
+type Item struct {
+	ID            string                 `json:"id"`
+	ParentFolder  string                 `json:"parentFolder"`
+	Type          string                 `json:"type"`
+	Name          string                 `json:"name"`
+	Size          int64                  `json:"size"`
+	Code          string                 `json:"code"`
+	CreateTime    int64                  `json:"createTime"`
+	ModTime       int64                  `json:"modTime"`
+	Link          string                 `json:"link"`
+	MD5           string                 `json:"md5"`
+	MimeType      string                 `json:"mimetype"`
+	ChildrenCount int                    `json:"childrenCount"`
+	DirectLinks   map[string]*DirectLink `json:"directLinks"`
+	//Public         bool     `json:"public"`
+	//ServerSelected string   `json:"serverSelected"`
+	//Thumbnail      string   `json:"thumbnail"`
+	//DownloadCount int      `json:"downloadCount"`
+	//TotalDownloadCount int64            `json:"totalDownloadCount"`
+	//TotalSize int64            `json:"totalSize"`
+	//ChildrenIDs   []string               `json:"childrenIds"`
+	Children map[string]*Item `json:"children"`
+}
+
+// ToNativeTime converts a go time to a native time
+func ToNativeTime(t time.Time) int64 {
+	return t.Unix()
+}
+
+// FromNativeTime converts native time to a go time
+func FromNativeTime(t int64) time.Time {
+	return time.Unix(t, 0)
+}
+
+// DirectLink describes a direct link to a file so it can be
+// downloaded by third parties.
+type DirectLink struct {
+	ExpireTime       int64  `json:"expireTime"`
+	SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
+	DomainsAllowed   []any  `json:"domainsAllowed"`
+	Auth             []any  `json:"auth"`
+	IsReqLink        bool   `json:"isReqLink"`
+	DirectLink       string `json:"directLink"`
+}
+
+// Contents is returned from the /contents call
+type Contents struct {
+	Error
+	Data struct {
+		Item
+	} `json:"data"`
+	Metadata Metadata `json:"metadata"`
+}
+
+// Metadata is returned when paging is in use
+type Metadata struct {
+	TotalCount  int  `json:"totalCount"`
+	TotalPages  int  `json:"totalPages"`
+	Page        int  `json:"page"`
+	PageSize    int  `json:"pageSize"`
+	HasNextPage bool `json:"hasNextPage"`
+}
+
+// AccountsGetID is the result of /accounts/getid
+type AccountsGetID struct {
+	Error
+	Data struct {
+		ID string `json:"id"`
+	} `json:"data"`
+}
+
+// Stats of storage and traffic
+type Stats struct {
+	FolderCount            int64 `json:"folderCount"`
+	FileCount              int64 `json:"fileCount"`
+	Storage                int64 `json:"storage"`
+	TrafficDirectGenerated int64 `json:"trafficDirectGenerated"`
+	TrafficReqDownloaded   int64 `json:"trafficReqDownloaded"`
+	TrafficWebDownloaded   int64 `json:"trafficWebDownloaded"`
+}
+
+// AccountsGet is the result of /accounts/{id}
+type AccountsGet struct {
+	Error
+	Data struct {
+		ID                             string `json:"id"`
+		Email                          string `json:"email"`
+		Tier                           string `json:"tier"`
+		PremiumType                    string `json:"premiumType"`
+		Token                          string `json:"token"`
+		RootFolder                     string `json:"rootFolder"`
+		SubscriptionProvider           string `json:"subscriptionProvider"`
+		SubscriptionEndDate            int    `json:"subscriptionEndDate"`
+		SubscriptionLimitDirectTraffic int64  `json:"subscriptionLimitDirectTraffic"`
+		SubscriptionLimitStorage       int64  `json:"subscriptionLimitStorage"`
+		StatsCurrent                   Stats  `json:"statsCurrent"`
+		// StatsHistory                   map[int]map[int]map[int]Stats `json:"statsHistory"`
+	} `json:"data"`
+}
+
+// CreateFolderRequest is the input to /contents/createFolder
+type CreateFolderRequest struct {
+	ParentFolderID string `json:"parentFolderId"`
+	FolderName     string `json:"folderName"`
+	ModTime        int64  `json:"modTime,omitempty"`
+}
+
+// CreateFolderResponse is the output from /contents/createFolder
+type CreateFolderResponse struct {
+	Error
+	Data Item `json:"data"`
+}
+
+// DeleteRequest is the input to DELETE /contents
+type DeleteRequest struct {
+	ContentsID string `json:"contentsId"` // comma separated list of IDs
+}
+
+// DeleteResponse is the input to DELETE /contents
+type DeleteResponse struct {
+	Error
+	Data map[string]Error
+}
+
+// Server is an upload server
+type Server struct {
+	Name string `json:"name"`
+	Zone string `json:"zone"`
+}
+
+// String returns a string representation of the Server
+func (s *Server) String() string {
+	return fmt.Sprintf("%s (%s)", s.Name, s.Zone)
+}
+
+// Root returns the root URL for the server
+func (s *Server) Root() string {
+	return fmt.Sprintf("https://%s.gofile.io/", s.Name)
+}
+
+// URL returns the upload URL for the server
+func (s *Server) URL() string {
+	return fmt.Sprintf("https://%s.gofile.io/contents/uploadfile", s.Name)
+}
+
+// ServersResponse is the output from /servers
+type ServersResponse struct {
+	Error
+	Data struct {
+		Servers []Server `json:"servers"`
+	} `json:"data"`
+}
+
+// UploadResponse is returned by POST /contents/uploadfile
+type UploadResponse struct {
+	Error
+	Data Item `json:"data"`
+}
+
+// DirectLinksRequest specifies the parameters for the direct link
+type DirectLinksRequest struct {
+	ExpireTime       int64 `json:"expireTime,omitempty"`
+	SourceIpsAllowed []any `json:"sourceIpsAllowed,omitempty"`
+	DomainsAllowed   []any `json:"domainsAllowed,omitempty"`
+	Auth             []any `json:"auth,omitempty"`
+}
+
+// DirectLinksResult is returned from POST /contents/{id}/directlinks
+type DirectLinksResult struct {
+	Error
+	Data struct {
+		ExpireTime       int64  `json:"expireTime"`
+		SourceIpsAllowed []any  `json:"sourceIpsAllowed"`
+		DomainsAllowed   []any  `json:"domainsAllowed"`
+		Auth             []any  `json:"auth"`
+		IsReqLink        bool   `json:"isReqLink"`
+		ID               string `json:"id"`
+		DirectLink       string `json:"directLink"`
+	} `json:"data"`
+}
+
+// UpdateItemRequest describes the updates to be done to an item for PUT /contents/{id}/update
+//
+// The Value of the attribute to define :
+// For Attribute "name" : The name of the content (file or folder)
+// For Attribute "description" : The description displayed on the download page (folder only)
+// For Attribute "tags" : A comma-separated list of tags (folder only)
+// For Attribute "public" : either true or false (folder only)
+// For Attribute "expiry" : A unix timestamp of the expiration date (folder only)
+// For Attribute "password" : The password to set (folder only)
+type UpdateItemRequest struct {
+	Attribute string `json:"attribute"`
+	Value     any    `json:"attributeValue"`
+}
+
+// UpdateItemResponse is returned by PUT /contents/{id}/update
+type UpdateItemResponse struct {
+	Error
+	Data Item `json:"data"`
+}
+
+// MoveRequest is the input to /contents/move
+type MoveRequest struct {
+	FolderID   string `json:"folderId"`
+	ContentsID string `json:"contentsId"` // comma separated list of IDs
+}
+
+// MoveResponse is returned by POST /contents/move
+type MoveResponse struct {
+	Error
+	Data map[string]struct {
+		Error
+		Item `json:"data"`
+	} `json:"data"`
+}
+
+// CopyRequest is the input to /contents/copy
+type CopyRequest struct {
+	FolderID   string `json:"folderId"`
+	ContentsID string `json:"contentsId"` // comma separated list of IDs
+}
+
+// CopyResponse is returned by POST /contents/copy
+type CopyResponse struct {
+	Error
+	Data map[string]struct {
+		Error
+		Item `json:"data"`
+	} `json:"data"`
+}
+
+// UploadServerStatus is returned when fetching the root of an upload server
+type UploadServerStatus struct {
+	Error
+	Data struct {
+		Server string `json:"server"`
+		Test   string `json:"test"`
+	} `json:"data"`
+}

backend/gofile/gofile_test.go

@@ -0,0 +1,17 @@
+// Test Gofile filesystem interface
+package gofile_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/gofile"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestGoFile:",
+		NilObject:  (*gofile.Object)(nil),
+	})
+}

backend/googlecloudstorage/googlecloudstorage.go

@@ -697,7 +697,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			// is this a directory marker?
 			if isDirectory {
 				// Don't insert the root directory
-				if remote == directory {
+				if remote == f.opt.Enc.ToStandardPath(directory) {
 					continue
 				}
 				// process directory markers as directories
@@ -1310,10 +1310,11 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	url := o.url
 	if o.fs.opt.UserProject != "" {
-		o.url = o.url + "&userProject=" + o.fs.opt.UserProject
+		url += "&userProject=" + o.fs.opt.UserProject
 	}
-	req, err := http.NewRequestWithContext(ctx, "GET", o.url, nil)
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
 		return nil, err
 	}

backend/googlephotos/api/types.go

@@ -56,8 +56,7 @@ type MediaItem struct {
 		CreationTime time.Time `json:"creationTime"`
 		Width        string    `json:"width"`
 		Height       string    `json:"height"`
-		Photo        struct {
-		} `json:"photo"`
+		Photo        struct{}  `json:"photo"`
 	} `json:"mediaMetadata"`
 	Filename string `json:"filename"`
 }
@@ -68,7 +67,7 @@ type MediaItems struct {
 	NextPageToken string      `json:"nextPageToken"`
 }
 
-//Content categories
+// Content categories
 // NONE	Default content category. This category is ignored when any other category is used in the filter.
 // LANDSCAPES	Media items containing landscapes.
 // RECEIPTS	Media items containing receipts.
@@ -187,5 +186,5 @@ type BatchCreateResponse struct {
 
 // BatchRemoveItems is for removing items from an album
 type BatchRemoveItems struct {
-	MediaItemIds []string `json:"mediaItemIds"`
+	MediaItemIDs []string `json:"mediaItemIds"`
 }

backend/googlephotos/googlephotos.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/log"
+	"github.com/rclone/rclone/lib/batcher"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
@@ -71,6 +72,14 @@ var (
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectURL,
 	}
+
+	// Configure the batcher
+	defaultBatcherOptions = batcher.Options{
+		MaxBatchSize:          50,
+		DefaultTimeoutSync:    1000 * time.Millisecond,
+		DefaultTimeoutAsync:   10 * time.Second,
+		DefaultBatchSizeAsync: 50,
+	}
 )
 
 // Register with Fs
@@ -111,7 +120,7 @@ will count towards storage in your Google Account.`)
 			}
 			return nil, fmt.Errorf("unknown state %q", config.State)
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: append(append(oauthutil.SharedOptions, []fs.Option{{
 			Name:    "read_only",
 			Default: false,
 			Help: `Set to make the Google Photos backend read only.
@@ -158,7 +167,7 @@ listings and won't be transferred.`,
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}}...), defaultBatcherOptions.FsOptions("")...),
 	})
 }
 
@@ -169,6 +178,9 @@ type Options struct {
 	StartYear       int                  `config:"start_year"`
 	IncludeArchived bool                 `config:"include_archived"`
 	Enc             encoder.MultiEncoder `config:"encoding"`
+	BatchMode       string               `config:"batch_mode"`
+	BatchSize       int                  `config:"batch_size"`
+	BatchTimeout    fs.Duration          `config:"batch_timeout"`
 }
 
 // Fs represents a remote storage server
@@ -187,6 +199,7 @@ type Fs struct {
 	uploadedMu sync.Mutex             // to protect the below
 	uploaded   dirtree.DirTree        // record of uploaded items
 	createMu   sync.Mutex             // held when creating albums to prevent dupes
+	batcher    *batcher.Batcher[uploadedItem, *api.MediaItem]
 }
 
 // Object describes a storage object
@@ -267,7 +280,7 @@ func errorHandler(resp *http.Response) error {
 	if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
 		body = []byte("Image not found or broken")
 	}
-	var e = api.Error{
+	e := api.Error{
 		Details: api.ErrorDetails{
 			Code:    resp.StatusCode,
 			Message: string(body),
@@ -312,6 +325,14 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		albums:    map[bool]*albums{},
 		uploaded:  dirtree.New(),
 	}
+	batcherOptions := defaultBatcherOptions
+	batcherOptions.Mode = f.opt.BatchMode
+	batcherOptions.Size = f.opt.BatchSize
+	batcherOptions.Timeout = time.Duration(f.opt.BatchTimeout)
+	f.batcher, err = batcher.New(ctx, f, f.commitBatch, batcherOptions)
+	if err != nil {
+		return nil, err
+	}
 	f.features = (&fs.Features{
 		ReadMimeType: true,
 	}).Fill(ctx, f)
@@ -599,9 +620,7 @@ func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter
 		if err != nil {
 			return err
 		}
-		if entry != nil {
-			entries = append(entries, entry)
-		}
+		entries = append(entries, entry)
 		return nil
 	})
 	if err != nil {
@@ -681,7 +700,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 		Path:       "/albums",
 		Parameters: url.Values{},
 	}
-	var request = api.CreateAlbum{
+	request := api.CreateAlbum{
 		Album: &api.Album{
 			Title: albumTitle,
 		},
@@ -781,6 +800,13 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.None)
 }
 
+// Shutdown the backend, closing any background tasks and any
+// cached connections.
+func (f *Fs) Shutdown(ctx context.Context) error {
+	f.batcher.Shutdown()
+	return nil
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -961,6 +987,82 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return resp.Body, err
 }
 
+// input to the batcher
+type uploadedItem struct {
+	AlbumID     string // desired album
+	UploadToken string // upload ID
+}
+
+// Commit a batch of items to albumID returning the errors in errors
+func (f *Fs) commitBatchAlbumID(ctx context.Context, items []uploadedItem, results []*api.MediaItem, errors []error, albumID string) {
+	// Create the media item from an UploadToken, optionally adding to an album
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/mediaItems:batchCreate",
+	}
+	request := api.BatchCreateRequest{
+		AlbumID: albumID,
+	}
+	itemsInBatch := 0
+	for i := range items {
+		if items[i].AlbumID == albumID {
+			request.NewMediaItems = append(request.NewMediaItems, api.NewMediaItem{
+				SimpleMediaItem: api.SimpleMediaItem{
+					UploadToken: items[i].UploadToken,
+				},
+			})
+			itemsInBatch++
+		}
+	}
+	var result api.BatchCreateResponse
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
+		return shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		err = fmt.Errorf("failed to create media item: %w", err)
+	}
+	if err == nil && len(result.NewMediaItemResults) != itemsInBatch {
+		err = fmt.Errorf("bad response to BatchCreate expecting %d items but got %d", itemsInBatch, len(result.NewMediaItemResults))
+	}
+	j := 0
+	for i := range items {
+		if items[i].AlbumID == albumID {
+			if err == nil {
+				media := &result.NewMediaItemResults[j]
+				if media.Status.Code != 0 {
+					errors[i] = fmt.Errorf("upload failed: %s (%d)", media.Status.Message, media.Status.Code)
+				} else {
+					results[i] = &media.MediaItem
+				}
+			} else {
+				errors[i] = err
+			}
+			j++
+		}
+	}
+}
+
+// Called by the batcher to commit a batch
+func (f *Fs) commitBatch(ctx context.Context, items []uploadedItem, results []*api.MediaItem, errors []error) (err error) {
+	// Discover all the AlbumIDs as we have to upload these separately
+	//
+	// Should maybe have one batcher per AlbumID
+	albumIDs := map[string]struct{}{}
+	for i := range items {
+		albumIDs[items[i].AlbumID] = struct{}{}
+	}
+
+	// batch the albums
+	for albumID := range albumIDs {
+		// errors returned in errors
+		f.commitBatchAlbumID(ctx, items, results, errors, albumID)
+	}
+	return nil
+}
+
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // The new object may have been created if an error is returned
@@ -1021,37 +1123,29 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return errors.New("empty upload token")
 	}
 
-	// Create the media item from an UploadToken, optionally adding to an album
-	opts = rest.Opts{
-		Method: "POST",
-		Path:   "/mediaItems:batchCreate",
+	uploaded := uploadedItem{
+		AlbumID:     albumID,
+		UploadToken: uploadToken,
 	}
-	var request = api.BatchCreateRequest{
-		AlbumID: albumID,
-		NewMediaItems: []api.NewMediaItem{
-			{
-				SimpleMediaItem: api.SimpleMediaItem{
-					UploadToken: uploadToken,
-				},
-			},
-		},
+
+	// Save the upload into an album
+	var info *api.MediaItem
+	if o.fs.batcher.Batching() {
+		info, err = o.fs.batcher.Commit(ctx, o.remote, uploaded)
+	} else {
+		errors := make([]error, 1)
+		results := make([]*api.MediaItem, 1)
+		err = o.fs.commitBatch(ctx, []uploadedItem{uploaded}, results, errors)
+		if err != nil {
+			err = errors[0]
+			info = results[0]
+		}
 	}
-	var result api.BatchCreateResponse
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
-		return shouldRetry(ctx, resp, err)
-	})
 	if err != nil {
-		return fmt.Errorf("failed to create media item: %w", err)
+		return fmt.Errorf("failed to commit batch: %w", err)
 	}
-	if len(result.NewMediaItemResults) != 1 {
-		return errors.New("bad response to BatchCreate wrong number of items")
-	}
-	mediaItemResult := result.NewMediaItemResults[0]
-	if mediaItemResult.Status.Code != 0 {
-		return fmt.Errorf("upload failed: %s (%d)", mediaItemResult.Status.Message, mediaItemResult.Status.Code)
-	}
-	o.setMetaData(&mediaItemResult.MediaItem)
+
+	o.setMetaData(info)
 
 	// Add upload to internal storage
 	if pattern.isUpload {
@@ -1078,8 +1172,8 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		Path:       "/albums/" + album.ID + ":batchRemoveMediaItems",
 		NoResponse: true,
 	}
-	var request = api.BatchRemoveItems{
-		MediaItemIds: []string{o.id},
+	request := api.BatchRemoveItems{
+		MediaItemIDs: []string{o.id},
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/googlephotos/pattern.go

@@ -38,7 +38,7 @@ type dirPattern struct {
 	toEntries func(ctx context.Context, f lister, prefix string, match []string) (fs.DirEntries, error)
 }
 
-// dirPatters is a slice of all the directory patterns
+// dirPatterns is a slice of all the directory patterns
 type dirPatterns []dirPattern
 
 // patterns describes the layout of the google photos backend file system.

backend/hasher/commands.go

@@ -80,6 +80,14 @@ func (f *Fs) dbDump(ctx context.Context, full bool, root string) error {
 		}
 		root = fspath.JoinRootPath(remoteFs.Root(), f.Root())
 	}
+	if f.db == nil {
+		if f.opt.MaxAge == 0 {
+			fs.Errorf(f, "db not found. (disabled with max_age = 0)")
+		} else {
+			fs.Errorf(f, "db not found.")
+		}
+		return kv.ErrInactive
+	}
 	op := &kvDump{
 		full: full,
 		root: root,

backend/hasher/hasher.go

@@ -114,6 +114,13 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 		root: rpath,
 		opt:  opt,
 	}
+	// Correct root if definitely pointing to a file
+	if err == fs.ErrorIsFile {
+		f.root = path.Dir(f.root)
+		if f.root == "." || f.root == "/" {
+			f.root = ""
+		}
+	}
 	baseFeatures := baseFs.Features()
 	f.fpTime = baseFs.Precision() != fs.ModTimeNotSupported
 
@@ -157,16 +164,21 @@ func NewFs(ctx context.Context, fsname, rpath string, cmap configmap.Mapper) (fs
 	}
 
 	stubFeatures := &fs.Features{
-		CanHaveEmptyDirectories: true,
-		IsLocal:                 true,
-		ReadMimeType:            true,
-		WriteMimeType:           true,
-		SetTier:                 true,
-		GetTier:                 true,
-		ReadMetadata:            true,
-		WriteMetadata:           true,
-		UserMetadata:            true,
-		PartialUploads:          true,
+		CanHaveEmptyDirectories:  true,
+		IsLocal:                  true,
+		ReadMimeType:             true,
+		WriteMimeType:            true,
+		SetTier:                  true,
+		GetTier:                  true,
+		ReadMetadata:             true,
+		WriteMetadata:            true,
+		UserMetadata:             true,
+		ReadDirMetadata:          true,
+		WriteDirMetadata:         true,
+		WriteDirSetModTime:       true,
+		UserDirMetadata:          true,
+		DirModTimeUpdatesOnWrite: true,
+		PartialUploads:           true,
 	}
 	f.features = stubFeatures.Fill(ctx, f).Mask(ctx, f.Fs).WrapsFs(f, f.Fs)
 
@@ -334,6 +346,22 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	return errors.New("MergeDirs not supported")
 }
 
+// DirSetModTime sets the directory modtime for dir
+func (f *Fs) DirSetModTime(ctx context.Context, dir string, modTime time.Time) error {
+	if do := f.Fs.Features().DirSetModTime; do != nil {
+		return do(ctx, dir, modTime)
+	}
+	return fs.ErrorNotImplemented
+}
+
+// MkdirMetadata makes the root directory of the Fs object
+func (f *Fs) MkdirMetadata(ctx context.Context, dir string, metadata fs.Metadata) (fs.Directory, error) {
+	if do := f.Fs.Features().MkdirMetadata; do != nil {
+		return do(ctx, dir, metadata)
+	}
+	return nil, fs.ErrorNotImplemented
+}
+
 // DirCacheFlush resets the directory cache - used in testing
 // as an optional interface
 func (f *Fs) DirCacheFlush() {
@@ -411,7 +439,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Shutdown the backend, closing any background tasks and any cached connections.
 func (f *Fs) Shutdown(ctx context.Context) (err error) {
-	err = f.db.Stop(false)
+	if f.db != nil && !f.db.IsStopped() {
+		err = f.db.Stop(false)
+	}
 	if do := f.Fs.Features().Shutdown; do != nil {
 		if err2 := do(ctx); err2 != nil {
 			err = err2
@@ -505,6 +535,17 @@ func (o *Object) Metadata(ctx context.Context) (fs.Metadata, error) {
 	return do.Metadata(ctx)
 }
 
+// SetMetadata sets metadata for an Object
+//
+// It should return fs.ErrorNotImplemented if it can't set metadata
+func (o *Object) SetMetadata(ctx context.Context, metadata fs.Metadata) error {
+	do, ok := o.Object.(fs.SetMetadataer)
+	if !ok {
+		return fs.ErrorNotImplemented
+	}
+	return do.SetMetadata(ctx, metadata)
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -521,6 +562,8 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Wrapper         = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.DirSetModTimer  = (*Fs)(nil)
+	_ fs.MkdirMetadataer = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)

backend/hasher/hasher_internal_test.go

@@ -60,9 +60,11 @@ func (f *Fs) testUploadFromCrypt(t *testing.T) {
 	assert.NotNil(t, dst)
 
 	// check that hash was created
-	hash, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
-	assert.NoError(t, err)
-	assert.NotEmpty(t, hash)
+	if f.opt.MaxAge > 0 {
+		hash, err = f.getRawHash(ctx, hashType, fileName, anyFingerprint, longTime)
+		assert.NoError(t, err)
+		assert.NotEmpty(t, hash)
+	}
 	//t.Logf("hash is %q", hash)
 	_ = operations.Purge(ctx, f, dirName)
 }

backend/hasher/hasher_test.go

@@ -23,6 +23,7 @@ func TestIntegration(t *testing.T) {
 		NilObject:  (*hasher.Object)(nil),
 		UnimplementableFsMethods: []string{
 			"OpenWriterAt",
+			"OpenChunkWriter",
 		},
 		UnimplementableObjectMethods: []string{},
 	}
@@ -36,4 +37,9 @@ func TestIntegration(t *testing.T) {
 		opt.QuickTestOK = true
 	}
 	fstests.Run(t, &opt)
+	// test again with MaxAge = 0
+	if *fstest.RemoteName == "" {
+		opt.ExtraConfig = append(opt.ExtraConfig, fstests.ExtraConfigItem{Name: "TestHasher", Key: "max_age", Value: "0"})
+		fstests.Run(t, &opt)
+	}
 }

backend/hasher/object.go

@@ -71,7 +71,14 @@ func (o *Object) Hash(ctx context.Context, hashType hash.Type) (hashVal string,
 	f := o.f
 	if f.passHashes.Contains(hashType) {
 		fs.Debugf(o, "pass %s", hashType)
-		return o.Object.Hash(ctx, hashType)
+		hashVal, err = o.Object.Hash(ctx, hashType)
+		if hashVal != "" {
+			return hashVal, err
+		}
+		if err != nil {
+			fs.Debugf(o, "error passing %s: %v", hashType, err)
+		}
+		fs.Debugf(o, "passed %s is blank -- trying other methods", hashType)
 	}
 	if !f.suppHashes.Contains(hashType) {
 		fs.Debugf(o, "unsupp %s", hashType)

backend/hdfs/fs.go

@@ -1,5 +1,4 @@
 //go:build !plan9
-// +build !plan9
 
 package hdfs
 
@@ -21,6 +20,7 @@ import (
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/pacer"
 )
 
 // Fs represents a HDFS server
@@ -31,8 +31,15 @@ type Fs struct {
 	opt      Options        // options for this backend
 	ci       *fs.ConfigInfo // global config
 	client   *hdfs.Client
+	pacer    *fs.Pacer // pacer for API calls
 }
 
+const (
+	minSleep      = 20 * time.Millisecond
+	maxSleep      = 10 * time.Second
+	decayConstant = 2 // bigger for slower decay, exponential
+)
+
 // copy-paste from https://github.com/colinmarc/hdfs/blob/master/cmd/hdfs/kerberos.go
 func getKerberosClient() (*krb.Client, error) {
 	configPath := os.Getenv("KRB5_CONFIG")
@@ -85,7 +92,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 	}
 
 	options := hdfs.ClientOptions{
-		Addresses:           []string{opt.Namenode},
+		Addresses:           opt.Namenode,
 		UseDatanodeHostname: false,
 	}
 
@@ -114,6 +121,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		opt:    *opt,
 		ci:     fs.GetConfig(ctx),
 		client: client,
+		pacer:  fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 
 	f.features = (&fs.Features{
@@ -141,7 +149,7 @@ func (f *Fs) Root() string {
 
 // String returns a description of the FS
 func (f *Fs) String() string {
-	return fmt.Sprintf("hdfs://%s", f.opt.Namenode)
+	return fmt.Sprintf("hdfs://%s/%s", f.opt.Namenode, f.root)
 }
 
 // Features returns the optional features of this Fs
@@ -201,7 +209,8 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				fs:      f,
 				remote:  remote,
 				size:    x.Size(),
-				modTime: x.ModTime()})
+				modTime: x.ModTime(),
+			})
 		}
 	}
 	return entries, nil

backend/hdfs/hdfs.go

@@ -1,5 +1,4 @@
 //go:build !plan9
-// +build !plan9
 
 // Package hdfs provides an interface to the HDFS storage system.
 package hdfs
@@ -20,9 +19,10 @@ func init() {
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name:      "namenode",
-			Help:      "Hadoop name node and port.\n\nE.g. \"namenode:8020\" to connect to host namenode at port 8020.",
+			Help:      "Hadoop name nodes and ports.\n\nE.g. \"namenode-1:8020,namenode-2:8020,...\" to connect to host namenodes at port 8020.",
 			Required:  true,
 			Sensitive: true,
+			Default:   fs.CommaSepList{},
 		}, {
 			Name: "username",
 			Help: "Hadoop user name.",
@@ -65,7 +65,7 @@ and 'privacy'. Used only with KERBEROS enabled.`,
 
 // Options for this backend
 type Options struct {
-	Namenode               string               `config:"namenode"`
+	Namenode               fs.CommaSepList      `config:"namenode"`
 	Username               string               `config:"username"`
 	ServicePrincipalName   string               `config:"service_principal_name"`
 	DataTransferProtection string               `config:"data_transfer_protection"`

backend/hdfs/hdfs_test.go

@@ -1,7 +1,6 @@
 // Test HDFS filesystem interface
 
 //go:build !plan9
-// +build !plan9
 
 package hdfs_test
 

backend/hdfs/hdfs_unsupported.go

@@ -2,6 +2,6 @@
 // about "no buildable Go source files "
 
 //go:build plan9
-// +build plan9
 
+// Package hdfs provides an interface to the HDFS storage system.
 package hdfs

backend/hdfs/object.go

@@ -1,14 +1,15 @@
 //go:build !plan9
-// +build !plan9
 
 package hdfs
 
 import (
 	"context"
+	"errors"
 	"io"
 	"path"
 	"time"
 
+	"github.com/colinmarc/hdfs/v2"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/readers"
@@ -106,7 +107,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 
 // Update object
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	realpath := o.fs.realpath(src.Remote())
+	realpath := o.fs.realpath(o.remote)
 	dirname := path.Dir(realpath)
 	fs.Debugf(o.fs, "update [%s]", realpath)
 
@@ -141,7 +142,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	err = out.Close()
+	// If the datanodes have acknowledged all writes but not yet
+	// to the namenode, FileWriter.Close can return ErrReplicating
+	// (wrapped in an os.PathError). This indicates that all data
+	// has been written, but the lease is still open for the file.
+	//
+	// It is safe in this case to either ignore the error (and let
+	// the lease expire on its own) or to call Close multiple
+	// times until it completes without an error. The Java client,
+	// for context, always chooses to retry, with exponential
+	// backoff.
+	err = o.fs.pacer.Call(func() (bool, error) {
+		err := out.Close()
+		if err == nil {
+			return false, nil
+		}
+		return errors.Is(err, hdfs.ErrReplicating), err
+	})
 	if err != nil {
 		cleanup()
 		return err

backend/hidrive/hidrive.go

@@ -762,6 +762,12 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return nil
 }
 
+// Shutdown shutdown the fs
+func (f *Fs) Shutdown(ctx context.Context) error {
+	f.tokenRenewer.Shutdown()
+	return nil
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs.
@@ -997,6 +1003,7 @@ var (
 	_ fs.Copier         = (*Fs)(nil)
 	_ fs.Mover          = (*Fs)(nil)
 	_ fs.DirMover       = (*Fs)(nil)
+	_ fs.Shutdowner     = (*Fs)(nil)
 	_ fs.Object         = (*Object)(nil)
 	_ fs.IDer           = (*Object)(nil)
 )

backend/http/http.go

@@ -36,6 +36,7 @@ func init() {
 		Name:        "http",
 		Description: "HTTP",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "url",
 			Help:     "URL of HTTP host to connect to.\n\nE.g. \"https://example.com\", or \"https://user:pass@example.com\" to use a username and password.",
@@ -88,6 +89,10 @@ that directory listings are much quicker, but rclone won't have the times or
 sizes of any files, and some files that don't exist may be in the listing.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:    "no_escape",
+			Help:    "Do not escape URL metacharacters in path names.",
+			Default: false,
 		}},
 	}
 	fs.Register(fsi)
@@ -99,6 +104,7 @@ type Options struct {
 	NoSlash  bool            `config:"no_slash"`
 	NoHead   bool            `config:"no_head"`
 	Headers  fs.CommaSepList `config:"headers"`
+	NoEscape bool            `config:"no_escape"`
 }
 
 // Fs stores the interface to the remote HTTP files
@@ -210,6 +216,42 @@ func getFsEndpoint(ctx context.Context, client *http.Client, url string, opt *Op
 	return createFileResult()
 }
 
+// Make the http connection with opt
+func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err error) {
+	if len(opt.Headers)%2 != 0 {
+		return false, errors.New("odd number of headers supplied")
+	}
+
+	if !strings.HasSuffix(opt.Endpoint, "/") {
+		opt.Endpoint += "/"
+	}
+
+	// Parse the endpoint and stick the root onto it
+	base, err := url.Parse(opt.Endpoint)
+	if err != nil {
+		return false, err
+	}
+	u, err := rest.URLJoin(base, rest.URLPathEscape(f.root))
+	if err != nil {
+		return false, err
+	}
+
+	client := fshttp.NewClient(ctx)
+
+	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
+	fs.Debugf(nil, "Root: %s", endpoint)
+	u, err = url.Parse(endpoint)
+	if err != nil {
+		return false, err
+	}
+
+	// Update f with the new parameters
+	f.httpClient = client
+	f.endpoint = u
+	f.endpointURL = u.String()
+	return isFile, nil
+}
+
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -220,47 +262,23 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		return nil, err
 	}
 
-	if len(opt.Headers)%2 != 0 {
-		return nil, errors.New("odd number of headers supplied")
-	}
-
-	if !strings.HasSuffix(opt.Endpoint, "/") {
-		opt.Endpoint += "/"
-	}
-
-	// Parse the endpoint and stick the root onto it
-	base, err := url.Parse(opt.Endpoint)
-	if err != nil {
-		return nil, err
-	}
-	u, err := rest.URLJoin(base, rest.URLPathEscape(root))
-	if err != nil {
-		return nil, err
-	}
-
-	client := fshttp.NewClient(ctx)
-
-	endpoint, isFile := getFsEndpoint(ctx, client, u.String(), opt)
-	fs.Debugf(nil, "Root: %s", endpoint)
-	u, err = url.Parse(endpoint)
-	if err != nil {
-		return nil, err
-	}
-
 	ci := fs.GetConfig(ctx)
 	f := &Fs{
-		name:        name,
-		root:        root,
-		opt:         *opt,
-		ci:          ci,
-		httpClient:  client,
-		endpoint:    u,
-		endpointURL: u.String(),
+		name: name,
+		root: root,
+		opt:  *opt,
+		ci:   ci,
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(ctx, f)
 
+	// Make the http connection
+	isFile, err := f.httpConnection(ctx, opt)
+	if err != nil {
+		return nil, err
+	}
+
 	if isFile {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
@@ -313,6 +331,11 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Join's the remote onto the base URL
 func (f *Fs) url(remote string) string {
+	if f.opt.NoEscape {
+		// Directly concatenate without escaping, no_escape behavior
+		return f.endpointURL + remote
+	}
+	// Default behavior
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
@@ -685,10 +708,66 @@ func (o *Object) MimeType(ctx context.Context) string {
 	return o.contentType
 }
 
+var commandHelp = []fs.CommandHelp{{
+	Name:  "set",
+	Short: "Set command for updating the config parameters.",
+	Long: `This set command can be used to update the config parameters
+for a running http backend.
+
+Usage Examples:
+
+    rclone backend set remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
+    rclone rc backend/command command=set fs=remote: [-o opt_name=opt_value] [-o opt_name2=opt_value2]
+    rclone rc backend/command command=set fs=remote: -o url=https://example.com
+
+The option keys are named as they are in the config file.
+
+This rebuilds the connection to the http backend when it is called with
+the new parameters. Only new parameters need be passed as the values
+will default to those currently in use.
+
+It doesn't return anything.
+`,
+}}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "set":
+		newOpt := f.opt
+		err := configstruct.Set(configmap.Simple(opt), &newOpt)
+		if err != nil {
+			return nil, fmt.Errorf("reading config: %w", err)
+		}
+		_, err = f.httpConnection(ctx, &newOpt)
+		if err != nil {
+			return nil, fmt.Errorf("updating session: %w", err)
+		}
+		f.opt = newOpt
+		keys := []string{}
+		for k := range opt {
+			keys = append(keys, k)
+		}
+		fs.Logf(f, "Updated config values: %s", strings.Join(keys, ", "))
+		return nil, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
+	_ fs.Commander   = &Fs{}
 )

backend/imagekit/client/client.go

@@ -0,0 +1,66 @@
+// Package client provides a client for interacting with the ImageKit API.
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fshttp"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// ImageKit main struct
+type ImageKit struct {
+	Prefix        string
+	UploadPrefix  string
+	Timeout       int64
+	UploadTimeout int64
+	PrivateKey    string
+	PublicKey     string
+	URLEndpoint   string
+	HTTPClient    *rest.Client
+}
+
+// NewParams is a struct to define parameters to imagekit
+type NewParams struct {
+	PrivateKey  string
+	PublicKey   string
+	URLEndpoint string
+}
+
+// New returns ImageKit object from environment variables
+func New(ctx context.Context, params NewParams) (*ImageKit, error) {
+
+	privateKey := params.PrivateKey
+	publicKey := params.PublicKey
+	endpointURL := params.URLEndpoint
+
+	switch {
+	case privateKey == "":
+		return nil, fmt.Errorf("ImageKit.io URL endpoint is required")
+	case publicKey == "":
+		return nil, fmt.Errorf("ImageKit.io public key is required")
+	case endpointURL == "":
+		return nil, fmt.Errorf("ImageKit.io private key is required")
+	}
+
+	cliCtx, cliCfg := fs.AddConfig(ctx)
+
+	cliCfg.UserAgent = "rclone/imagekit"
+	client := rest.NewClient(fshttp.NewClient(cliCtx))
+
+	client.SetUserPass(privateKey, "")
+	client.SetHeader("Accept", "application/json")
+
+	return &ImageKit{
+		Prefix:        "https://api.imagekit.io/v2",
+		UploadPrefix:  "https://upload.imagekit.io/api/v2",
+		Timeout:       60,
+		UploadTimeout: 3600,
+		PrivateKey:    params.PrivateKey,
+		PublicKey:     params.PublicKey,
+		URLEndpoint:   params.URLEndpoint,
+		HTTPClient:    client,
+	}, nil
+}

backend/imagekit/client/media.go

@@ -0,0 +1,252 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/rclone/rclone/lib/rest"
+	"gopkg.in/validator.v2"
+)
+
+// FilesOrFolderParam struct is a parameter type to ListFiles() function to search / list media library files.
+type FilesOrFolderParam struct {
+	Path        string `json:"path,omitempty"`
+	Limit       int    `json:"limit,omitempty"`
+	Skip        int    `json:"skip,omitempty"`
+	SearchQuery string `json:"searchQuery,omitempty"`
+}
+
+// AITag represents an AI tag for a media library file.
+type AITag struct {
+	Name       string  `json:"name"`
+	Confidence float32 `json:"confidence"`
+	Source     string  `json:"source"`
+}
+
+// File represents media library File details.
+type File struct {
+	FileID            string            `json:"fileId"`
+	Name              string            `json:"name"`
+	FilePath          string            `json:"filePath"`
+	Type              string            `json:"type"`
+	VersionInfo       map[string]string `json:"versionInfo"`
+	IsPrivateFile     *bool             `json:"isPrivateFile"`
+	CustomCoordinates *string           `json:"customCoordinates"`
+	URL               string            `json:"url"`
+	Thumbnail         string            `json:"thumbnail"`
+	FileType          string            `json:"fileType"`
+	Mime              string            `json:"mime"`
+	Height            int               `json:"height"`
+	Width             int               `json:"Width"`
+	Size              uint64            `json:"size"`
+	HasAlpha          bool              `json:"hasAlpha"`
+	CustomMetadata    map[string]any    `json:"customMetadata,omitempty"`
+	EmbeddedMetadata  map[string]any    `json:"embeddedMetadata"`
+	CreatedAt         time.Time         `json:"createdAt"`
+	UpdatedAt         time.Time         `json:"updatedAt"`
+	Tags              []string          `json:"tags"`
+	AITags            []AITag           `json:"AITags"`
+}
+
+// Folder represents media library Folder details.
+type Folder struct {
+	*File
+	FolderPath string `json:"folderPath"`
+}
+
+// CreateFolderParam represents parameter to create folder api
+type CreateFolderParam struct {
+	FolderName       string `validate:"nonzero" json:"folderName"`
+	ParentFolderPath string `validate:"nonzero" json:"parentFolderPath"`
+}
+
+// DeleteFolderParam represents parameter to delete folder api
+type DeleteFolderParam struct {
+	FolderPath string `validate:"nonzero" json:"folderPath"`
+}
+
+// MoveFolderParam represents parameter to move folder api
+type MoveFolderParam struct {
+	SourceFolderPath string `validate:"nonzero" json:"sourceFolderPath"`
+	DestinationPath  string `validate:"nonzero" json:"destinationPath"`
+}
+
+// JobIDResponse respresents response struct with JobID for folder operations
+type JobIDResponse struct {
+	JobID string `json:"jobId"`
+}
+
+// JobStatus represents response Data to job status api
+type JobStatus struct {
+	JobID  string `json:"jobId"`
+	Type   string `json:"type"`
+	Status string `json:"status"`
+}
+
+// File represents media library File details.
+func (ik *ImageKit) File(ctx context.Context, fileID string) (*http.Response, *File, error) {
+	data := &File{}
+	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:       "GET",
+		Path:         fmt.Sprintf("/files/%s/details", fileID),
+		RootURL:      ik.Prefix,
+		IgnoreStatus: true,
+	}, nil, data)
+
+	return response, data, err
+}
+
+// Files retrieves media library files. Filter options can be supplied as FilesOrFolderParam.
+func (ik *ImageKit) Files(ctx context.Context, params FilesOrFolderParam, includeVersion bool) (*http.Response, *[]File, error) {
+	var SearchQuery = `type = "file"`
+
+	if includeVersion {
+		SearchQuery = `type IN ["file", "file-version"]`
+	}
+	if params.SearchQuery != "" {
+		SearchQuery = params.SearchQuery
+	}
+
+	parameters := url.Values{}
+
+	parameters.Set("skip", fmt.Sprintf("%d", params.Skip))
+	parameters.Set("limit", fmt.Sprintf("%d", params.Limit))
+	parameters.Set("path", params.Path)
+	parameters.Set("searchQuery", SearchQuery)
+
+	data := &[]File{}
+
+	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:     "GET",
+		Path:       "/files",
+		RootURL:    ik.Prefix,
+		Parameters: parameters,
+	}, nil, data)
+
+	return response, data, err
+}
+
+// DeleteFile removes file by FileID from media library
+func (ik *ImageKit) DeleteFile(ctx context.Context, fileID string) (*http.Response, error) {
+	var err error
+
+	if fileID == "" {
+		return nil, errors.New("fileID can not be empty")
+	}
+
+	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:     "DELETE",
+		Path:       fmt.Sprintf("/files/%s", fileID),
+		RootURL:    ik.Prefix,
+		NoResponse: true,
+	}, nil, nil)
+
+	return response, err
+}
+
+// Folders retrieves media library files. Filter options can be supplied as FilesOrFolderParam.
+func (ik *ImageKit) Folders(ctx context.Context, params FilesOrFolderParam) (*http.Response, *[]Folder, error) {
+	var SearchQuery = `type = "folder"`
+
+	if params.SearchQuery != "" {
+		SearchQuery = params.SearchQuery
+	}
+
+	parameters := url.Values{}
+
+	parameters.Set("skip", fmt.Sprintf("%d", params.Skip))
+	parameters.Set("limit", fmt.Sprintf("%d", params.Limit))
+	parameters.Set("path", params.Path)
+	parameters.Set("searchQuery", SearchQuery)
+
+	data := &[]Folder{}
+
+	resp, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:     "GET",
+		Path:       "/files",
+		RootURL:    ik.Prefix,
+		Parameters: parameters,
+	}, nil, data)
+
+	if err != nil {
+		return resp, data, err
+	}
+
+	return resp, data, err
+}
+
+// CreateFolder creates a new folder in media library
+func (ik *ImageKit) CreateFolder(ctx context.Context, param CreateFolderParam) (*http.Response, error) {
+	var err error
+
+	if err = validator.Validate(&param); err != nil {
+		return nil, err
+	}
+
+	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:     "POST",
+		Path:       "/folder",
+		RootURL:    ik.Prefix,
+		NoResponse: true,
+	}, param, nil)
+
+	return response, err
+}
+
+// DeleteFolder removes the folder from media library
+func (ik *ImageKit) DeleteFolder(ctx context.Context, param DeleteFolderParam) (*http.Response, error) {
+	var err error
+
+	if err = validator.Validate(&param); err != nil {
+		return nil, err
+	}
+
+	response, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:     "DELETE",
+		Path:       "/folder",
+		RootURL:    ik.Prefix,
+		NoResponse: true,
+	}, param, nil)
+
+	return response, err
+}
+
+// MoveFolder moves given folder to new path in media library
+func (ik *ImageKit) MoveFolder(ctx context.Context, param MoveFolderParam) (*http.Response, *JobIDResponse, error) {
+	var err error
+	var response = &JobIDResponse{}
+
+	if err = validator.Validate(&param); err != nil {
+		return nil, nil, err
+	}
+
+	resp, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:  "PUT",
+		Path:    "bulkJobs/moveFolder",
+		RootURL: ik.Prefix,
+	}, param, response)
+
+	return resp, response, err
+}
+
+// BulkJobStatus retrieves the status of a bulk job by job ID.
+func (ik *ImageKit) BulkJobStatus(ctx context.Context, jobID string) (*http.Response, *JobStatus, error) {
+	var err error
+	var response = &JobStatus{}
+
+	if jobID == "" {
+		return nil, nil, errors.New("jobId can not be blank")
+	}
+
+	resp, err := ik.HTTPClient.CallJSON(ctx, &rest.Opts{
+		Method:  "GET",
+		Path:    "bulkJobs/" + jobID,
+		RootURL: ik.Prefix,
+	}, nil, response)
+
+	return resp, response, err
+}

backend/imagekit/client/upload.go

@@ -0,0 +1,96 @@
+package client
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// UploadParam defines upload parameters
+type UploadParam struct {
+	FileName      string `json:"fileName"`
+	Folder        string `json:"folder,omitempty"` // default value:  /
+	Tags          string `json:"tags,omitempty"`
+	IsPrivateFile *bool  `json:"isPrivateFile,omitempty"` // default: false
+}
+
+// UploadResult defines the response structure for the upload API
+type UploadResult struct {
+	FileID       string            `json:"fileId"`
+	Name         string            `json:"name"`
+	URL          string            `json:"url"`
+	ThumbnailURL string            `json:"thumbnailUrl"`
+	Height       int               `json:"height"`
+	Width        int               `json:"Width"`
+	Size         uint64            `json:"size"`
+	FilePath     string            `json:"filePath"`
+	AITags       []map[string]any  `json:"AITags"`
+	VersionInfo  map[string]string `json:"versionInfo"`
+}
+
+// Upload uploads an asset to a imagekit account.
+//
+// The asset can be:
+//   - the actual data (io.Reader)
+//   - the Data URI (Base64 encoded), max ~60 MB (62,910,000 chars)
+//   - the remote FTP, HTTP or HTTPS URL address of an existing file
+//
+// https://docs.imagekit.io/api-reference/upload-file-api/server-side-file-upload
+func (ik *ImageKit) Upload(ctx context.Context, file io.Reader, param UploadParam) (*http.Response, *UploadResult, error) {
+	var err error
+
+	if param.FileName == "" {
+		return nil, nil, errors.New("Upload: Filename is required")
+	}
+
+	// Initialize URL values
+	formParams := url.Values{}
+
+	formParams.Add("useUniqueFileName", fmt.Sprint(false))
+
+	// Add individual fields to URL values
+	if param.FileName != "" {
+		formParams.Add("fileName", param.FileName)
+	}
+
+	if param.Tags != "" {
+		formParams.Add("tags", param.Tags)
+	}
+
+	if param.Folder != "" {
+		formParams.Add("folder", param.Folder)
+	}
+
+	if param.IsPrivateFile != nil {
+		formParams.Add("isPrivateFile", fmt.Sprintf("%v", *param.IsPrivateFile))
+	}
+
+	response := &UploadResult{}
+
+	formReader, contentType, _, err := rest.MultipartUpload(ctx, file, formParams, "file", param.FileName)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to make multipart upload: %w", err)
+	}
+
+	opts := rest.Opts{
+		Method:      "POST",
+		Path:        "/files/upload",
+		RootURL:     ik.UploadPrefix,
+		Body:        formReader,
+		ContentType: contentType,
+	}
+
+	resp, err := ik.HTTPClient.CallJSON(ctx, &opts, nil, response)
+
+	if err != nil {
+		return resp, response, err
+	}
+
+	return resp, response, err
+}

backend/imagekit/client/url.go

@@ -0,0 +1,72 @@
+package client
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/hex"
+	"fmt"
+	neturl "net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// URLParam represents parameters for generating url
+type URLParam struct {
+	Path            string
+	Src             string
+	URLEndpoint     string
+	Signed          bool
+	ExpireSeconds   int64
+	QueryParameters map[string]string
+}
+
+// URL generates url from URLParam
+func (ik *ImageKit) URL(params URLParam) (string, error) {
+	var resultURL string
+	var url *neturl.URL
+	var err error
+	var endpoint = params.URLEndpoint
+
+	if endpoint == "" {
+		endpoint = ik.URLEndpoint
+	}
+
+	endpoint = strings.TrimRight(endpoint, "/") + "/"
+
+	if params.QueryParameters == nil {
+		params.QueryParameters = make(map[string]string)
+	}
+
+	if url, err = neturl.Parse(params.Src); err != nil {
+		return "", err
+	}
+
+	query := url.Query()
+
+	for k, v := range params.QueryParameters {
+		query.Set(k, v)
+	}
+	url.RawQuery = query.Encode()
+	resultURL = url.String()
+
+	if params.Signed {
+		now := time.Now().Unix()
+
+		var expires = strconv.FormatInt(now+params.ExpireSeconds, 10)
+		var path = strings.Replace(resultURL, endpoint, "", 1)
+
+		path += expires
+		mac := hmac.New(sha1.New, []byte(ik.PrivateKey))
+		mac.Write([]byte(path))
+		signature := hex.EncodeToString(mac.Sum(nil))
+
+		if strings.Contains(resultURL, "?") {
+			resultURL = resultURL + "&" + fmt.Sprintf("ik-t=%s&ik-s=%s", expires, signature)
+		} else {
+			resultURL = resultURL + "?" + fmt.Sprintf("ik-t=%s&ik-s=%s", expires, signature)
+		}
+	}
+
+	return resultURL, nil
+}

backend/imagekit/imagekit.go

@@ -0,0 +1,828 @@
+// Package imagekit provides an interface to the ImageKit.io media library.
+package imagekit
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/rclone/rclone/backend/imagekit/client"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/version"
+)
+
+const (
+	minSleep      = 1 * time.Millisecond
+	maxSleep      = 100 * time.Millisecond
+	decayConstant = 2
+)
+
+var systemMetadataInfo = map[string]fs.MetadataHelp{
+	"btime": {
+		Help:     "Time of file birth (creation) read from Last-Modified header",
+		Type:     "RFC 3339",
+		Example:  "2006-01-02T15:04:05.999999999Z07:00",
+		ReadOnly: true,
+	},
+	"size": {
+		Help:     "Size of the object in bytes",
+		Type:     "int64",
+		ReadOnly: true,
+	},
+	"file-type": {
+		Help:     "Type of the file",
+		Type:     "string",
+		Example:  "image",
+		ReadOnly: true,
+	},
+	"height": {
+		Help:     "Height of the image or video in pixels",
+		Type:     "int",
+		ReadOnly: true,
+	},
+	"width": {
+		Help:     "Width of the image or video in pixels",
+		Type:     "int",
+		ReadOnly: true,
+	},
+	"has-alpha": {
+		Help:     "Whether the image has alpha channel or not",
+		Type:     "bool",
+		ReadOnly: true,
+	},
+	"tags": {
+		Help:     "Tags associated with the file",
+		Type:     "string",
+		Example:  "tag1,tag2",
+		ReadOnly: true,
+	},
+	"google-tags": {
+		Help:     "AI generated tags by Google Cloud Vision associated with the image",
+		Type:     "string",
+		Example:  "tag1,tag2",
+		ReadOnly: true,
+	},
+	"aws-tags": {
+		Help:     "AI generated tags by AWS Rekognition associated with the image",
+		Type:     "string",
+		Example:  "tag1,tag2",
+		ReadOnly: true,
+	},
+	"is-private-file": {
+		Help:     "Whether the file is private or not",
+		Type:     "bool",
+		ReadOnly: true,
+	},
+	"custom-coordinates": {
+		Help:     "Custom coordinates of the file",
+		Type:     "string",
+		Example:  "0,0,100,100",
+		ReadOnly: true,
+	},
+}
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "imagekit",
+		Description: "ImageKit.io",
+		NewFs:       NewFs,
+		MetadataInfo: &fs.MetadataInfo{
+			System: systemMetadataInfo,
+			Help:   `Any metadata supported by the underlying remote is read and written.`,
+		},
+		Options: []fs.Option{
+			{
+				Name:     "endpoint",
+				Help:     "You can find your ImageKit.io URL endpoint in your [dashboard](https://imagekit.io/dashboard/developer/api-keys)",
+				Required: true,
+			},
+			{
+				Name:      "public_key",
+				Help:      "You can find your ImageKit.io public key in your [dashboard](https://imagekit.io/dashboard/developer/api-keys)",
+				Required:  true,
+				Sensitive: true,
+			},
+			{
+				Name:      "private_key",
+				Help:      "You can find your ImageKit.io private key in your [dashboard](https://imagekit.io/dashboard/developer/api-keys)",
+				Required:  true,
+				Sensitive: true,
+			},
+			{
+				Name:     "only_signed",
+				Help:     "If you have configured `Restrict unsigned image URLs` in your dashboard settings, set this to true.",
+				Default:  false,
+				Advanced: true,
+			},
+			{
+				Name:     "versions",
+				Help:     "Include old versions in directory listings.",
+				Default:  false,
+				Advanced: true,
+			},
+			{
+				Name:     "upload_tags",
+				Help:     "Tags to add to the uploaded files, e.g. \"tag1,tag2\".",
+				Default:  "",
+				Advanced: true,
+			},
+			{
+				Name:     config.ConfigEncoding,
+				Help:     config.ConfigEncodingHelp,
+				Advanced: true,
+				Default: (encoder.EncodeZero |
+					encoder.EncodeSlash |
+					encoder.EncodeQuestion |
+					encoder.EncodeHashPercent |
+					encoder.EncodeCtl |
+					encoder.EncodeDel |
+					encoder.EncodeDot |
+					encoder.EncodeDoubleQuote |
+					encoder.EncodePercent |
+					encoder.EncodeBackSlash |
+					encoder.EncodeDollar |
+					encoder.EncodeLtGt |
+					encoder.EncodeSquareBracket |
+					encoder.EncodeInvalidUtf8),
+			},
+		},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	Endpoint   string               `config:"endpoint"`
+	PublicKey  string               `config:"public_key"`
+	PrivateKey string               `config:"private_key"`
+	OnlySigned bool                 `config:"only_signed"`
+	Versions   bool                 `config:"versions"`
+	Enc        encoder.MultiEncoder `config:"encoding"`
+}
+
+// Fs represents a remote to ImageKit
+type Fs struct {
+	name     string           // name of remote
+	root     string           // root path
+	opt      Options          // parsed options
+	features *fs.Features     // optional features
+	ik       *client.ImageKit // ImageKit client
+	pacer    *fs.Pacer        // pacer for API calls
+}
+
+// Object describes a ImageKit file
+type Object struct {
+	fs          *Fs         // The Fs this object is part of
+	remote      string      // The remote path
+	filePath    string      // The path to the file
+	contentType string      // The content type of the object if known - may be ""
+	timestamp   time.Time   // The timestamp of the object if known - may be zero
+	file        client.File // The media file if known - may be nil
+	versionID   string      // If present this points to an object version
+}
+
+// NewFs constructs an Fs from the path, container:path
+func NewFs(ctx context.Context, name string, root string, m configmap.Mapper) (fs.Fs, error) {
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+
+	if err != nil {
+		return nil, err
+	}
+
+	ik, err := client.New(ctx, client.NewParams{
+		URLEndpoint: opt.Endpoint,
+		PublicKey:   opt.PublicKey,
+		PrivateKey:  opt.PrivateKey,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	f := &Fs{
+		name:  name,
+		opt:   *opt,
+		ik:    ik,
+		pacer: fs.NewPacer(ctx, pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+	}
+
+	f.root = path.Join("/", root)
+
+	f.features = (&fs.Features{
+		CaseInsensitive:         false,
+		DuplicateFiles:          false,
+		ReadMimeType:            true,
+		WriteMimeType:           false,
+		CanHaveEmptyDirectories: true,
+		BucketBased:             false,
+		ServerSideAcrossConfigs: false,
+		IsLocal:                 false,
+		SlowHash:                true,
+		ReadMetadata:            true,
+		WriteMetadata:           false,
+		UserMetadata:            false,
+		FilterAware:             true,
+		PartialUploads:          false,
+		NoMultiThreading:        false,
+	}).Fill(ctx, f)
+
+	if f.root != "/" {
+
+		r := f.root
+
+		folderPath := f.EncodePath(r[:strings.LastIndex(r, "/")+1])
+		fileName := f.EncodeFileName(r[strings.LastIndex(r, "/")+1:])
+
+		file := f.getFileByName(ctx, folderPath, fileName)
+
+		if file != nil {
+			newRoot := path.Dir(f.root)
+			f.root = newRoot
+			return f, fs.ErrorIsFile
+		}
+
+	}
+	return f, nil
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return strings.TrimLeft(f.root, "/")
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("FS imagekit: %s", f.root)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	return fs.ModTimeNotSupported
+}
+
+// Hashes returns the supported hash types of the filesystem.
+func (f *Fs) Hashes() hash.Set {
+	return hash.NewHashSet()
+}
+
+// Features returns the optional features of this Fs.
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+
+	remote := path.Join(f.root, dir)
+
+	remote = f.EncodePath(remote)
+
+	if remote != "/" {
+		parentFolderPath, folderName := path.Split(remote)
+		folderExists, err := f.getFolderByName(ctx, parentFolderPath, folderName)
+
+		if err != nil {
+			return make(fs.DirEntries, 0), err
+		}
+
+		if folderExists == nil {
+			return make(fs.DirEntries, 0), fs.ErrorDirNotFound
+		}
+	}
+
+	folders, folderError := f.getFolders(ctx, remote)
+
+	if folderError != nil {
+		return make(fs.DirEntries, 0), folderError
+	}
+
+	files, fileError := f.getFiles(ctx, remote, f.opt.Versions)
+
+	if fileError != nil {
+		return make(fs.DirEntries, 0), fileError
+	}
+
+	res := make([]fs.DirEntry, 0, len(folders)+len(files))
+
+	for _, folder := range folders {
+		folderPath := f.DecodePath(strings.TrimLeft(strings.Replace(folder.FolderPath, f.EncodePath(f.root), "", 1), "/"))
+		res = append(res, fs.NewDir(folderPath, folder.UpdatedAt))
+	}
+
+	for _, file := range files {
+		res = append(res, f.newObject(ctx, remote, file))
+	}
+
+	return res, nil
+}
+
+func (f *Fs) newObject(ctx context.Context, remote string, file client.File) *Object {
+	remoteFile := strings.TrimLeft(strings.Replace(file.FilePath, f.EncodePath(f.root), "", 1), "/")
+
+	folderPath, fileName := path.Split(remoteFile)
+
+	folderPath = f.DecodePath(folderPath)
+	fileName = f.DecodeFileName(fileName)
+
+	remoteFile = path.Join(folderPath, fileName)
+
+	if file.Type == "file-version" {
+		remoteFile = version.Add(remoteFile, file.UpdatedAt)
+
+		return &Object{
+			fs:          f,
+			remote:      remoteFile,
+			filePath:    file.FilePath,
+			contentType: file.Mime,
+			timestamp:   file.UpdatedAt,
+			file:        file,
+			versionID:   file.VersionInfo["id"],
+		}
+	}
+
+	return &Object{
+		fs:          f,
+		remote:      remoteFile,
+		filePath:    file.FilePath,
+		contentType: file.Mime,
+		timestamp:   file.UpdatedAt,
+		file:        file,
+	}
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error ErrorObjectNotFound.
+//
+// If remote points to a directory then it should return
+// ErrorIsDir if possible without doing any extra work,
+// otherwise ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	r := path.Join(f.root, remote)
+
+	folderPath, fileName := path.Split(r)
+
+	folderPath = f.EncodePath(folderPath)
+	fileName = f.EncodeFileName(fileName)
+
+	isFolder, err := f.getFolderByName(ctx, folderPath, fileName)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if isFolder != nil {
+		return nil, fs.ErrorIsDir
+	}
+
+	file := f.getFileByName(ctx, folderPath, fileName)
+
+	if file == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+
+	return f.newObject(ctx, r, *file), nil
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Put should either
+// return an error or upload it properly (rather than e.g. calling panic).
+//
+// May create the object even if it returns an error - if so
+// will return the object and the error, otherwise will return
+// nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return uploadFile(ctx, f, in, src.Remote(), options...)
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
+	remote := path.Join(f.root, dir)
+	parentFolderPath, folderName := path.Split(remote)
+
+	parentFolderPath = f.EncodePath(parentFolderPath)
+	folderName = f.EncodeFileName(folderName)
+
+	err = f.pacer.Call(func() (bool, error) {
+		var res *http.Response
+		res, err = f.ik.CreateFolder(ctx, client.CreateFolderParam{
+			ParentFolderPath: parentFolderPath,
+			FolderName:       folderName,
+		})
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	return err
+}
+
+// Rmdir removes the directory (container, bucket) if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
+
+	entries, err := f.List(ctx, dir)
+
+	if err != nil {
+		return err
+	}
+
+	if len(entries) > 0 {
+		return errors.New("directory is not empty")
+	}
+
+	err = f.pacer.Call(func() (bool, error) {
+		var res *http.Response
+		res, err = f.ik.DeleteFolder(ctx, client.DeleteFolderParam{
+			FolderPath: f.EncodePath(path.Join(f.root, dir)),
+		})
+
+		if res.StatusCode == http.StatusNotFound {
+			return false, fs.ErrorDirNotFound
+		}
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	return err
+}
+
+// Purge deletes all the files and the container
+//
+// Optional interface: Only implement this if you have a way of
+// deleting all the files quicker than just running Remove() on the
+// result of List()
+func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
+
+	remote := path.Join(f.root, dir)
+
+	err = f.pacer.Call(func() (bool, error) {
+		var res *http.Response
+		res, err = f.ik.DeleteFolder(ctx, client.DeleteFolderParam{
+			FolderPath: f.EncodePath(remote),
+		})
+
+		if res.StatusCode == http.StatusNotFound {
+			return false, fs.ErrorDirNotFound
+		}
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	return err
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+
+	duration := time.Duration(math.Abs(float64(expire)))
+
+	expireSeconds := duration.Seconds()
+
+	fileRemote := path.Join(f.root, remote)
+
+	folderPath, fileName := path.Split(fileRemote)
+	folderPath = f.EncodePath(folderPath)
+	fileName = f.EncodeFileName(fileName)
+
+	file := f.getFileByName(ctx, folderPath, fileName)
+
+	if file == nil {
+		return "", fs.ErrorObjectNotFound
+	}
+
+	// Pacer not needed as this doesn't use the API
+	url, err := f.ik.URL(client.URLParam{
+		Src:           file.URL,
+		Signed:        *file.IsPrivateFile || f.opt.OnlySigned,
+		ExpireSeconds: int64(expireSeconds),
+		QueryParameters: map[string]string{
+			"updatedAt": file.UpdatedAt.String(),
+		},
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	return url, nil
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
+	return "", hash.ErrUnsupported
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+	return true
+}
+
+// String returns a description of the Object
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.file.Name
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns the modification date of the file
+// It should return a best guess if one isn't available
+func (o *Object) ModTime(context.Context) time.Time {
+	return o.file.UpdatedAt
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return int64(o.file.Size)
+}
+
+// MimeType returns the MIME type of the file
+func (o *Object) MimeType(context.Context) string {
+	return o.contentType
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	// Offset and Count for range download
+	var offset int64
+	var count int64
+
+	fs.FixRangeOption(options, -1)
+	partialContent := false
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			offset, count = x.Decode(-1)
+			partialContent = true
+		case *fs.SeekOption:
+			offset = x.Offset
+			partialContent = true
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+
+	// Pacer not needed as this doesn't use the API
+	url, err := o.fs.ik.URL(client.URLParam{
+		Src:    o.file.URL,
+		Signed: *o.file.IsPrivateFile || o.fs.opt.OnlySigned,
+		QueryParameters: map[string]string{
+			"tr":        "orig-true",
+			"updatedAt": o.file.UpdatedAt.String(),
+		},
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{}
+	req, _ := http.NewRequest("GET", url, nil)
+	req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+count-1))
+	resp, err := client.Do(req)
+
+	if err != nil {
+		return nil, err
+	}
+
+	end := resp.ContentLength
+
+	if partialContent && resp.StatusCode == http.StatusOK {
+		skip := offset
+
+		if offset < 0 {
+			skip = end + offset + 1
+		}
+
+		_, err = io.CopyN(io.Discard, resp.Body, skip)
+		if err != nil {
+			if resp != nil {
+				_ = resp.Body.Close()
+			}
+			return nil, err
+		}
+
+		return readers.NewLimitedReadCloser(resp.Body, end-skip), nil
+	}
+
+	return resp.Body, nil
+}
+
+// Update in to the object with the modTime given of the given size
+//
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
+// return an error or update the object properly (rather than e.g. calling panic).
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+
+	srcRemote := o.Remote()
+
+	remote := path.Join(o.fs.root, srcRemote)
+	folderPath, fileName := path.Split(remote)
+
+	UseUniqueFileName := new(bool)
+	*UseUniqueFileName = false
+
+	var resp *client.UploadResult
+
+	err = o.fs.pacer.Call(func() (bool, error) {
+		var res *http.Response
+		res, resp, err = o.fs.ik.Upload(ctx, in, client.UploadParam{
+			FileName:      fileName,
+			Folder:        folderPath,
+			IsPrivateFile: o.file.IsPrivateFile,
+		})
+
+		return o.fs.shouldRetry(ctx, res, err)
+	})
+
+	if err != nil {
+		return err
+	}
+
+	fileID := resp.FileID
+
+	_, file, err := o.fs.ik.File(ctx, fileID)
+
+	if err != nil {
+		return err
+	}
+
+	o.file = *file
+
+	return nil
+}
+
+// Remove this object
+func (o *Object) Remove(ctx context.Context) (err error) {
+	err = o.fs.pacer.Call(func() (bool, error) {
+		var res *http.Response
+		res, err = o.fs.ik.DeleteFile(ctx, o.file.FileID)
+
+		return o.fs.shouldRetry(ctx, res, err)
+	})
+
+	return err
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+func uploadFile(ctx context.Context, f *Fs, in io.Reader, srcRemote string, options ...fs.OpenOption) (fs.Object, error) {
+	remote := path.Join(f.root, srcRemote)
+	folderPath, fileName := path.Split(remote)
+
+	folderPath = f.EncodePath(folderPath)
+	fileName = f.EncodeFileName(fileName)
+
+	UseUniqueFileName := new(bool)
+	*UseUniqueFileName = false
+
+	err := f.pacer.Call(func() (bool, error) {
+		var res *http.Response
+		var err error
+		res, _, err = f.ik.Upload(ctx, in, client.UploadParam{
+			FileName:      fileName,
+			Folder:        folderPath,
+			IsPrivateFile: &f.opt.OnlySigned,
+		})
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return f.NewObject(ctx, srcRemote)
+}
+
+// Metadata returns the metadata for the object
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+
+	metadata.Set("btime", o.file.CreatedAt.Format(time.RFC3339))
+	metadata.Set("size", strconv.FormatUint(o.file.Size, 10))
+	metadata.Set("file-type", o.file.FileType)
+	metadata.Set("height", strconv.Itoa(o.file.Height))
+	metadata.Set("width", strconv.Itoa(o.file.Width))
+	metadata.Set("has-alpha", strconv.FormatBool(o.file.HasAlpha))
+
+	for k, v := range o.file.EmbeddedMetadata {
+		metadata.Set(k, fmt.Sprint(v))
+	}
+
+	if o.file.Tags != nil {
+		metadata.Set("tags", strings.Join(o.file.Tags, ","))
+	}
+
+	if o.file.CustomCoordinates != nil {
+		metadata.Set("custom-coordinates", *o.file.CustomCoordinates)
+	}
+
+	if o.file.IsPrivateFile != nil {
+		metadata.Set("is-private-file", strconv.FormatBool(*o.file.IsPrivateFile))
+	}
+
+	if o.file.AITags != nil {
+		googleTags := []string{}
+		awsTags := []string{}
+
+		for _, tag := range o.file.AITags {
+			if tag.Source == "google-auto-tagging" {
+				googleTags = append(googleTags, tag.Name)
+			} else if tag.Source == "aws-auto-tagging" {
+				awsTags = append(awsTags, tag.Name)
+			}
+		}
+
+		if len(googleTags) > 0 {
+			metadata.Set("google-tags", strings.Join(googleTags, ","))
+		}
+
+		if len(awsTags) > 0 {
+			metadata.Set("aws-tags", strings.Join(awsTags, ","))
+		}
+	}
+
+	return metadata, nil
+}
+
+// Copy src to this remote using server-side move operations.
+//
+// This is stored with the remote path given.
+//
+// It returns the destination Object and a possible error.
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantMove
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	srcObj, ok := src.(*Object)
+	if !ok {
+		return nil, fs.ErrorCantMove
+	}
+
+	file, err := srcObj.Open(ctx)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return uploadFile(ctx, f, file, remote)
+}
+
+// Check the interfaces are satisfied.
+var (
+	_ fs.Fs           = &Fs{}
+	_ fs.Purger       = &Fs{}
+	_ fs.PublicLinker = &Fs{}
+	_ fs.Object       = &Object{}
+	_ fs.Copier       = &Fs{}
+)

backend/imagekit/imagekit_test.go

@@ -0,0 +1,18 @@
+package imagekit
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+func TestIntegration(t *testing.T) {
+	debug := true
+	fstest.Verbose = &debug
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:      "TestImageKit:",
+		NilObject:       (*Object)(nil),
+		SkipFsCheckWrap: true,
+	})
+}

backend/imagekit/util.go

@@ -0,0 +1,193 @@
+package imagekit
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/rclone/rclone/backend/imagekit/client"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/lib/pacer"
+)
+
+func (f *Fs) getFiles(ctx context.Context, path string, includeVersions bool) (files []client.File, err error) {
+
+	files = make([]client.File, 0)
+
+	var hasMore = true
+
+	for hasMore {
+		err = f.pacer.Call(func() (bool, error) {
+			var data *[]client.File
+			var res *http.Response
+			res, data, err = f.ik.Files(ctx, client.FilesOrFolderParam{
+				Skip:  len(files),
+				Limit: 100,
+				Path:  path,
+			}, includeVersions)
+
+			hasMore = !(len(*data) == 0 || len(*data) < 100)
+
+			if len(*data) > 0 {
+				files = append(files, *data...)
+			}
+
+			return f.shouldRetry(ctx, res, err)
+		})
+	}
+
+	if err != nil {
+		return make([]client.File, 0), err
+	}
+
+	return files, nil
+}
+
+func (f *Fs) getFolders(ctx context.Context, path string) (folders []client.Folder, err error) {
+
+	folders = make([]client.Folder, 0)
+
+	var hasMore = true
+
+	for hasMore {
+		err = f.pacer.Call(func() (bool, error) {
+			var data *[]client.Folder
+			var res *http.Response
+			res, data, err = f.ik.Folders(ctx, client.FilesOrFolderParam{
+				Skip:  len(folders),
+				Limit: 100,
+				Path:  path,
+			})
+
+			hasMore = !(len(*data) == 0 || len(*data) < 100)
+
+			if len(*data) > 0 {
+				folders = append(folders, *data...)
+			}
+
+			return f.shouldRetry(ctx, res, err)
+		})
+	}
+
+	if err != nil {
+		return make([]client.Folder, 0), err
+	}
+
+	return folders, nil
+}
+
+func (f *Fs) getFileByName(ctx context.Context, path string, name string) (file *client.File) {
+
+	err := f.pacer.Call(func() (bool, error) {
+		res, data, err := f.ik.Files(ctx, client.FilesOrFolderParam{
+			Limit:       1,
+			Path:        path,
+			SearchQuery: fmt.Sprintf(`type = "file" AND name = %s`, strconv.Quote(name)),
+		}, false)
+
+		if len(*data) == 0 {
+			file = nil
+		} else {
+			file = &(*data)[0]
+		}
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	if err != nil {
+		return nil
+	}
+
+	return file
+}
+
+func (f *Fs) getFolderByName(ctx context.Context, path string, name string) (folder *client.Folder, err error) {
+	err = f.pacer.Call(func() (bool, error) {
+		res, data, err := f.ik.Folders(ctx, client.FilesOrFolderParam{
+			Limit:       1,
+			Path:        path,
+			SearchQuery: fmt.Sprintf(`type = "folder" AND name = %s`, strconv.Quote(name)),
+		})
+
+		if len(*data) == 0 {
+			folder = nil
+		} else {
+			folder = &(*data)[0]
+		}
+
+		return f.shouldRetry(ctx, res, err)
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return folder, nil
+}
+
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	401, // Unauthorized (e.g. "Token has expired")
+	408, // Request Timeout
+	429, // Rate exceeded.
+	500, // Get occasional 500 Internal Server Error
+	503, // Service Unavailable
+	504, // Gateway Time-out
+}
+
+func shouldRetryHTTP(resp *http.Response, retryErrorCodes []int) bool {
+	if resp == nil {
+		return false
+	}
+	for _, e := range retryErrorCodes {
+		if resp.StatusCode == e {
+			return true
+		}
+	}
+	return false
+}
+
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if fserrors.ContextError(ctx, &err) {
+		return false, err
+	}
+
+	if resp != nil && (resp.StatusCode == 429 || resp.StatusCode == 503) {
+		var retryAfter = 1
+		retryAfterString := resp.Header.Get("X-RateLimit-Reset")
+		if retryAfterString != "" {
+			var err error
+			retryAfter, err = strconv.Atoi(retryAfterString)
+			if err != nil {
+				fs.Errorf(f, "Malformed %s header %q: %v", "X-RateLimit-Reset", retryAfterString, err)
+			}
+		}
+
+		return true, pacer.RetryAfterError(err, time.Duration(retryAfter)*time.Millisecond)
+	}
+
+	return fserrors.ShouldRetry(err) || shouldRetryHTTP(resp, retryErrorCodes), err
+}
+
+// EncodePath encapsulates the logic for encoding a path
+func (f *Fs) EncodePath(str string) string {
+	return f.opt.Enc.FromStandardPath(str)
+}
+
+// DecodePath encapsulates the logic for decoding a path
+func (f *Fs) DecodePath(str string) string {
+	return f.opt.Enc.ToStandardPath(str)
+}
+
+// EncodeFileName encapsulates the logic for encoding a file name
+func (f *Fs) EncodeFileName(str string) string {
+	return f.opt.Enc.FromStandardName(str)
+}
+
+// DecodeFileName encapsulates the logic for decoding a file name
+func (f *Fs) DecodeFileName(str string) string {
+	return f.opt.Enc.ToStandardName(str)
+}

backend/internetarchive/internetarchive.go

@@ -802,7 +802,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		headers["x-archive-size-hint"] = fmt.Sprintf("%d", size)
 	}
 	var mdata fs.Metadata
-	mdata, err = fs.GetMetadataOptions(ctx, src, options)
+	mdata, err = fs.GetMetadataOptions(ctx, o.fs, src, options)
 	if err == nil && mdata != nil {
 		for mk, mv := range mdata {
 			mk = strings.ToLower(mk)

backend/jottacloud/api/types.go

@@ -395,7 +395,7 @@ type JottaFile struct {
 	State           string    `xml:"currentRevision>state"`
 	CreatedAt       JottaTime `xml:"currentRevision>created"`
 	ModifiedAt      JottaTime `xml:"currentRevision>modified"`
-	Updated         JottaTime `xml:"currentRevision>updated"`
+	UpdatedAt       JottaTime `xml:"currentRevision>updated"`
 	Size            int64     `xml:"currentRevision>size"`
 	MimeType        string    `xml:"currentRevision>mime"`
 	MD5             string    `xml:"currentRevision>md5"`

backend/jottacloud/jottacloud.go

@@ -67,9 +67,13 @@ const (
 	legacyEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
 	legacyConfigVersion         = 0
 
-	teliaCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
-	teliaCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
-	teliaCloudClientID = "desktop"
+	teliaseCloudTokenURL = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/token"
+	teliaseCloudAuthURL  = "https://cloud-auth.telia.se/auth/realms/telia_se/protocol/openid-connect/auth"
+	teliaseCloudClientID = "desktop"
+
+	telianoCloudTokenURL = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/token"
+	telianoCloudAuthURL  = "https://sky-auth.telia.no/auth/realms/get/protocol/openid-connect/auth"
+	telianoCloudClientID = "desktop"
 
 	tele2CloudTokenURL = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/token"
 	tele2CloudAuthURL  = "https://mittcloud-auth.tele2.se/auth/realms/comhem/protocol/openid-connect/auth"
@@ -88,6 +92,33 @@ func init() {
 		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config:      Config,
+		MetadataInfo: &fs.MetadataInfo{
+			Help: `Jottacloud has limited support for metadata, currently an extended set of timestamps.`,
+			System: map[string]fs.MetadataHelp{
+				"btime": {
+					Help:    "Time of file birth (creation), read from rclone metadata",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z07:00",
+				},
+				"mtime": {
+					Help:    "Time of last modification, read from rclone metadata",
+					Type:    "RFC 3339",
+					Example: "2006-01-02T15:04:05.999999999Z07:00",
+				},
+				"utime": {
+					Help:     "Time of last upload, when current revision was created, generated by backend",
+					Type:     "RFC 3339",
+					Example:  "2006-01-02T15:04:05.999999999Z07:00",
+					ReadOnly: true,
+				},
+				"content-type": {
+					Help:     "MIME type, also known as media type",
+					Type:     "string",
+					Example:  "text/plain",
+					ReadOnly: true,
+				},
+			},
+		},
 		Options: append(oauthutil.SharedOptions, []fs.Option{{
 			Name:     "md5_memory_limit",
 			Help:     "Files bigger than this will be cached on disk to calculate the MD5 if required.",
@@ -138,8 +169,11 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf
 			Value: "legacy",
 			Help:  "Legacy authentication.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.",
 		}, {
-			Value: "telia",
-			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud.",
+			Value: "telia_se",
+			Help:  "Telia Cloud authentication.\nUse this if you are using Telia Cloud (Sweden).",
+		}, {
+			Value: "telia_no",
+			Help:  "Telia Sky authentication.\nUse this if you are using Telia Sky (Norway).",
 		}, {
 			Value: "tele2",
 			Help:  "Tele2 Cloud authentication.\nUse this if you are using Tele2 Cloud.",
@@ -238,17 +272,32 @@ machines.`)
 			return nil, fmt.Errorf("error while saving token: %w", err)
 		}
 		return fs.ConfigGoto("choose_device")
-	case "telia": // telia cloud config
+	case "telia_se": // telia_se cloud config
 		m.Set("configVersion", fmt.Sprint(configVersion))
-		m.Set(configClientID, teliaCloudClientID)
-		m.Set(configTokenURL, teliaCloudTokenURL)
+		m.Set(configClientID, teliaseCloudClientID)
+		m.Set(configTokenURL, teliaseCloudTokenURL)
 		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
 			OAuth2Config: &oauth2.Config{
 				Endpoint: oauth2.Endpoint{
-					AuthURL:  teliaCloudAuthURL,
-					TokenURL: teliaCloudTokenURL,
+					AuthURL:  teliaseCloudAuthURL,
+					TokenURL: teliaseCloudTokenURL,
 				},
-				ClientID:    teliaCloudClientID,
+				ClientID:    teliaseCloudClientID,
+				Scopes:      []string{"openid", "jotta-default", "offline_access"},
+				RedirectURL: oauthutil.RedirectLocalhostURL,
+			},
+		})
+	case "telia_no": // telia_no cloud config
+		m.Set("configVersion", fmt.Sprint(configVersion))
+		m.Set(configClientID, telianoCloudClientID)
+		m.Set(configTokenURL, telianoCloudTokenURL)
+		return oauthutil.ConfigOut("choose_device", &oauthutil.Options{
+			OAuth2Config: &oauth2.Config{
+				Endpoint: oauth2.Endpoint{
+					AuthURL:  telianoCloudAuthURL,
+					TokenURL: telianoCloudTokenURL,
+				},
+				ClientID:    telianoCloudClientID,
 				Scopes:      []string{"openid", "jotta-default", "offline_access"},
 				RedirectURL: oauthutil.RedirectLocalhostURL,
 			},
@@ -475,7 +524,9 @@ type Object struct {
 	remote      string
 	hasMetaData bool
 	size        int64
+	createTime  time.Time
 	modTime     time.Time
+	updateTime  time.Time
 	md5         string
 	mimeType    string
 }
@@ -950,6 +1001,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
 		CanHaveEmptyDirectories: true,
 		ReadMimeType:            true,
 		WriteMimeType:           false,
+		ReadMetadata:            true,
+		WriteMetadata:           true,
+		UserMetadata:            false,
 	}).Fill(ctx, f)
 	f.jfsSrv.SetErrorHandler(errorHandler)
 	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
@@ -1136,6 +1190,7 @@ func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback
 			remote:      filesystem.opt.Enc.ToStandardPath(path.Join(f.Path, f.Name)),
 			size:        f.Size,
 			md5:         f.Checksum,
+			createTime:  time.Time(f.Created),
 			modTime:     time.Time(f.Modified),
 		})
 	}
@@ -1365,7 +1420,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 // is currently in trash, but can be made to match, it will be
 // restored. Returns ErrorObjectNotFound if upload will be necessary
 // to get a matching remote file.
-func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
+func (f *Fs) createOrUpdate(ctx context.Context, file string, createTime time.Time, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:       "POST",
 		Path:         f.filePath(file),
@@ -1375,11 +1430,10 @@ func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time,
 
 	opts.Parameters.Set("cphash", "true")
 
-	fileDate := api.JottaTime(modTime).String()
 	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
 	opts.ExtraHeaders["JMd5"] = md5
-	opts.ExtraHeaders["JCreated"] = fileDate
-	opts.ExtraHeaders["JModified"] = fileDate
+	opts.ExtraHeaders["JCreated"] = api.JottaTime(createTime).String()
+	opts.ExtraHeaders["JModified"] = api.JottaTime(modTime).String()
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -1433,16 +1487,38 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantMove
 	}
 
-	err := f.mkParentDir(ctx, remote)
+	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
 	if err != nil {
 		return nil, err
 	}
+
+	if err := f.mkParentDir(ctx, remote); err != nil {
+		return nil, err
+	}
 	info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
 
-	// if destination was a trashed file then after a successful copy the copied file is still in trash (bug in api?)
-	if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
-		fs.Debugf(src, "Server-side copied to trashed destination, restoring")
-		info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
+	if err == nil {
+		var createTime time.Time
+		var createTimeMeta bool
+		var modTime time.Time
+		var modTimeMeta bool
+		if meta != nil {
+			createTime, createTimeMeta = srcObj.parseFsMetadataTime(meta, "btime")
+			if !createTimeMeta {
+				createTime = srcObj.createTime
+			}
+			modTime, modTimeMeta = srcObj.parseFsMetadataTime(meta, "mtime")
+			if !modTimeMeta {
+				modTime = srcObj.modTime
+			}
+		}
+		if bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
+			// Workaround necessary when destination was a trashed file, to avoid the copied file also being in trash (bug in api?)
+			fs.Debugf(src, "Server-side copied to trashed destination, restoring")
+			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
+		} else if createTimeMeta || modTimeMeta {
+			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
+		}
 	}
 
 	if err != nil {
@@ -1469,12 +1545,30 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantMove
 	}
 
-	err := f.mkParentDir(ctx, remote)
+	meta, err := fs.GetMetadataOptions(ctx, f, src, fs.MetadataAsOpenOptions(ctx))
 	if err != nil {
 		return nil, err
 	}
+
+	if err := f.mkParentDir(ctx, remote); err != nil {
+		return nil, err
+	}
 	info, err := f.copyOrMove(ctx, "mv", srcObj.filePath(), remote)
 
+	if err == nil && meta != nil {
+		createTime, createTimeMeta := srcObj.parseFsMetadataTime(meta, "btime")
+		if !createTimeMeta {
+			createTime = srcObj.createTime
+		}
+		modTime, modTimeMeta := srcObj.parseFsMetadataTime(meta, "mtime")
+		if !modTimeMeta {
+			modTime = srcObj.modTime
+		}
+		if createTimeMeta || modTimeMeta {
+			info, err = f.createOrUpdate(ctx, remote, createTime, modTime, info.Size, info.MD5)
+		}
+	}
+
 	if err != nil {
 		return nil, fmt.Errorf("couldn't move file: %w", err)
 	}
@@ -1626,6 +1720,12 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return nil
 }
 
+// Shutdown shutdown the fs
+func (f *Fs) Shutdown(ctx context.Context) error {
+	f.tokenRenewer.Shutdown()
+	return nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
@@ -1705,7 +1805,9 @@ func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.size = info.Size
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
+	o.createTime = time.Time(info.CreatedAt)
 	o.modTime = time.Time(info.ModifiedAt)
+	o.updateTime = time.Time(info.UpdatedAt)
 	return nil
 }
 
@@ -1724,6 +1826,20 @@ func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
 	return o.setMetaData(info)
 }
 
+// parseFsMetadataTime parses a time string from fs.Metadata with key
+func (o *Object) parseFsMetadataTime(m fs.Metadata, key string) (t time.Time, ok bool) {
+	value, ok := m[key]
+	if ok {
+		var err error
+		t, err = time.Parse(time.RFC3339Nano, value) // metadata stores RFC3339Nano timestamps
+		if err != nil {
+			fs.Debugf(o, "failed to parse metadata %s: %q: %v", key, value, err)
+			ok = false
+		}
+	}
+	return t, ok
+}
+
 // ModTime returns the modification time of the object
 //
 // It attempts to read the objects mtime and if that isn't present the
@@ -1750,7 +1866,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	// (note that if size/md5 does not match, the file content will
 	// also be modified if deduplication is possible, i.e. it is
 	// important to use correct/latest values)
-	_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
+	_, err = o.fs.createOrUpdate(ctx, o.remote, o.createTime, modTime, o.size, o.md5)
 	if err != nil {
 		if err == fs.ErrorObjectNotFound {
 			// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
@@ -1887,6 +2003,27 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// Wrap the accounting back onto the stream
 		in = wrap(in)
 	}
+	// Fetch metadata if --metadata is in use
+	meta, err := fs.GetMetadataOptions(ctx, o.fs, src, options)
+	if err != nil {
+		return fmt.Errorf("failed to read metadata from source object: %w", err)
+	}
+	var createdTime string
+	var modTime string
+	if meta != nil {
+		if t, ok := o.parseFsMetadataTime(meta, "btime"); ok {
+			createdTime = api.Rfc3339Time(t).String() // jottacloud api wants RFC3339 timestamps
+		}
+		if t, ok := o.parseFsMetadataTime(meta, "mtime"); ok {
+			modTime = api.Rfc3339Time(t).String()
+		}
+	}
+	if modTime == "" { // prefer mtime in meta as Modified time, fallback to source ModTime
+		modTime = api.Rfc3339Time(src.ModTime(ctx)).String()
+	}
+	if createdTime == "" { // if no Created time set same as Modified
+		createdTime = modTime
+	}
 
 	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
@@ -1896,13 +2033,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Options:      options,
 		ExtraHeaders: make(map[string]string),
 	}
-	fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
 
 	// the allocate request
 	var request = api.AllocateFileRequest{
 		Bytes:    size,
-		Created:  fileDate,
-		Modified: fileDate,
+		Created:  createdTime,
+		Modified: modTime,
 		Md5:      md5String,
 		Path:     o.fs.allocatePathRaw(o.remote, true),
 	}
@@ -1917,7 +2053,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRUPT, try to upload a then
+	// If the file state is INCOMPLETE and CORRUPT, we must upload it.
+	// Else, if the file state is COMPLETE, we don't need to upload it because
+	// the content is already there, possibly it was created with deduplication,
+	// and also any metadata changes are already performed by the allocate request.
 	if response.State != "COMPLETED" {
 		// how much do we still have to upload?
 		remainingBytes := size - response.ResumePos
@@ -1941,22 +2080,18 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		}
 
 		// send the remaining bytes
-		resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
+		_, err = o.fs.apiSrv.CallJSON(ctx, &opts, nil, &result)
 		if err != nil {
 			return err
 		}
 
-		// finally update the meta data
-		o.hasMetaData = true
-		o.size = result.Bytes
-		o.md5 = result.Md5
-		o.modTime = time.Unix(result.Modified/1000, 0)
-	} else {
-		// If the file state is COMPLETE we don't need to upload it because the file was already found but we still need to update our metadata
-		return o.readMetaData(ctx, true)
+		// Upload response contains main metadata properties (size, md5 and modTime)
+		// which could be set back to the object, but it does not contain the
+		// necessary information to set the createTime and updateTime properties,
+		// so must therefore perform a read instead.
 	}
-
-	return nil
+	// in any case we must update the object meta data
+	return o.readMetaData(ctx, true)
 }
 
 func (o *Object) remove(ctx context.Context, hard bool) error {
@@ -1991,6 +2126,22 @@ func (o *Object) Remove(ctx context.Context) error {
 	return o.remove(ctx, o.fs.opt.HardDelete)
 }
 
+// Metadata returns metadata for an object
+//
+// It should return nil if there is no Metadata
+func (o *Object) Metadata(ctx context.Context) (metadata fs.Metadata, err error) {
+	err = o.readMetaData(ctx, false)
+	if err != nil {
+		fs.Logf(o, "Failed to read metadata: %v", err)
+		return nil, err
+	}
+	metadata.Set("btime", o.createTime.Format(time.RFC3339Nano)) // metadata timestamps should be RFC3339Nano
+	metadata.Set("mtime", o.modTime.Format(time.RFC3339Nano))
+	metadata.Set("utime", o.updateTime.Format(time.RFC3339Nano))
+	metadata.Set("content-type", o.mimeType)
+	return metadata, nil
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs           = (*Fs)(nil)
@@ -2003,6 +2154,8 @@ var (
 	_ fs.Abouter      = (*Fs)(nil)
 	_ fs.UserInfoer   = (*Fs)(nil)
 	_ fs.CleanUpper   = (*Fs)(nil)
+	_ fs.Shutdowner   = (*Fs)(nil)
 	_ fs.Object       = (*Object)(nil)
 	_ fs.MimeTyper    = (*Object)(nil)
+	_ fs.Metadataer   = (*Object)(nil)
 )

backend/jottacloud/jottacloud_internal_test.go

@@ -1,11 +1,17 @@
 package jottacloud
 
 import (
+	"context"
 	"crypto/md5"
 	"fmt"
 	"io"
 	"testing"
+	"time"
 
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,3 +46,48 @@ func TestReadMD5(t *testing.T) {
 		})
 	}
 }
+
+func (f *Fs) InternalTestMetadata(t *testing.T) {
+	ctx := context.Background()
+	contents := random.String(1000)
+
+	item := fstest.NewItem("test-metadata", contents, fstest.Time("2001-05-06T04:05:06.499999999Z"))
+	utime := time.Now()
+	metadata := fs.Metadata{
+		"btime": "2009-05-06T04:05:06.499999999Z",
+		"mtime": "2010-06-07T08:09:07.599999999Z",
+		//"utime" - read-only
+		//"content-type" - read-only
+	}
+	obj := fstests.PutTestContentsMetadata(ctx, t, f, &item, false, contents, true, "text/html", metadata)
+	defer func() {
+		assert.NoError(t, obj.Remove(ctx))
+	}()
+	o := obj.(*Object)
+	gotMetadata, err := o.Metadata(ctx)
+	require.NoError(t, err)
+	for k, v := range metadata {
+		got := gotMetadata[k]
+		switch k {
+		case "btime":
+			assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
+		case "mtime":
+			assert.True(t, fstest.Time(v).Truncate(f.Precision()).Equal(fstest.Time(got)), fmt.Sprintf("btime not equal want %v got %v", v, got))
+		case "utime":
+			gotUtime := fstest.Time(got)
+			dt := gotUtime.Sub(utime)
+			assert.True(t, dt < time.Minute && dt > -time.Minute, fmt.Sprintf("utime more than 1 minute out want %v got %v delta %v", utime, gotUtime, dt))
+			assert.True(t, fstest.Time(v).Equal(fstest.Time(got)))
+		case "content-type":
+			assert.True(t, o.MimeType(ctx) == got)
+		default:
+			assert.Equal(t, v, got, k)
+		}
+	}
+}
+
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("Metadata", f.InternalTestMetadata)
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)