fs: Debug the names of env vars we read values from as part of the config

See: https://forum.rclone.org/t/how-can-i-check-if-environment-variables-are-being-used/17490

.appveyor.yml

@@ -1,49 +0,0 @@
-version: "{build}"
-
-os: Windows Server 2012 R2
-
-clone_folder: c:\gopath\src\github.com\rclone\rclone
-
-cache:
-  - '%LocalAppData%\go-build'
-
-environment:
-  GOPATH: C:\gopath
-  CPATH: C:\Program Files (x86)\WinFsp\inc\fuse
-  ORIGPATH: '%PATH%'
-  NOCCPATH: C:\MinGW\bin;%GOPATH%\bin;%PATH%
-  PATHCC64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%NOCCPATH%
-  PATHCC32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%NOCCPATH%
-  PATH: '%PATHCC64%'
-  RCLONE_CONFIG_PASS:
-    secure: sq9CPBbwaeKJv+yd24U44neORYPQVy6jsjnQptC+5yk=
-
-install:
-- choco install winfsp -y
-- choco install zip -y
-- copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe
-
-build_script:
-- echo %PATH%
-- echo %GOPATH%
-- go version
-- go env
-- go install
-- go build
-- make log_since_last_release > %TEMP%\git-log.txt
-- make version > %TEMP%\version
-- set /p RCLONE_VERSION=<%TEMP%\version
-- set PATH=%PATHCC32%
-- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/386" -cgo -tags cmount %RCLONE_VERSION%
-- set PATH=%PATHCC64%
-- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/amd64" -cgo -no-clean -tags cmount %RCLONE_VERSION%
-
-test_script:
-- make GOTAGS=cmount quicktest
-
-artifacts:
-- path: rclone.exe
-- path: build/*-v*.zip
-
-deploy_script:
-- IF "%APPVEYOR_REPO_NAME%" == "rclone/rclone" IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload

.circleci/config.yml

@@ -1,50 +0,0 @@
----
-version: 2
-
-jobs:
-
-  build:
-    machine: true
-
-    working_directory: ~/.go_workspace/src/github.com/rclone/rclone
-
-    steps:
-    - checkout
-
-    - run:
-        name: Cross-compile rclone
-        command: |
-            docker pull rclone/xgo-cgofuse
-            go get -v github.com/karalabe/xgo
-            xgo \
-                --image=rclone/xgo-cgofuse \
-                --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
-                -tags cmount \
-                .
-            xgo \
-                --targets=android/*,ios/* \
-                .
-
-    - run:
-        name: Prepare artifacts
-        command: |
-            mkdir -p /tmp/rclone.dist
-            cp -R rclone-* /tmp/rclone.dist
-            mkdir build
-            cp -R rclone-* build/
-
-    - run:
-        name: Build rclone
-        command: |
-            go version
-            go build
-
-    - run:
-        name: Upload artifacts
-        command: |
-          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
-              make circleci_upload
-          fi
-
-    - store_artifacts:
-        path: /tmp/rclone.dist

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+github: [ncw]
+patreon: njcw
+liberapay: ncw
+custom: ["https://rclone.org/donate/"]

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Rclone Forum Community Support
+    url: https://forum.rclone.org/
+    about: Please ask and answer questions here.

.github/workflows/build.yml

@@ -0,0 +1,255 @@
+---
+# Github Actions build for rclone
+# -*- compile-command: "yamllint -f parsable build.yml" -*-
+
+name: build
+
+# Trigger the workflow on push or pull request
+on:
+  push:
+    branches:
+      - '*'
+    tags:
+      - '*'
+  pull_request:
+
+jobs:
+  build:
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'modules_race', 'go1.11', 'go1.12', 'go1.13']
+
+        include:
+          - job_name: linux
+            os: ubuntu-latest
+            go: '1.14.x'
+            modules: 'off'
+            gotags: cmount
+            build_flags: '-include "^linux/"'
+            check: true
+            quicktest: true
+            deploy: true
+
+          - job_name: mac
+            os: macOS-latest
+            go: '1.14.x'
+            modules: 'off'
+            gotags: ''  # cmount doesn't work on osx travis for some reason
+            build_flags: '-include "^darwin/amd64" -cgo'
+            quicktest: true
+            racequicktest: true
+            deploy: true
+
+          - job_name: windows_amd64
+            os: windows-latest
+            go: '1.14.x'
+            modules: 'off'
+            gotags: cmount
+            build_flags: '-include "^windows/amd64" -cgo'
+            quicktest: true
+            racequicktest: true
+            deploy: true
+
+          - job_name: windows_386
+            os: windows-latest
+            go: '1.14.x'
+            modules: 'off'
+            gotags: cmount
+            goarch: '386'
+            cgo: '1'
+            build_flags: '-include "^windows/386" -cgo'
+            quicktest: true
+            deploy: true
+
+          - job_name: other_os
+            os: ubuntu-latest
+            go: '1.14.x'
+            modules: 'off'
+            build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
+            compile_all: true
+            deploy: true
+
+          - job_name: modules_race
+            os: ubuntu-latest
+            go: '1.14.x'
+            modules: 'on'
+            quicktest: true
+            racequicktest: true
+
+          - job_name: go1.11
+            os: ubuntu-latest
+            go: '1.11.x'
+            modules: 'off'
+            quicktest: true
+
+          - job_name: go1.12
+            os: ubuntu-latest
+            go: '1.12.x'
+            modules: 'off'
+            quicktest: true
+
+          - job_name: go1.13
+            os: ubuntu-latest
+            go: '1.13.x'
+            modules: 'off'
+            quicktest: true
+
+    name: ${{ matrix.job_name }}
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
+
+      - name: Install Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: ${{ matrix.go }}
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          echo '::set-env name=GOPATH::${{ runner.workspace }}'
+          echo '::add-path::${{ runner.workspace }}/bin'
+          echo '::set-env name=GO111MODULE::${{ matrix.modules }}'
+          echo '::set-env name=GOTAGS::${{ matrix.gotags }}'
+          echo '::set-env name=BUILD_FLAGS::${{ matrix.build_flags }}'
+          if [[ "${{ matrix.goarch }}" != "" ]]; then echo '::set-env name=GOARCH::${{ matrix.goarch }}' ; fi
+          if [[ "${{ matrix.cgo }}" != "" ]]; then echo '::set-env name=CGO_ENABLED::${{ matrix.cgo }}' ; fi
+
+      - name: Install Libraries on Linux
+        shell: bash
+        run: |
+          sudo modprobe fuse
+          sudo chmod 666 /dev/fuse
+          sudo chown root:$USER /etc/fuse.conf
+          sudo apt-get install fuse libfuse-dev rpm pkg-config
+        if: matrix.os == 'ubuntu-latest'
+
+      - name: Install Libraries on macOS
+        shell: bash
+        run: |
+          brew update
+          brew cask install osxfuse
+        if: matrix.os == 'macOS-latest'
+
+      - name: Install Libraries on Windows
+        shell: powershell
+        run: |
+          $ProgressPreference = 'SilentlyContinue'
+          choco install -y winfsp zip
+          Write-Host "::set-env name=CPATH::C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse"
+          if ($env:GOARCH -eq "386") {
+            choco install -y mingw --forcex86 --force
+            Write-Host "::add-path::C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin"
+          }
+          # Copy mingw32-make.exe to make.exe so the same command line
+          # can be used on Windows as on macOS and Linux
+          $path = (get-command mingw32-make.exe).Path
+          Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
+        if: matrix.os == 'windows-latest'
+
+      - name: Print Go version and environment
+        shell: bash
+        run: |
+          printf "Using go at: $(which go)\n"
+          printf "Go version: $(go version)\n"
+          printf "\n\nGo environment:\n\n"
+          go env
+          printf "\n\nRclone environment:\n\n"
+          make vars
+          printf "\n\nSystem environment:\n\n"
+          env
+
+      - name: Run tests
+        shell: bash
+        run: |
+          make
+          make quicktest
+        if: matrix.quicktest
+
+      - name: Race test
+        shell: bash
+        run: |
+          make racequicktest
+        if: matrix.racequicktest
+
+      - name: Code quality test
+        shell: bash
+        run: |
+          make build_dep
+          make check
+        if: matrix.check
+
+      - name: Compile all architectures test
+        shell: bash
+        run: |
+          make
+          make compile_all
+        if: matrix.compile_all
+
+      - name: Deploy built binaries
+        shell: bash
+        run: |
+          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
+          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
+          make ci_beta
+        env:
+          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
+        # working-directory: '$(modulePath)'
+        # Deploy binaries if enabled in config && not a PR && not a fork
+        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+
+  xgo:
+    timeout-minutes: 60
+    name: "xgo cross compile"
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          echo '::set-env name=GOPATH::${{ runner.workspace }}'
+          echo '::add-path::${{ runner.workspace }}/bin'
+
+      - name: Cross-compile rclone
+        run: |
+          docker pull billziss/xgo-cgofuse
+          GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
+          xgo \
+              -image=billziss/xgo-cgofuse \
+              -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+              -tags cmount \
+              -dest build \
+              .
+          xgo \
+              -image=billziss/xgo-cgofuse \
+              -targets=android/*,ios/* \
+              -dest build \
+              .
+
+      - name: Build rclone
+        run: |
+          docker pull golang
+          docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=vendor -v
+
+      - name: Upload artifacts
+        run: |
+          make ci_upload
+        env:
+          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
+        # Upload artifacts if not a PR && not a fork
+        if: github.head_ref == '' && github.repository == 'rclone/rclone'

.github/workflows/build_publish_docker_image.yml

@@ -0,0 +1,25 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                fetch-depth: 0
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              with:
+                tag: beta
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -0,0 +1,33 @@
+name: Docker release build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                fetch-depth: 0
+            - name: Get actual patch version
+              id: actual_patch_version
+              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
+            - name: Get actual minor version
+              id: actual_minor_version
+              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
+            - name: Get actual major version
+              id: actual_major_version
+              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              with:
+                tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.gitignore

@@ -7,4 +7,5 @@ rclone.iml
 .idea
 .history
 *.test
-*.log
+*.log
+*.iml

.pkgr.yml

@@ -1,2 +0,0 @@
-default_dependencies: false
-cli: rclone

.travis.yml

@@ -1,128 +0,0 @@
----
-language: go
-sudo: required
-dist: xenial
-os:
-  - linux
-go_import_path: github.com/rclone/rclone
-before_install:
-  - git fetch --unshallow --tags
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      sudo modprobe fuse
-      sudo chmod 666 /dev/fuse
-      sudo chown root:$USER /etc/fuse.conf
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      brew update
-      brew tap caskroom/cask
-      brew cask install osxfuse
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-      choco install -y winfsp zip make
-      cd ../..                  # fix crlf in git checkout
-      mv $TRAVIS_REPO_SLUG _old
-      git config --global core.autocrlf false
-      git clone _old $TRAVIS_REPO_SLUG
-      cd $TRAVIS_REPO_SLUG
-    fi
-install:
-  - make vars
-env:
-  global:
-    - GOTAGS=cmount
-    - GOMAXPROCS=8              # workaround for cmd/mount tests locking up - see #3154
-    - GO111MODULE=off
-    - GITHUB_USER=ncw
-    - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
-    - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
-addons:
-  apt:
-    packages:
-      - fuse
-      - libfuse-dev
-      - rpm
-      - pkg-config
-cache:
-  directories:
-    - $HOME/.cache/go-build
-matrix:
-  allow_failures:
-    - go: tip
-  include:
-    - go: 1.9.x
-      script:
-        - make quicktest
-    - go: 1.10.x
-      script:
-        - make quicktest
-    - go: 1.11.x
-      script:
-        - make quicktest
-    - go: 1.12.x
-      name: Linux
-      env:
-        - GOTAGS=cmount
-        - BUILD_FLAGS='-include "^linux/"'
-        - DEPLOY=true
-      script:
-        - make build_dep
-        - make check
-        - make quicktest
-    - go: 1.12.x
-      name: Go Modules / Race
-      env:
-        - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      script:
-        - make quicktest
-        - make racequicktest
-    - go: 1.12.x
-      name: Other OS
-      env:
-        - DEPLOY=true
-        - BUILD_FLAGS='-exclude "^(windows|darwin|linux)/"'
-      script:
-        - make
-        - make compile_all
-    - go: 1.12.x
-      name: macOS
-      os: osx
-      env:
-        - GOTAGS=  # cmount doesn't work on osx travis for some reason
-        - BUILD_FLAGS='-include "^darwin/" -cgo'
-        - DEPLOY=true
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-      script:
-        - make
-        - make quicktest
-        - make racequicktest
-    # - os: windows
-    #   name: Windows
-    #   go: 1.12.x
-    #   env:
-    #     - GOTAGS=cmount
-    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
-    #     - BUILD_FLAGS='-include "^windows/amd64" -cgo' # 386 doesn't build yet
-    #   #filter_secrets: false     # works around a problem with secrets under windows
-    #   cache:
-    #     directories:
-    #       - ${LocalAppData}/go-build
-    #   script:
-    #     - make
-    #     - make quicktest
-    #     - make racequicktest
-    - go: tip
-      script:
-        - make quicktest
-
-deploy:
-  provider: script
-  script: make travis_beta
-  skip_cleanup: true
-  on:
-    repo: rclone/rclone
-    all_branches: true
-    condition: $TRAVIS_PULL_REQUEST == false && $DEPLOY == true

CONTRIBUTING.md

@@ -79,16 +79,19 @@ request](https://help.github.com/articles/creating-a-pull-request/).
 
 You patch will get reviewed and you might get asked to fix some stuff.
 
-If so, then make the changes in the same branch, squash the commits,
-rebase it to master then push it to GitHub with `--force`.
+If so, then make the changes in the same branch, squash the commits (make multiple commits one commit) by running:
+```
+git log # See how many commits you want to squash
+git reset --soft HEAD~2 # This squashes the 2 latest commits together.
+git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
+git commit # Add a new commit message.
+git push --force # Push the squashed commit to your GitHub repo.
+# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also reccommends wizardzines.com
+```
 
-## Enabling CI for your fork ##
+## CI for your fork ##
 
-The CI config files for rclone have taken care of forks of the project, so you can enable CI for your fork repo easily.
-
-rclone currently uses [Travis CI](https://travis-ci.org/), [AppVeyor](https://ci.appveyor.com/), and
-[Circle CI](https://circleci.com/) to build the project. To enable them for your fork, simply go into their
-websites, find your fork of rclone, and enable building there.
+rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
 
 ## Testing ##
 
@@ -152,6 +155,7 @@ with modules beneath.
     * ...commands
   * docs - the documentation and website
     * content - adjust these docs only - everything else is autogenerated
+      * command - these are auto generated - edit the corresponding .go file
   * fs - main rclone definitions - minimal amount of code
     * accounting - bandwidth limiting and statistics
     * asyncreader - an io.Reader which reads ahead
@@ -161,7 +165,7 @@ with modules beneath.
     * fserrors - rclone specific error handling
     * fshttp - http handling for rclone
     * fspath - path handling for rclone
-    * hash - defines rclones hash types and functions
+    * hash - defines rclone's hash types and functions
     * list - list a remote
     * log - logging facilities
     * march - iterates directories in lock step
@@ -207,6 +211,9 @@ don't need to run these when adding a feature.
 Documentation for rclone sub commands is with their code, eg
 `cmd/ls/ls.go`.
 
+Note that you can use [GitHub's online editor](https://help.github.com/en/github/managing-files-in-a-repository/editing-files-in-another-users-repository)
+for small changes in the docs which makes it very easy.
+
 ## Making a release ##
 
 There are separate instructions for making a release in the RELEASE.md
@@ -295,7 +302,7 @@ If you need to update a dependency then run
     GO111MODULE=on go get -u github.com/pkg/errors
     GO111MODULE=on go mod vendor
 
-Check in in a single commit as above.
+Check in a single commit as above.
 
 ## Updating all the dependencies ##
 
@@ -341,6 +348,11 @@ Getting going
   * Add your remote to the imports in `backend/all/all.go`
   * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
   * Try to implement as many optional methods as possible as it makes the remote more usable.
+  * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
+    * `rclone purge -v TestRemote:rclone-info`
+    * `rclone info --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
+    * `go run cmd/info/internal/build_csv/main.go -o remote.csv remote.json`
+    * open `remote.csv` in a spreadsheet and examine
 
 Unit tests
 
@@ -353,7 +365,7 @@ Integration tests
   * Add your backend to `fstest/test_all/config.yaml`
       * Once you've done that then you can use the integration test framework from the project root:
       * go install ./...
-      * test_all -backend remote
+      * test_all -backends remote
 
 Or if you want to run the integration tests manually:
 
@@ -367,12 +379,54 @@ Or if you want to run the integration tests manually:
 
 See the [testing](#testing) section for more information on integration tests.
 
-Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in alphabetical order but with the local file system last.
+Add your fs to the docs - you'll need to pick an icon for it from
+[fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in
+alphabetical order of full name of remote (eg `drive` is ordered as
+`Google Drive`) but with the local file system last.
 
   * `README.md` - main GitHub page
   * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
+    * make sure this has the `autogenerated options` comments in (see your reference backend docs)
+    * update them with `make backenddocs` - revert any changes in other backends
   * `docs/content/overview.md` - overview docs
   * `docs/content/docs.md` - list of remotes in config section
-  * `docs/content/about.md` - front page of rclone.org
+  * `docs/content/_index.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
+
+Once you've written the docs, run `make serve` and check they look OK
+in the web browser and the links (internal and external) all work.
+
+## Writing a plugin ##
+
+New features (backends, commands) can also be added "out-of-tree", through Go plugins.
+Changes will be kept in a dynamically loaded file instead of being compiled into the main binary.
+This is useful if you can't merge your changes upstream or don't want to maintain a fork of rclone.
+
+Usage
+
+ - Naming
+   - Plugins names must have the pattern `librcloneplugin_KIND_NAME.so`.
+   - `KIND` should be one of `backend`, `command` or `bundle`.
+   - Example: A plugin with backend support for PiFS would be called
+     `librcloneplugin_backend_pifs.so`.
+ - Loading
+   - Supported on macOS & Linux as of now. ([Go issue for Windows support](https://github.com/golang/go/issues/19282))
+   - Supported on rclone v1.50 or greater.
+   - All plugins in the folder specified by variable `$RCLONE_PLUGIN_PATH` are loaded.
+   - If this variable doesn't exist, plugin support is disabled.
+   - Plugins must be compiled against the exact version of rclone to work.
+     (The rclone used during building the plugin must be the same as the source of rclone)
+     
+Building
+
+To turn your existing additions into a Go plugin, move them to an external repository
+and change the top-level package name to `main`.
+
+Check `rclone --version` and make sure that the plugin's rclone dependency and host Go version match.
+
+Then, run `go build -buildmode=plugin -o PLUGIN_NAME.so .` to build the plugin.
+
+[Go reference](https://godoc.org/github.com/rclone/rclone/lib/plugin)
+
+[Minimal example](https://gist.github.com/terorie/21b517ee347828e899e1913efc1d684f)

Dockerfile

@@ -0,0 +1,22 @@
+FROM golang AS builder
+
+COPY . /go/src/github.com/rclone/rclone/
+WORKDIR /go/src/github.com/rclone/rclone/
+
+RUN \
+  CGO_ENABLED=0 \
+  make
+RUN ./rclone version
+
+# Begin final image
+FROM alpine:latest
+
+RUN apk --no-cache add ca-certificates fuse tzdata && \
+  echo "user_allow_other" >> /etc/fuse.conf
+
+COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
+
+ENTRYPOINT [ "rclone" ]
+
+WORKDIR /data
+ENV XDG_CONFIG_HOME=/config

MAINTAINERS.md

@@ -2,16 +2,20 @@
 
 Current active maintainers of rclone are:
 
-| Name             | GitHub ID   | Specific Responsibilities    |
-| :--------------- | :---------- | :--------------------------  |
-| Nick Craig-Wood  | @ncw        | overall project health       |
-| Stefan Breunig   | @breunigs   |                              |
-| Ishuah Kariuki   | @ishuah     |                              |
-| Remus Bunduc     | @remusb     | cache backend                |
-| Fabian Möller    | @B4dM4n     |                              |
-| Alex Chen        | @Cnly       | onedrive backend             |
-| Sandeep Ummadi   | @sandeepkru | azureblob backend            |
-| Sebastian Bünger | @buengese   | jottacloud & yandex backends |
+| Name             | GitHub ID         | Specific Responsibilities    |
+| :--------------- | :---------------- | :--------------------------  |
+| Nick Craig-Wood  | @ncw              | overall project health       |
+| Stefan Breunig   | @breunigs         |                              |
+| Ishuah Kariuki   | @ishuah           |                              |
+| Remus Bunduc     | @remusb           | cache backend                |
+| Fabian Möller    | @B4dM4n           |                              |
+| Alex Chen        | @Cnly             | onedrive backend             |
+| Sandeep Ummadi   | @sandeepkru       | azureblob backend            |
+| Sebastian Bünger | @buengese         | jottacloud & yandex backends |
+| Ivan Andreev     | @ivandeex         | chunker & mailru backends    |
+| Max Sum          | @Max-Sum          | union backend                |
+| Fred             | @creativeprojects | seafile backend              |
+| Caleb Case       | @calebcase        | tardigrade backend           |
 
 **This is a work in progress Draft**
 
@@ -29,7 +33,7 @@ Rclone uses the labels like this:
 * `duplicate` - normally close these and ask the user to subscribe to the original
 * `enhancement: new remote` - a new rclone backend
 * `enhancement` - a new feature
-* `FUSE` - do do with `rclone mount` command
+* `FUSE` - to do with `rclone mount` command
 * `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project
 * `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project
 * `IMPORTANT` - note to maintainers not to forget to fix this for the release

Makefile

@@ -1,18 +1,29 @@
 SHELL = bash
-BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(shell git rev-parse --abbrev-ref HEAD))
+# Branch we are working on
+BRANCH := $(or $(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
+# Tag of the current commit, if any.  If this is not "" then we are building a release
+RELEASE_TAG := $(shell git tag -l --points-at HEAD)
+# Version of last release (may not be on this branch)
+VERSION := $(shell cat VERSION)
+# Last tag on this branch
 LAST_TAG := $(shell git describe --tags --abbrev=0)
-ifeq ($(BRANCH),$(LAST_TAG))
+# If we are working on a release, override branch to master
+ifdef RELEASE_TAG
 	BRANCH := master
 endif
 TAG_BRANCH := -$(BRANCH)
 BRANCH_PATH := branch/
+# If building HEAD or master then unset TAG_BRANCH and BRANCH_PATH
 ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	TAG_BRANCH :=
 	BRANCH_PATH :=
 endif
-TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
-ifneq ($(TAG),$(LAST_TAG))
+# Make version suffix -DDD-gCCCCCCCC (D=commits since last relase, C=Commit) or blank
+VERSION_SUFFIX := $(shell git describe --abbrev=8 --tags | perl -lpe 's/^v\d+\.\d+\.\d+//; s/^-(\d+)/"-".sprintf("%03d",$$1)/e;')
+# TAG is current version + number of commits since last release + branch
+TAG := $(VERSION)$(VERSION_SUFFIX)$(TAG_BRANCH)
+NEXT_VERSION := $(shell echo $(VERSION) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+ifndef RELEASE_TAG
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
@@ -30,28 +41,35 @@ BUILDTAGS=-tags "$(GOTAGS)"
 LINTTAGS=--build-tags "$(GOTAGS)"
 endif
 
-.PHONY: rclone vars version
+.PHONY: rclone test_all vars version
 
 rclone:
-	touch fs/version.go
-	go install -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
-	cp -av `go env GOPATH`/bin/rclone .
+	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
+	mkdir -p `go env GOPATH`/bin/
+	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
+	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
+
+test_all:
+	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/rclone/rclone/fstest/test_all
 
 vars:
 	@echo SHELL="'$(SHELL)'"
 	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
-	@echo LAST_TAG="'$(LAST_TAG)'"
-	@echo NEW_TAG="'$(NEW_TAG)'"
+	@echo VERSION="'$(VERSION)'"
+	@echo NEXT_VERSION="'$(NEXT_VERSION)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
+btest:
+	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
+	@echo "Copied markdown of beta release to clip board"
+
 version:
 	@echo '$(TAG)'
 
 # Full suite of integration tests
-test:	rclone
-	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/rclone/rclone/fstest/test_all
+test:	rclone test_all
 	-test_all 2>&1 | tee test_all.log
 	@echo "Written logs in test_all.log"
 
@@ -72,10 +90,14 @@ check:	rclone
 build_dep:
 	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
-# Get the release dependencies
-release_dep:
-	go get -u github.com/goreleaser/nfpm/...
-	go get -u github.com/aktau/github-release
+# Get the release dependencies we only install on linux
+release_dep_linux:
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+	go run bin/get-github-release.go -extract github-release aktau/github-release 'linux-amd64-github-release.tar.bz2'
+
+# Get the release dependencies we only install on Windows
+release_dep_windows:
+	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
 
 # Update dependencies
 update:
@@ -83,19 +105,24 @@ update:
 	GO111MODULE=on go mod tidy
 	GO111MODULE=on go mod vendor
 
+# Tidy the module dependencies
+tidy:
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
+
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
-	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
+	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
 MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
+	pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
 
 MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
+	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
@@ -117,11 +144,19 @@ clean:
 	rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log
 
 website:
+	rm -rf docs/public
 	cd docs && hugo
+	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
 	rclone -v sync docs/public memstore:www-rclone-org
 
+upload_test_website:	website
+	rclone -P sync docs/public test-rclone-org:
+
+validate_website: website
+	find docs/public -type f -name "*.html" | xargs tidy --mute-id yes -errors --gnu-emacs yes --drop-empty-elements no --warn-proprietary-attributes no --mute MISMATCHED_ATTRIBUTE_WARN
+
 tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
@@ -156,24 +191,17 @@ log_since_last_release:
 compile_all:
 	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
 
-appveyor_upload:
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
-ifndef BRANCH_PATH
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
-endif
-	@echo Beta release ready at $(BETA_URL)
-
-circleci_upload:
+ci_upload:
+	sudo chown -R $$USER build
+	find build -type l -delete
+	gzip -r9v build
 	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
 ifndef BRANCH_PATH
 	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
-travis_beta:
-ifeq (linux,$(filter linux,$(subst Linux,linux,$(TRAVIS_OS_NAME) $(AGENT_OS))))
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
-endif
+ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -182,32 +210,33 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-# Fetch the binary builds from travis and appveyor
+# Fetch the binary builds from GitHub actions
 fetch_binaries:
 	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server -v -w
+	cd docs && hugo server -v -w --disableFastRender
 
 tag:	doc
-	@echo "Old tag is $(LAST_TAG)"
-	@echo "New tag is $(NEW_TAG)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go
-	echo -n "$(NEW_TAG)" > docs/layouts/partials/version.html
-	git tag -s -m "Version $(NEW_TAG)" $(NEW_TAG)
-	bin/make_changelog.py $(LAST_TAG) $(NEW_TAG) > docs/content/changelog.md.new
+	@echo "Old tag is $(VERSION)"
+	@echo "New tag is $(NEXT_VERSION)"
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)\"\n" | gofmt > fs/version.go
+	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
+	echo "$(NEXT_VERSION)" > VERSION
+	git tag -s -m "Version $(NEXT_VERSION)" $(NEXT_VERSION)
+	bin/make_changelog.py $(LAST_TAG) $(NEXT_VERSION) > docs/content/changelog.md.new
 	mv docs/content/changelog.md.new docs/content/changelog.md
 	@echo "Edit the new changelog in docs/content/changelog.md"
 	@echo "Then commit all the changes"
-	@echo git commit -m \"Version $(NEW_TAG)\" -a -v
+	@echo git commit -m \"Version $(NEXT_VERSION)\" -a -v
 	@echo "And finally run make retag before make cross etc"
 
 retag:
-	git tag -f -s -m "Version $(LAST_TAG)" $(LAST_TAG)
+	git tag -f -s -m "Version $(VERSION)" $(VERSION)
 
 startdev:
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(LAST_TAG)-DEV\"\n" | gofmt > fs/version.go
-	git commit -m "Start $(LAST_TAG)-DEV development" fs/version.go
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(VERSION)-DEV\"\n" | gofmt > fs/version.go
+	git commit -m "Start $(VERSION)-DEV development" fs/version.go
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -8,12 +8,10 @@
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/)
 
-[![Build Status](https://travis-ci.org/rclone/rclone.svg?branch=master)](https://travis-ci.org/rclone/rclone)
-[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/rclone/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/rclone/rclone)
-[![Build Status](https://dev.azure.com/rclone/rclone/_apis/build/status/rclone.rclone?branchName=master)](https://dev.azure.com/rclone/rclone/_build/latest?definitionId=2&branchName=master)
-[![CircleCI](https://circleci.com/gh/rclone/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/rclone/rclone/tree/master)
+[![Build Status](https://github.com/rclone/rclone/workflows/build/badge.svg)](https://github.com/rclone/rclone/actions?query=workflow%3Abuild)
 [![Go Report Card](https://goreportcard.com/badge/github.com/rclone/rclone)](https://goreportcard.com/report/github.com/rclone/rclone)
 [![GoDoc](https://godoc.org/github.com/rclone/rclone?status.svg)](https://godoc.org/github.com/rclone/rclone) 
+[![Docker Pulls](https://img.shields.io/docker/pulls/rclone/rclone)](https://hub.docker.com/r/rclone/rclone)
 
 # Rclone
 
@@ -21,17 +19,19 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
 
 ## Storage providers
 
-  * 1Fichier [:page_facing_up:](https://rclone.org/ficher/)
+  * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
+  * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
+  * GetSky [:page_facing_up:](https://rclone.org/jottacloud/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
@@ -40,8 +40,10 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
   * IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
   * Koofr [:page_facing_up:](https://rclone.org/koofr/)
+  * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
+  * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
   * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
   * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
@@ -57,7 +59,11 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
+  * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
+  * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
+  * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
+  * Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
@@ -74,6 +80,7 @@ Please see [the full list of all storage providers and their features](https://r
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
   * Can sync to and from network, e.g. two different cloud accounts
+  * Optional large file chunking ([Chunker](https://rclone.org/chunker/))
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))

RELEASE.md

@@ -1,20 +1,28 @@
-Extra required software for making a release
+# Release
+
+This file describes how to make the various kinds of releases
+
+## Extra required software for making a release
+
   * [github-release](https://github.com/aktau/github-release) for uploading packages
   * pandoc for making the html and man pages
 
-Making a release
+## Making a release
+
+  * git checkout master
+  * git pull
   * git status - make sure everything is checked in
-  * Check travis & appveyor builds are green
-  * make check
+  * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
   * make tag
-  * edit docs/content/changelog.md
+  * edit docs/content/changelog.md # make sure to remove duplicate logs from point releases
+  * make tidy
   * make doc
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
   * git push --tags origin master
-  * # Wait for the appveyor and travis builds to complete then...
+  * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
   * make tarball
   * make sign_upload
@@ -23,11 +31,11 @@ Making a release
   * make upload_website
   * make upload_github
   * make startdev
-  * # announce with forum post, twitter post, G+ post
+  * # announce with forum post, twitter post, patreon post
 
 Early in the next release cycle update the vendored dependencies
+
   * Review any pinned packages in go.mod and remove if possible
-      * GO111MODULE=on go get -u github.com/spf13/cobra@master
   * make update
   * git status
   * git add new files
@@ -48,24 +56,57 @@ Can be fixed with
     * GO111MODULE=on go mod vendor
  
 
-Making a point release.  If rclone needs a point release due to some
-horrendous bug, then
-  * git branch v1.XX v1.XX-fixes
+## Making a point release
+
+If rclone needs a point release due to some horrendous bug:
+
+First make the release branch.  If this is a second point release then
+this will be done already.
+
+  * BASE_TAG=v1.XX          # eg v1.52
+  * NEW_TAG=${BASE_TAG}.Y   # eg v1.52.1
+  * echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
+  * git branch ${BASE_TAG} ${BASE_TAG}-stable
+
+Now
+
+  * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
   * Test (see above)
-  * make NEW_TAG=v1.XX.1 tag
+  * make NEXT_VERSION=${NEW_TAG} tag
   * edit docs/content/changelog.md
-  * make TAG=v1.43.1 doc
-  * git commit -a -v -m "Version v1.XX.1"
-  * git tag -d -v1.XX.1
-  * git tag -s -m "Version v1.XX.1" v1.XX.1
-  * git push --tags -u origin v1.XX-fixes
-  * make BRANCH_PATH= TAG=v1.43.1 fetch_binaries
-  * make TAG=v1.43.1 tarball
-  * make TAG=v1.43.1 sign_upload
-  * make TAG=v1.43.1 check_sign
-  * make TAG=v1.43.1 upload
-  * make TAG=v1.43.1 upload_website
-  * make TAG=v1.43.1 upload_github
-  * NB this overwrites the current beta so after the release, rebuild the last travis build
+  * make TAG=${NEW_TAG} doc
+  * git commit -a -v -m "Version ${NEW_TAG}"
+  * git tag -d ${NEW_TAG}
+  * git tag -s -m "Version ${NEW_TAG}" ${NEW_TAG}
+  * git push --tags -u origin ${BASE_TAG}-stable
+  * Wait for builds to complete
+  * make BRANCH_PATH= TAG=${NEW_TAG} fetch_binaries
+  * make TAG=${NEW_TAG} tarball
+  * make TAG=${NEW_TAG} sign_upload
+  * make TAG=${NEW_TAG} check_sign
+  * make TAG=${NEW_TAG} upload
+  * make TAG=${NEW_TAG} upload_website
+  * make TAG=${NEW_TAG} upload_github
+  * NB this overwrites the current beta so we need to do this
+  * git co master
+  * make VERSION=${NEW_TAG} startdev
+  * # cherry pick the changes to the changelog and VERSION
+  * git checkout ${BASE_TAG}-stable VERSION docs/content/changelog.md
+  * git commit --amend
+  * git push
   * Announce!
+
+## Making a manual build of docker
+
+The rclone docker image should autobuild on via GitHub actions.  If it doesn't
+or needs to be updated then rebuild like this.
+
+```
+docker pull golang
+docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
+docker push rclone/rclone:1.52.0
+docker push rclone/rclone:1.52
+docker push rclone/rclone:1
+docker push rclone/rclone:latest
+```

VERSION

@@ -0,0 +1 @@
+v1.52.2

azure-pipelines.yml

@@ -1,239 +0,0 @@
----
-# Azure pipelines build for rclone
-# Parts stolen shamelessly from all round the Internet, especially Caddy
-# -*- compile-command: "yamllint -f parsable azure-pipelines.yml" -*-
-
-trigger:
-  branches:
-    include:
-      - '*'
-  tags:
-    include:
-      - '*'
-
-variables:
-  GOROOT: $(gorootDir)/go
-  GOPATH: $(system.defaultWorkingDirectory)/gopath
-  GOCACHE: $(system.defaultWorkingDirectory)/gocache
-  GOBIN: $(GOPATH)/bin
-  modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
-  GO111MODULE: 'off'
-  GOTAGS: cmount
-  GO_LATEST: false
-  CPATH: ''
-  GO_INSTALL_ARCH: amd64
-
-strategy:
-  matrix:
-    linux:
-      imageName: ubuntu-16.04
-      gorootDir: /usr/local
-      GO_VERSION: latest
-      GOTAGS: cmount
-      BUILD_FLAGS: '-include "^linux/"'
-      MAKE_CHECK: true
-      MAKE_QUICKTEST: true
-      DEPLOY: true
-    mac:
-      imageName: macos-10.13
-      gorootDir: /usr/local
-      GO_VERSION: latest
-      GOTAGS: ""  # cmount doesn't work on osx travis for some reason
-      BUILD_FLAGS: '-include "^darwin/" -cgo'
-      MAKE_QUICKTEST: true
-      MAKE_RACEQUICKTEST: true
-      DEPLOY: true
-    windows_amd64:
-      imageName: windows-2019
-      gorootDir: C:\
-      GO_VERSION: latest
-      BUILD_FLAGS: '-include "^windows/amd64" -cgo'
-      MAKE_QUICKTEST: true
-      DEPLOY: true
-    windows_386:
-      imageName: windows-2019
-      gorootDir: C:\
-      GO_VERSION: latest
-      GO_INSTALL_ARCH: 386
-      BUILD_FLAGS: '-include "^windows/386" -cgo'
-      MAKE_QUICKTEST: true
-      DEPLOY: true
-    other_os:
-      imageName: ubuntu-16.04
-      gorootDir: /usr/local
-      GO_VERSION: latest
-      BUILD_FLAGS: '-exclude "^(windows|darwin|linux)/"'
-      MAKE_COMPILE_ALL: true
-      DEPLOY: true
-    modules_race:
-      imageName: ubuntu-16.04
-      gorootDir: /usr/local
-      GO_VERSION: latest
-      GO111MODULE: on
-      GOPROXY: https://proxy.golang.org
-      MAKE_QUICKTEST: true
-      MAKE_RACEQUICKTEST: true
-    go1.9:
-      imageName: ubuntu-16.04
-      gorootDir: /usr/local
-      GOCACHE: ''               # build caching only came in go1.10
-      GO_VERSION: go1.9.7
-      MAKE_QUICKTEST: true
-    go1.10:
-      imageName: ubuntu-16.04
-      gorootDir: /usr/local
-      GO_VERSION: go1.10.8
-      MAKE_QUICKTEST: true
-    go1.11:
-      imageName: ubuntu-16.04
-      gorootDir: /usr/local
-      GO_VERSION: go1.11.12
-      MAKE_QUICKTEST: true
-
-pool:
-  vmImage: $(imageName)
-
-steps:
-  - bash: |
-      latestGo=$(curl "https://golang.org/VERSION?m=text")
-      echo "##vso[task.setvariable variable=GO_VERSION]$latestGo"
-      echo "##vso[task.setvariable variable=GO_LATEST]true"
-      echo "Latest Go version: $latestGo"
-    condition: eq( variables['GO_VERSION'], 'latest' )
-    continueOnError: false
-    displayName: "Get latest Go version"
-
-  - bash: |
-      sudo rm -f $(which go)
-      echo '##vso[task.prependpath]$(GOBIN)'
-      echo '##vso[task.prependpath]$(GOROOT)/bin'
-      mkdir -p '$(modulePath)'
-      shopt -s extglob
-      shopt -s dotglob
-      mv !(gopath) '$(modulePath)'
-    continueOnError: false
-    displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
-
-  - task: CacheBeta@0
-    inputs:
-      key: go-build-cache | "$(Agent.JobName)"
-      path: $(GOCACHE)
-    continueOnError: true
-    displayName: Cache go build
-    condition: ne( variables['GOCACHE'], '' )
-
-  # Install Libraries (varies by platform)
-
-  - bash: |
-      sudo modprobe fuse
-      sudo chmod 666 /dev/fuse
-      sudo chown root:$USER /etc/fuse.conf
-      sudo apt-get install fuse libfuse-dev rpm pkg-config
-    condition: eq( variables['Agent.OS'], 'Linux' )
-    continueOnError: false
-    displayName: Install Libraries on Linux
-
-  - bash: |
-      brew update
-      brew tap caskroom/cask
-      brew cask install osxfuse
-    condition: eq( variables['Agent.OS'], 'Darwin' )
-    continueOnError: false
-    displayName: Install Libraries on macOS
-
-  - powershell: |
-      $ProgressPreference = 'SilentlyContinue'
-      choco install -y winfsp zip
-      Write-Host "##vso[task.setvariable variable=CPATH]C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse"
-      if ($env:GO_INSTALL_ARCH -eq "386") {
-        choco install -y mingw --forcex86 --force
-        Write-Host "##vso[task.prependpath]C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin"
-      }
-      # Copy mingw32-make.exe to make.exe so the same command line
-      # can be used on Windows as on macOS and Linux
-      $path = (get-command mingw32-make.exe).Path
-      Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
-    condition: eq( variables['Agent.OS'], 'Windows_NT' )
-    continueOnError: false
-    displayName: Install Libraries on Windows
-
-
-  # Install Go (this varies by platform)
-
-  - bash: |
-      wget "https://dl.google.com/go/$(GO_VERSION).linux-$(GO_INSTALL_ARCH).tar.gz"
-      sudo mkdir $(gorootDir)
-      sudo chown ${USER}:${USER} $(gorootDir)
-      tar -C $(gorootDir) -xzf "$(GO_VERSION).linux-$(GO_INSTALL_ARCH).tar.gz"
-    condition: eq( variables['Agent.OS'], 'Linux' )
-    continueOnError: false
-    displayName: Install Go on Linux
-
-  - bash: |
-      wget "https://dl.google.com/go/$(GO_VERSION).darwin-$(GO_INSTALL_ARCH).tar.gz"
-      sudo tar -C $(gorootDir) -xzf "$(GO_VERSION).darwin-$(GO_INSTALL_ARCH).tar.gz"
-    condition: eq( variables['Agent.OS'], 'Darwin' )
-    continueOnError: false
-    displayName: Install Go on macOS
-
-  - powershell: |
-      $ProgressPreference = 'SilentlyContinue'
-      Write-Host "Downloading Go $(GO_VERSION) for $(GO_INSTALL_ARCH)"
-      (New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip", "$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip")
-      Write-Host "Extracting Go"
-      Expand-Archive "$(GO_VERSION).windows-$(GO_INSTALL_ARCH).zip" -DestinationPath "$(gorootDir)"
-    condition: eq( variables['Agent.OS'], 'Windows_NT' )
-    continueOnError: false
-    displayName: Install Go on Windows
-
-  # Display environment for debugging
-
-  - bash: |
-      printf "Using go at: $(which go)\n"
-      printf "Go version: $(go version)\n"
-      printf "\n\nGo environment:\n\n"
-      go env
-      printf "\n\nRclone environment:\n\n"
-      make vars
-      printf "\n\nSystem environment:\n\n"
-      env
-    workingDirectory: '$(modulePath)'
-    displayName: Print Go version and environment
-
-  # Run Tests
-
-  - bash: |
-      make
-      make quicktest
-    workingDirectory: '$(modulePath)'
-    displayName: Run tests
-    condition: eq( variables['MAKE_QUICKTEST'], 'true' )
-
-  - bash: |
-      make racequicktest
-    workingDirectory: '$(modulePath)'
-    displayName: Race test
-    condition: eq( variables['MAKE_RACEQUICKTEST'], 'true' )
-
-  - bash: |
-      make build_dep
-      make check
-    workingDirectory: '$(modulePath)'
-    displayName: Code quality test
-    condition: eq( variables['MAKE_CHECK'], 'true' )
-
-  - bash: |
-      make
-      make compile_all
-    workingDirectory: '$(modulePath)'
-    displayName: Compile all architectures test
-    condition: eq( variables['MAKE_COMPILE_ALL'], 'true' )
-
-  - bash: |
-      make travis_beta
-    env:
-      RCLONE_CONFIG_PASS: $(RCLONE_CONFIG_PASS)
-      BETA_SUBDIR: 'azure_pipelines'  # FIXME remove when removing travis/appveyor
-    workingDirectory: '$(modulePath)'
-    displayName: Deploy built binaries
-    condition: and( eq( variables['DEPLOY'], 'true' ), ne( variables['Build.Reason'], 'PullRequest' ) )

backend/all/all.go

@@ -8,6 +8,7 @@ import (
 	_ "github.com/rclone/rclone/backend/b2"
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
+	_ "github.com/rclone/rclone/backend/chunker"
 	_ "github.com/rclone/rclone/backend/crypt"
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/dropbox"
@@ -20,7 +21,9 @@ import (
 	_ "github.com/rclone/rclone/backend/jottacloud"
 	_ "github.com/rclone/rclone/backend/koofr"
 	_ "github.com/rclone/rclone/backend/local"
+	_ "github.com/rclone/rclone/backend/mailru"
 	_ "github.com/rclone/rclone/backend/mega"
+	_ "github.com/rclone/rclone/backend/memory"
 	_ "github.com/rclone/rclone/backend/onedrive"
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/pcloud"
@@ -28,8 +31,12 @@ import (
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"
+	_ "github.com/rclone/rclone/backend/seafile"
 	_ "github.com/rclone/rclone/backend/sftp"
+	_ "github.com/rclone/rclone/backend/sharefile"
+	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
+	_ "github.com/rclone/rclone/backend/tardigrade"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/webdav"
 	_ "github.com/rclone/rclone/backend/yandex"

backend/amazonclouddrive/amazonclouddrive.go

@@ -32,6 +32,7 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"golang.org/x/oauth2"
@@ -70,7 +71,7 @@ func init() {
 		Description: "Amazon Drive",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("amazon cloud drive", name, m, acdConfig)
+			err := oauthutil.Config("amazon cloud drive", name, m, acdConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -135,15 +136,23 @@ which downloads the file through a temporary URL directly from the
 underlying S3 storage.`,
 			Default:  defaultTempLinkThreshold,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Checkpoint        string        `config:"checkpoint"`
-	UploadWaitPerGB   fs.Duration   `config:"upload_wait_per_gb"`
-	TempLinkThreshold fs.SizeSuffix `config:"templink_threshold"`
+	Checkpoint        string               `config:"checkpoint"`
+	UploadWaitPerGB   fs.Duration          `config:"upload_wait_per_gb"`
+	TempLinkThreshold fs.SizeSuffix        `config:"templink_threshold"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote acd server
@@ -160,7 +169,7 @@ type Fs struct {
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
-// Object describes a acd object
+// Object describes an acd object
 //
 // Will definitely have info but maybe not meta
 type Object struct {
@@ -220,7 +229,7 @@ func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 		}
 		// Work around receiving this error sporadically on authentication
 		//
-		// HTTP code 403: "403 Forbidden", reponse body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
+		// HTTP code 403: "403 Forbidden", response body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
 		if resp.StatusCode == 403 && strings.Contains(err.Error(), "Authorization header requires") {
 			fs.Debugf(f, "403 \"Authorization header requires...\" error received - retry")
 			return true, err
@@ -384,7 +393,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	var resp *http.Response
 	var subFolder *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
-		subFolder, resp, err = folder.GetFolder(leaf)
+		subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -411,7 +420,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	var resp *http.Response
 	var info *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
-		info, resp, err = folder.CreateFolder(leaf)
+		info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -479,6 +488,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 				if !hasValidParent {
 					continue
 				}
+				*node.Name = f.opt.Enc.ToStandardName(*node.Name)
 				// Store the nodes up in case we have to retry the listing
 				out = append(out, node)
 			}
@@ -504,10 +514,6 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -655,7 +661,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return nil, err
 	}
 	// If not create it
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -668,7 +674,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		start := time.Now()
 		f.tokenRenewer.Start()
-		info, resp, err = folder.Put(in, leaf)
+		info, resp, err = folder.Put(in, f.opt.Enc.FromStandardName(leaf))
 		f.tokenRenewer.Stop()
 		var ok bool
 		ok, info, err = f.checkUpload(ctx, resp, in, src, info, err, time.Since(start))
@@ -686,13 +692,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -714,10 +714,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// create the destination directory if necessary
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return nil, err
-	}
 	srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
 	if err != nil {
 		return nil, err
@@ -787,54 +783,24 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return errors.New("can't move root directory")
 	}
 
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
 	// Find ID of dst parent, creating subdirs if necessary
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, findPath, true)
+	dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, dstRemote, true)
 	if err != nil {
 		return err
 	}
 
 	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
+	_, err = f.dirCache.FindDir(ctx, dstRemote, false)
+	if err == fs.ErrorDirNotFound {
+		// OK
+	} else if err != nil {
+		return err
+	} else {
+		return fs.ErrorDirExists
 	}
 
 	// Find ID of src parent
-	findPath = srcRemote
-	var srcDirectoryID string
-	if srcRemote == "" {
-		srcDirectoryID, err = srcFs.dirCache.RootParentID()
-	} else {
-		_, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
-	}
+	_, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcRemote, false)
 	if err != nil {
 		return err
 	}
@@ -880,10 +846,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1027,7 +989,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.info != nil {
 		return nil
 	}
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return fs.ErrorObjectNotFound
@@ -1038,7 +1000,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	var resp *http.Response
 	var info *acd.File
 	err = o.fs.pacer.Call(func() (bool, error) {
-		info, resp, err = folder.GetFile(leaf)
+		info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1158,7 +1120,7 @@ func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) {
 func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		newInfo, resp, err = info.Rename(newName)
+		newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
 		return f.shouldRetry(resp, err)
 	})
 	return newInfo, err
@@ -1354,10 +1316,11 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoin
 					if len(node.Parents) > 0 {
 						if path, ok := f.dirCache.GetInv(node.Parents[0]); ok {
 							// and append the drive file name to compute the full file name
+							name := f.opt.Enc.ToStandardName(*node.Name)
 							if len(path) > 0 {
-								path = path + "/" + *node.Name
+								path = path + "/" + name
 							} else {
-								path = *node.Name
+								path = name
 							}
 							// this will now clear the actual file too
 							pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject})

backend/azureblob/azureblob.go

@@ -9,14 +9,12 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
-	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -26,6 +24,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -33,7 +32,11 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/readers"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -58,6 +61,8 @@ const (
 	emulatorAccount      = "devstoreaccount1"
 	emulatorAccountKey   = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 	emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
+	memoryPoolFlushTime  = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap    = false
 )
 
 // Register with Fs
@@ -124,21 +129,57 @@ If blobs are in "archive tier" at remote, trying to perform data transfer
 operations from remote will not be allowed. User should first restore by
 tiering blob to "Hot" or "Cool".`,
 			Advanced: true,
+		}, {
+			Name: "disable_checksum",
+			Help: `Don't store MD5 checksum with object metadata.
+
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     "memory_pool_flush_time",
+			Default:  memoryPoolFlushTime,
+			Advanced: true,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
+		}, {
+			Name:     "memory_pool_use_mmap",
+			Default:  memoryPoolUseMmap,
+			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.EncodeInvalidUtf8 |
+				encoder.EncodeSlash |
+				encoder.EncodeCtl |
+				encoder.EncodeDel |
+				encoder.EncodeBackSlash |
+				encoder.EncodeRightPeriod),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account       string        `config:"account"`
-	Key           string        `config:"key"`
-	Endpoint      string        `config:"endpoint"`
-	SASURL        string        `config:"sas_url"`
-	UploadCutoff  fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
-	ListChunkSize uint          `config:"list_chunk"`
-	AccessTier    string        `config:"access_tier"`
-	UseEmulator   bool          `config:"use_emulator"`
+	Account             string               `config:"account"`
+	Key                 string               `config:"key"`
+	Endpoint            string               `config:"endpoint"`
+	SASURL              string               `config:"sas_url"`
+	UploadCutoff        fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
+	ListChunkSize       uint                 `config:"list_chunk"`
+	AccessTier          string               `config:"access_tier"`
+	UseEmulator         bool                 `config:"use_emulator"`
+	DisableCheckSum     bool                 `config:"disable_checksum"`
+	MemoryPoolFlushTime fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap   bool                 `config:"memory_pool_use_mmap"`
+	Enc                 encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote azure server
@@ -157,9 +198,10 @@ type Fs struct {
 	cache         *bucket.Cache                   // cache for container creation status
 	pacer         *fs.Pacer                       // To pace and retry the API calls
 	uploadToken   *pacer.TokenDispenser           // control concurrency
+	pool          *pool.Pool                      // memory pool
 }
 
-// Object describes a azure object
+// Object describes an azure object
 type Object struct {
 	fs         *Fs                   // what this object is part of
 	remote     string                // The remote path
@@ -186,7 +228,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootContainer == "" {
-		return fmt.Sprintf("Azure root")
+		return "Azure root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("Azure container %s", f.rootContainer)
@@ -208,7 +250,8 @@ func parsePath(path string) (root string) {
 // split returns container and containerPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	containerName, containerPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
 }
 
 // split returns container and containerPath from the object
@@ -244,6 +287,12 @@ var retryErrorCodes = []int{
 func (f *Fs) shouldRetry(err error) (bool, error) {
 	// FIXME interpret special errors - more to do here
 	if storageErr, ok := err.(azblob.StorageError); ok {
+		switch storageErr.ServiceCode() {
+		case "InvalidBlobOrBlock":
+			// These errors happen sometimes in multipart uploads
+			// because of block concurrency issues
+			return true, err
+		}
 		statusCode := storageErr.Response().StatusCode
 		for _, e := range retryErrorCodes {
 			if statusCode == e {
@@ -289,7 +338,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 }
 
 // httpClientFactory creates a Factory object that sends HTTP requests
-// to a rclone's http.Client.
+// to an rclone's http.Client.
 //
 // copied from azblob.newDefaultHTTPClientFactory
 func httpClientFactory(client *http.Client) pipeline.Factory {
@@ -308,6 +357,9 @@ func httpClientFactory(client *http.Client) pipeline.Factory {
 //
 // this code was copied from azblob.NewPipeline
 func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline {
+	// Don't log stuff to syslog/Windows Event log
+	pipeline.SetForceLogEnabled(false)
+
 	// Closest to API goes first; closest to the wire goes last
 	factories := []pipeline.Factory{
 		azblob.NewTelemetryPolicyFactory(o.Telemetry),
@@ -366,6 +418,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		client:      fshttp.NewClient(fs.Config),
 		cache:       bucket.NewCache(),
 		cntURLcache: make(map[string]*azblob.ContainerURL, 1),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			fs.Config.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -575,18 +633,18 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		}
 		// Advance marker to next
 		marker = response.NextMarker
-
 		for i := range response.Segment.BlobItems {
 			file := &response.Segment.BlobItems[i]
 			// Finish if file name no longer has prefix
 			// if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
 			// 	return nil
 			// }
-			if !strings.HasPrefix(file.Name, prefix) {
-				fs.Debugf(f, "Odd name received %q", file.Name)
+			remote := f.opt.Enc.ToStandardPath(file.Name)
+			if !strings.HasPrefix(remote, prefix) {
+				fs.Debugf(f, "Odd name received %q", remote)
 				continue
 			}
-			remote := file.Name[len(prefix):]
+			remote = remote[len(prefix):]
 			if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
 				continue // skip directory marker
 			}
@@ -602,6 +660,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		// Send the subdirectories
 		for _, remote := range response.Segment.BlobPrefixes {
 			remote := strings.TrimRight(remote.Name, "/")
+			remote = f.opt.Enc.ToStandardPath(remote)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Debugf(f, "Odd directory name received %q", remote)
 				continue
@@ -665,7 +724,7 @@ func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err err
 		return entries, nil
 	}
 	err = f.listContainersToFn(func(container *azblob.ContainerItem) error {
-		d := fs.NewDir(container.Name, container.Properties.LastModified)
+		d := fs.NewDir(f.opt.Enc.ToStandardName(container.Name), container.Properties.LastModified)
 		f.cache.MarkOK(container.Name)
 		entries = append(entries, d)
 		return nil
@@ -799,6 +858,11 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	return fs, fs.Update(ctx, in, src, options...)
 }
 
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	container, _ := f.split(dir)
@@ -808,6 +872,10 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 // makeContainer creates the container if it doesn't exist
 func (f *Fs) makeContainer(ctx context.Context, container string) error {
 	return f.cache.Create(container, func() error {
+		// If this is a SAS URL limited to a container then assume it is already created
+		if f.isLimited {
+			return nil
+		}
 		// now try to create the container
 		return f.pacer.Call(func() (bool, error) {
 			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
@@ -961,6 +1029,19 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
+func (f *Fs) getMemoryPool(size int64) *pool.Pool {
+	if size == int64(f.opt.ChunkSize) {
+		return f.pool
+	}
+
+	return pool.New(
+		time.Duration(f.opt.MemoryPoolFlushTime),
+		int(size),
+		fs.Config.Transfers,
+		f.opt.MemoryPoolUseMmap,
+	)
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1103,22 +1184,6 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
-// parseTimeString converts a decimal string number of milliseconds
-// elapsed since January 1, 1970 UTC into a time.Time and stores it in
-// the modTime variable.
-func (o *Object) parseTimeString(timeString string) (err error) {
-	if timeString == "" {
-		return nil
-	}
-	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
-	if err != nil {
-		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return err
-	}
-	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
-	return nil
-}
-
 // ModTime returns the modification time of the object
 //
 // It attempts to read the objects mtime and if that isn't present the
@@ -1220,103 +1285,116 @@ type readSeeker struct {
 	io.Seeker
 }
 
+// increment the slice passed in as LSB binary
+func increment(xs []byte) {
+	for i, digit := range xs {
+		newDigit := digit + 1
+		xs[i] = newDigit
+		if newDigit >= digit {
+			// exit if no carry
+			break
+		}
+	}
+}
+
+var warnStreamUpload sync.Once
+
 // uploadMultipart uploads a file using multipart upload
 //
 // Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
-func (o *Object) uploadMultipart(in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) {
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) {
 	// Calculate correct chunkSize
 	chunkSize := int64(o.fs.opt.ChunkSize)
-	var totalParts int64
-	for {
-		// Calculate number of parts
-		var remainder int64
-		totalParts, remainder = size/chunkSize, size%chunkSize
-		if remainder != 0 {
-			totalParts++
+	totalParts := -1
+
+	// Note that the max size of file is 4.75 TB (100 MB X 50,000
+	// blocks) and this is bigger than the max uncommitted block
+	// size (9.52 TB) so we do not need to part commit block lists
+	// or garbage collect uncommitted blocks.
+	//
+	// See: https://docs.microsoft.com/en-gb/rest/api/storageservices/put-block
+
+	// size can be -1 here meaning we don't know the size of the incoming file.  We use ChunkSize
+	// buffers here (default 4MB). With a maximum number of parts (50,000) this will be a file of
+	// 195GB which seems like a not too unreasonable limit.
+	if size == -1 {
+		warnStreamUpload.Do(func() {
+			fs.Logf(o, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				o.fs.opt.ChunkSize, fs.SizeSuffix(chunkSize*maxTotalParts))
+		})
+	} else {
+		// Adjust partSize until the number of parts is small enough.
+		if size/chunkSize >= maxTotalParts {
+			// Calculate partition size rounded up to the nearest MB
+			chunkSize = (((size / maxTotalParts) >> 20) + 1) << 20
 		}
-		if totalParts < maxTotalParts {
-			break
-		}
-		// Double chunk size if the number of parts is too big
-		chunkSize *= 2
 		if chunkSize > int64(maxChunkSize) {
 			return errors.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), totalParts, fs.SizeSuffix(chunkSize/2))
 		}
+		totalParts = int(size / chunkSize)
+		if size%chunkSize != 0 {
+			totalParts++
+		}
 	}
+
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, fs.SizeSuffix(chunkSize))
 
-	// https://godoc.org/github.com/Azure/azure-storage-blob-go/2017-07-29/azblob#example-BlockBlobURL
-	// Utilities are cloned from above example
-	// These helper functions convert a binary block ID to a base-64 string and vice versa
-	// NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length
-	blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) }
-	// These helper functions convert an int block ID to a base-64 string and vice versa
-	blockIDIntToBase64 := func(blockID uint64) string {
-		binaryBlockID := (&[8]byte{})[:] // All block IDs are 8 bytes long
-		binary.LittleEndian.PutUint64(binaryBlockID, blockID)
-		return blockIDBinaryToBase64(binaryBlockID)
-	}
-
-	// block ID variables
-	var (
-		rawID   uint64
-		blockID = "" // id in base64 encoded form
-		blocks  []string
-	)
-
-	// increment the blockID
-	nextID := func() {
-		rawID++
-		blockID = blockIDIntToBase64(rawID)
-		blocks = append(blocks, blockID)
-	}
-
-	// Get BlockBlobURL, we will use default pipeline here
-	blockBlobURL := blob.ToBlockBlobURL()
-	ctx := context.Background()
-	ac := azblob.LeaseAccessConditions{} // Use default lease access conditions
-
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
 	in, wrap := accounting.UnWrap(in)
 
 	// Upload the chunks
-	remaining := size
-	position := int64(0)
-	errs := make(chan error, 1)
-	var wg sync.WaitGroup
-outer:
-	for part := 0; part < int(totalParts); part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+	var (
+		g, gCtx       = errgroup.WithContext(ctx)
+		remaining     = size                           // remaining size in file for logging only, -1 if size < 0
+		position      = int64(0)                       // position in file
+		memPool       = o.fs.getMemoryPool(chunkSize)  // pool to get memory from
+		finished      = false                          // set when we have read EOF
+		blocks        []string                         // list of blocks for finalize
+		blockBlobURL  = blob.ToBlockBlobURL()          // Get BlockBlobURL, we will use default pipeline here
+		ac            = azblob.LeaseAccessConditions{} // Use default lease access conditions
+		binaryBlockID = make([]byte, 8)                // block counter as LSB first 8 bytes
+	)
+	for part := 0; !finished; part++ {
+		// Get a block of memory from the pool and a token which limits concurrency
+		o.fs.uploadToken.Get()
+		buf := memPool.Get()
+
+		free := func() {
+			memPool.Put(buf)       // return the buf
+			o.fs.uploadToken.Put() // return the token
 		}
 
-		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in uploading all the other parts.
+		if gCtx.Err() != nil {
+			free()
+			break
 		}
 
-		// Make a block of memory
-		buf := make([]byte, reqSize)
-
 		// Read the chunk
-		_, err = io.ReadFull(in, buf)
-		if err != nil {
-			err = errors.Wrap(err, "multipart upload failed to read source")
-			break outer
+		n, err := readers.ReadFill(in, buf) // this can never return 0, nil
+		if err == io.EOF {
+			if n == 0 { // end if no data
+				free()
+				break
+			}
+			finished = true
+		} else if err != nil {
+			free()
+			return errors.Wrap(err, "multipart upload failed to read source")
 		}
+		buf = buf[:n]
+
+		// increment the blockID and save the blocks for finalize
+		increment(binaryBlockID)
+		blockID := base64.StdEncoding.EncodeToString(binaryBlockID)
+		blocks = append(blocks, blockID)
 
 		// Transfer the chunk
-		nextID()
-		wg.Add(1)
-		o.fs.uploadToken.Get()
-		go func(part int, position int64, blockID string) {
-			defer wg.Done()
-			defer o.fs.uploadToken.Put()
-			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
+		fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
+		g.Go(func() (err error) {
+			defer free()
 
 			// Upload the block, with MD5 for check
 			md5sum := md5.Sum(buf)
@@ -1328,28 +1406,19 @@ outer:
 				_, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac, transactionalMD5)
 				return o.fs.shouldRetry(err)
 			})
-
 			if err != nil {
-				err = errors.Wrap(err, "multipart upload failed to upload part")
-				select {
-				case errs <- err:
-				default:
-				}
-				return
+				return errors.Wrap(err, "multipart upload failed to upload part")
 			}
-		}(part, position, blockID)
+			return nil
+		})
 
 		// ready for next block
-		remaining -= chunkSize
+		if size >= 0 {
+			remaining -= chunkSize
+		}
 		position += chunkSize
 	}
-	wg.Wait()
-	if err == nil {
-		select {
-		case err = <-errs:
-		default:
-		}
-	}
+	err = g.Wait()
 	if err != nil {
 		return err
 	}
@@ -1386,14 +1455,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	httpHeaders.ContentType = fs.MimeType(ctx, o)
 	// Compute the Content-MD5 of the file, for multiparts uploads it
 	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
+	// Note: If multipart, an MD5 checksum will also be computed for each uploaded block
 	// 		 in order to validate its integrity during transport
-	if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
-		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-		if err == nil {
-			httpHeaders.ContentMD5 = sourceMD5bytes
-		} else {
-			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+	if !o.fs.opt.DisableCheckSum {
+		if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
+			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+			if err == nil {
+				httpHeaders.ContentMD5 = sourceMD5bytes
+			} else {
+				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+			}
 		}
 	}
 
@@ -1407,7 +1478,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// is merged the SDK can't upload a single blob of exactly the chunk
 	// size, so upload with a multpart upload to work around.
 	// See: https://github.com/rclone/rclone/issues/2653
-	multipartUpload := size >= int64(o.fs.opt.UploadCutoff)
+	multipartUpload := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
 	if size == int64(o.fs.opt.ChunkSize) {
 		multipartUpload = true
 		fs.Debugf(o, "Setting multipart upload for file of chunk size (%d) to work around SDK bug", size)
@@ -1417,7 +1488,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		if multipartUpload {
 			// If a large file upload in chunks
-			err = o.uploadMultipart(in, size, &blob, &httpHeaders)
+			err = o.uploadMultipart(ctx, in, size, &blob, &httpHeaders)
 		} else {
 			// Write a small blob in one transaction
 			blockBlobURL := blob.ToBlockBlobURL()
@@ -1502,10 +1573,13 @@ func (o *Object) GetTier() string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs        = &Fs{}
-	_ fs.Copier    = &Fs{}
-	_ fs.Purger    = &Fs{}
-	_ fs.ListRer   = &Fs{}
-	_ fs.Object    = &Object{}
-	_ fs.MimeTyper = &Object{}
+	_ fs.Fs          = &Fs{}
+	_ fs.Copier      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.Purger      = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.Object      = &Object{}
+	_ fs.MimeTyper   = &Object{}
+	_ fs.GetTierer   = &Object{}
+	_ fs.SetTierer   = &Object{}
 )

backend/azureblob/azureblob_internal_test.go

@@ -16,3 +16,20 @@ func (f *Fs) InternalTest(t *testing.T) {
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+func TestIncrement(t *testing.T) {
+	for _, test := range []struct {
+		in   []byte
+		want []byte
+	}{
+		{[]byte{0, 0, 0, 0}, []byte{1, 0, 0, 0}},
+		{[]byte{0xFE, 0, 0, 0}, []byte{0xFF, 0, 0, 0}},
+		{[]byte{0xFF, 0, 0, 0}, []byte{0, 1, 0, 0}},
+		{[]byte{0, 1, 0, 0}, []byte{1, 1, 0, 0}},
+		{[]byte{0xFF, 0xFF, 0xFF, 0xFE}, []byte{0, 0, 0, 0xFF}},
+		{[]byte{0xFF, 0xFF, 0xFF, 0xFF}, []byte{0, 0, 0, 0}},
+	} {
+		increment(test.in)
+		assert.Equal(t, test.want, test.in)
+	}
+}

backend/b2/api/types.go

@@ -50,7 +50,7 @@ type Timestamp time.Time
 // MarshalJSON turns a Timestamp into JSON (in UTC)
 func (t *Timestamp) MarshalJSON() (out []byte, err error) {
 	timestamp := (*time.Time)(t).UTC().UnixNano()
-	return []byte(strconv.FormatInt(timestamp/1E6, 10)), nil
+	return []byte(strconv.FormatInt(timestamp/1e6, 10)), nil
 }
 
 // UnmarshalJSON turns JSON into a Timestamp
@@ -59,7 +59,7 @@ func (t *Timestamp) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		return err
 	}
-	*t = Timestamp(time.Unix(timestamp/1E3, (timestamp%1E3)*1E6).UTC())
+	*t = Timestamp(time.Unix(timestamp/1e3, (timestamp%1e3)*1e6).UTC())
 	return nil
 }
 
@@ -337,3 +337,11 @@ type CopyFileRequest struct {
 	Info              map[string]string `json:"fileInfo,omitempty"`            // This field stores the metadata that will be stored with the file. (REPLACE only)
 	DestBucketID      string            `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
 }
+
+// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
+type CopyPartRequest struct {
+	SourceID    string `json:"sourceFileId"`    // The ID of the source file being copied.
+	LargeFileID string `json:"largeFileId"`     // The ID of the large file the part will belong to, as returned by b2_start_large_file.
+	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
+	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
+}

backend/b2/b2.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -30,7 +31,9 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -52,6 +55,9 @@ const (
 	minChunkSize        = 5 * fs.MebiByte
 	defaultChunkSize    = 96 * fs.MebiByte
 	defaultUploadCutoff = 200 * fs.MebiByte
+	largeFileCopyCutoff = 4 * fs.GibiByte          // 5E9 is the max
+	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap   = false
 )
 
 // Globals
@@ -111,6 +117,16 @@ Files above this size will be uploaded in chunks of "--b2-chunk-size".
 This value should be set no larger than 4.657GiB (== 5GB).`,
 			Default:  defaultUploadCutoff,
 			Advanced: true,
+		}, {
+			Name: "copy_cutoff",
+			Help: `Cutoff for switching to multipart copy
+
+Any files larger than this that need to be server side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 4.6GB.`,
+			Default:  largeFileCopyCutoff,
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Upload chunk size. Must fit in memory.
@@ -122,8 +138,13 @@ minimum size.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
-			Name:     "disable_checksum",
-			Help:     `Disable checksums for large (> upload cutoff) files`,
+			Name: "disable_checksum",
+			Help: `Disable checksums for large (> upload cutoff) files
+
+Normally rclone will calculate the SHA1 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -143,23 +164,49 @@ The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
+		}, {
+			Name:     "memory_pool_flush_time",
+			Default:  memoryPoolFlushTime,
+			Advanced: true,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
+		}, {
+			Name:     "memory_pool_use_mmap",
+			Default:  memoryPoolUseMmap,
+			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// See: https://www.backblaze.com/b2/docs/files.html
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			// FIXME: allow /, but not leading, trailing or double
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account                       string        `config:"account"`
-	Key                           string        `config:"key"`
-	Endpoint                      string        `config:"endpoint"`
-	TestMode                      string        `config:"test_mode"`
-	Versions                      bool          `config:"versions"`
-	HardDelete                    bool          `config:"hard_delete"`
-	UploadCutoff                  fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize                     fs.SizeSuffix `config:"chunk_size"`
-	DisableCheckSum               bool          `config:"disable_checksum"`
-	DownloadURL                   string        `config:"download_url"`
-	DownloadAuthorizationDuration fs.Duration   `config:"download_auth_duration"`
+	Account                       string               `config:"account"`
+	Key                           string               `config:"key"`
+	Endpoint                      string               `config:"endpoint"`
+	TestMode                      string               `config:"test_mode"`
+	Versions                      bool                 `config:"versions"`
+	HardDelete                    bool                 `config:"hard_delete"`
+	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
+	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
+	ChunkSize                     fs.SizeSuffix        `config:"chunk_size"`
+	DisableCheckSum               bool                 `config:"disable_checksum"`
+	DownloadURL                   string               `config:"download_url"`
+	DownloadAuthorizationDuration fs.Duration          `config:"download_auth_duration"`
+	MemoryPoolFlushTime           fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap             bool                 `config:"memory_pool_use_mmap"`
+	Enc                           encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote b2 server
@@ -181,7 +228,8 @@ type Fs struct {
 	uploads         map[string][]*api.GetUploadURLResponse // Upload URLs by buckedID
 	authMu          sync.Mutex                             // lock for authorizing the account
 	pacer           *fs.Pacer                              // To pace and retry the API calls
-	bufferTokens    chan []byte                            // control concurrency of multipart uploads
+	uploadToken     *pacer.TokenDispenser                  // control concurrency
+	pool            *pool.Pool                             // memory pool
 }
 
 // Object describes a b2 object
@@ -273,11 +321,11 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if resp != nil && resp.StatusCode == 401 {
 		fs.Debugf(f, "Unauthorized: %v", err)
 		// Reauth
-		authErr := f.authorizeAccount()
+		authErr := f.authorizeAccount(ctx)
 		if authErr != nil {
 			err = authErr
 		}
@@ -317,7 +365,6 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	err = checkUploadChunkSize(cs)
 	if err == nil {
 		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-		f.fillBufferTokens() // reset the buffer tokens
 	}
 	return
 }
@@ -378,6 +425,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		_bucketType: make(map[string]string, 1),
 		uploads:     make(map[string][]*api.GetUploadURLResponse),
 		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			fs.Config.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -392,14 +446,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		f.srv.SetHeader(testModeHeader, testMode)
 		fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
 	}
-	f.fillBufferTokens()
-	err = f.authorizeAccount()
+	err = f.authorizeAccount(ctx)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to authorize account")
 	}
 	// If this is a key limited to a single bucket, it must exist already
 	if f.rootBucket != "" && f.info.Allowed.BucketID != "" {
-		allowedBucket := f.info.Allowed.BucketName
+		allowedBucket := f.opt.Enc.ToStandardName(f.info.Allowed.BucketName)
 		if allowedBucket == "" {
 			return nil, errors.New("bucket that application key is restricted to no longer exists")
 		}
@@ -431,7 +484,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 // authorizeAccount gets the API endpoint and auth token.  Can be used
 // for reauthentication too.
-func (f *Fs) authorizeAccount() error {
+func (f *Fs) authorizeAccount(ctx context.Context) error {
 	f.authMu.Lock()
 	defer f.authMu.Unlock()
 	opts := rest.Opts{
@@ -443,7 +496,7 @@ func (f *Fs) authorizeAccount() error {
 		ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request
 	}
 	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, nil, &f.info)
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &f.info)
 		return f.shouldRetryNoReauth(resp, err)
 	})
 	if err != nil {
@@ -466,10 +519,10 @@ func (f *Fs) hasPermission(permission string) bool {
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
 //
 // This should be returned with returnUploadURL when finished
-func (f *Fs) getUploadURL(bucket string) (upload *api.GetUploadURLResponse, err error) {
+func (f *Fs) getUploadURL(ctx context.Context, bucket string) (upload *api.GetUploadURLResponse, err error) {
 	f.uploadMu.Lock()
 	defer f.uploadMu.Unlock()
-	bucketID, err := f.getBucketID(bucket)
+	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -489,8 +542,8 @@ func (f *Fs) getUploadURL(bucket string) (upload *api.GetUploadURLResponse, err
 		BucketID: bucketID,
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &upload)
-		return f.shouldRetry(resp, err)
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &upload)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get upload URL")
@@ -515,32 +568,25 @@ func (f *Fs) clearUploadURL(bucketID string) {
 	f.uploadMu.Unlock()
 }
 
-// Fill up (or reset) the buffer tokens
-func (f *Fs) fillBufferTokens() {
-	f.bufferTokens = make(chan []byte, fs.Config.Transfers)
-	for i := 0; i < fs.Config.Transfers; i++ {
-		f.bufferTokens <- nil
+// getBuf gets a buffer of f.opt.ChunkSize and an upload token
+//
+// If noBuf is set then it just gets an upload token
+func (f *Fs) getBuf(noBuf bool) (buf []byte) {
+	f.uploadToken.Get()
+	if !noBuf {
+		buf = f.pool.Get()
 	}
-}
-
-// getUploadBlock gets a block from the pool of size chunkSize
-func (f *Fs) getUploadBlock() []byte {
-	buf := <-f.bufferTokens
-	if buf == nil {
-		buf = make([]byte, f.opt.ChunkSize)
-	}
-	// fs.Debugf(f, "Getting upload block %p", buf)
 	return buf
 }
 
-// putUploadBlock returns a block to the pool of size chunkSize
-func (f *Fs) putUploadBlock(buf []byte) {
-	buf = buf[:cap(buf)]
-	if len(buf) != int(f.opt.ChunkSize) {
-		panic("bad blocksize returned to pool")
+// putBuf returns a buffer to the memory pool and an upload token
+//
+// If noBuf is set then it just returns the upload token
+func (f *Fs) putBuf(buf []byte, noBuf bool) {
+	if !noBuf {
+		f.pool.Put(buf)
 	}
-	// fs.Debugf(f, "Returning upload block %p", buf)
-	f.bufferTokens <- buf
+	f.uploadToken.Put()
 }
 
 // Return an Object from a path
@@ -609,7 +655,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	if !recurse {
 		delimiter = "/"
 	}
-	bucketID, err := f.getBucketID(bucket)
+	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
 		return err
 	}
@@ -620,11 +666,11 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	var request = api.ListFileNamesRequest{
 		BucketID:     bucketID,
 		MaxFileCount: chunkSize,
-		Prefix:       directory,
+		Prefix:       f.opt.Enc.FromStandardPath(directory),
 		Delimiter:    delimiter,
 	}
 	if directory != "" {
-		request.StartFileName = directory
+		request.StartFileName = f.opt.Enc.FromStandardPath(directory)
 	}
 	opts := rest.Opts{
 		Method: "POST",
@@ -636,14 +682,15 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	for {
 		var response api.ListFileNamesResponse
 		err := f.pacer.Call(func() (bool, error) {
-			resp, err := f.srv.CallJSON(&opts, &request, &response)
-			return f.shouldRetry(resp, err)
+			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+			return f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return err
 		}
 		for i := range response.Files {
 			file := &response.Files[i]
+			file.Name = f.opt.Enc.ToStandardPath(file.Name)
 			// Finish if file name no longer has prefix
 			if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
 				return nil
@@ -654,7 +701,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			}
 			remote := file.Name[len(prefix):]
 			// Check for directory
-			isDirectory := strings.HasSuffix(remote, "/")
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if isDirectory {
 				remote = remote[:len(remote)-1]
 			}
@@ -727,7 +774,7 @@ func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addB
 
 // listBuckets returns all the buckets to out
 func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
-	err = f.listBucketsToFn(func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
 		d := fs.NewDir(bucket.Name, time.Time{})
 		entries = append(entries, d)
 		return nil
@@ -820,7 +867,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 type listBucketFn func(*api.Bucket) error
 
 // listBucketsToFn lists the buckets to the function supplied
-func (f *Fs) listBucketsToFn(fn listBucketFn) error {
+func (f *Fs) listBucketsToFn(ctx context.Context, fn listBucketFn) error {
 	var account = api.ListBucketsRequest{
 		AccountID: f.info.AccountID,
 		BucketID:  f.info.Allowed.BucketID,
@@ -832,8 +879,8 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error {
 		Path:   "/b2_list_buckets",
 	}
 	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &account, &response)
-		return f.shouldRetry(resp, err)
+		resp, err := f.srv.CallJSON(ctx, &opts, &account, &response)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -844,6 +891,7 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error {
 	f._bucketType = make(map[string]string, 1)
 	for i := range response.Buckets {
 		bucket := &response.Buckets[i]
+		bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
 		f.cache.MarkOK(bucket.Name)
 		f._bucketID[bucket.Name] = bucket.ID
 		f._bucketType[bucket.Name] = bucket.Type
@@ -862,14 +910,14 @@ func (f *Fs) listBucketsToFn(fn listBucketFn) error {
 
 // getbucketType finds the bucketType for the current bucket name
 // can be one of allPublic. allPrivate, or snapshot
-func (f *Fs) getbucketType(bucket string) (bucketType string, err error) {
+func (f *Fs) getbucketType(ctx context.Context, bucket string) (bucketType string, err error) {
 	f.bucketTypeMutex.Lock()
 	bucketType = f._bucketType[bucket]
 	f.bucketTypeMutex.Unlock()
 	if bucketType != "" {
 		return bucketType, nil
 	}
-	err = f.listBucketsToFn(func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
 		// listBucketsToFn reads bucket Types
 		return nil
 	})
@@ -897,14 +945,14 @@ func (f *Fs) clearBucketType(bucket string) {
 }
 
 // getBucketID finds the ID for the current bucket name
-func (f *Fs) getBucketID(bucket string) (bucketID string, err error) {
+func (f *Fs) getBucketID(ctx context.Context, bucket string) (bucketID string, err error) {
 	f.bucketIDMutex.Lock()
 	bucketID = f._bucketID[bucket]
 	f.bucketIDMutex.Unlock()
 	if bucketID != "" {
 		return bucketID, nil
 	}
-	err = f.listBucketsToFn(func(bucket *api.Bucket) error {
+	err = f.listBucketsToFn(ctx, func(bucket *api.Bucket) error {
 		// listBucketsToFn sets IDs
 		return nil
 	})
@@ -965,20 +1013,20 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 		}
 		var request = api.CreateBucketRequest{
 			AccountID: f.info.AccountID,
-			Name:      bucket,
+			Name:      f.opt.Enc.FromStandardName(bucket),
 			Type:      "allPrivate",
 		}
 		var response api.Bucket
 		err := f.pacer.Call(func() (bool, error) {
-			resp, err := f.srv.CallJSON(&opts, &request, &response)
-			return f.shouldRetry(resp, err)
+			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+			return f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			if apiErr, ok := err.(*api.Error); ok {
 				if apiErr.Code == "duplicate_bucket_name" {
 					// Check this is our bucket - buckets are globally unique and this
 					// might be someone elses.
-					_, getBucketErr := f.getBucketID(bucket)
+					_, getBucketErr := f.getBucketID(ctx, bucket)
 					if getBucketErr == nil {
 						// found so it is our bucket
 						return nil
@@ -1009,7 +1057,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 			Method: "POST",
 			Path:   "/b2_delete_bucket",
 		}
-		bucketID, err := f.getBucketID(bucket)
+		bucketID, err := f.getBucketID(ctx, bucket)
 		if err != nil {
 			return err
 		}
@@ -1019,8 +1067,8 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		}
 		var response api.Bucket
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err := f.srv.CallJSON(&opts, &request, &response)
-			return f.shouldRetry(resp, err)
+			resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+			return f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return errors.Wrap(err, "failed to delete bucket")
@@ -1038,8 +1086,8 @@ func (f *Fs) Precision() time.Duration {
 }
 
 // hide hides a file on the remote
-func (f *Fs) hide(bucket, bucketPath string) error {
-	bucketID, err := f.getBucketID(bucket)
+func (f *Fs) hide(ctx context.Context, bucket, bucketPath string) error {
+	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
 		return err
 	}
@@ -1049,12 +1097,12 @@ func (f *Fs) hide(bucket, bucketPath string) error {
 	}
 	var request = api.HideFileRequest{
 		BucketID: bucketID,
-		Name:     bucketPath,
+		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
 	var response api.File
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &response)
-		return f.shouldRetry(resp, err)
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1070,19 +1118,19 @@ func (f *Fs) hide(bucket, bucketPath string) error {
 }
 
 // deleteByID deletes a file version given Name and ID
-func (f *Fs) deleteByID(ID, Name string) error {
+func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_delete_file_version",
 	}
 	var request = api.DeleteFileRequest{
 		ID:   ID,
-		Name: Name,
+		Name: f.opt.Enc.FromStandardPath(Name),
 	}
 	var response api.File
 	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &response)
-		return f.shouldRetry(resp, err)
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return errors.Wrapf(err, "failed to delete %q", Name)
@@ -1132,7 +1180,7 @@ func (f *Fs) purge(ctx context.Context, bucket, directory string, oldOnly bool)
 					continue
 				}
 				tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
-				err = f.deleteByID(object.ID, object.Name)
+				err = f.deleteByID(ctx, object.ID, object.Name)
 				checkErr(err)
 				tr.Done(err)
 			}
@@ -1185,6 +1233,63 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return f.purge(ctx, f.rootBucket, f.rootDirectory, true)
 }
 
+// copy does a server side copy from dstObj <- srcObj
+//
+// If newInfo is nil then the metadata will be copied otherwise it
+// will be replaced with newInfo
+func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *api.File) (err error) {
+	if srcObj.size >= int64(f.opt.CopyCutoff) {
+		if newInfo == nil {
+			newInfo, err = srcObj.getMetaData(ctx)
+			if err != nil {
+				return err
+			}
+		}
+		up, err := f.newLargeUpload(ctx, dstObj, nil, srcObj, f.opt.CopyCutoff, true, newInfo)
+		if err != nil {
+			return err
+		}
+		return up.Upload(ctx)
+	}
+
+	dstBucket, dstPath := dstObj.split()
+	err = f.makeBucket(ctx, dstBucket)
+	if err != nil {
+		return err
+	}
+
+	destBucketID, err := f.getBucketID(ctx, dstBucket)
+	if err != nil {
+		return err
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_copy_file",
+	}
+	var request = api.CopyFileRequest{
+		SourceID:     srcObj.id,
+		Name:         f.opt.Enc.FromStandardPath(dstPath),
+		DestBucketID: destBucketID,
+	}
+	if newInfo == nil {
+		request.MetadataDirective = "COPY"
+	} else {
+		request.MetadataDirective = "REPLACE"
+		request.ContentType = newInfo.ContentType
+		request.Info = newInfo.Info
+	}
+	var response api.FileInfo
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	return dstObj.decodeMetaDataFileInfo(&response)
+}
+
 // Copy src to this remote using server side copy operations.
 //
 // This is stored with the remote path given
@@ -1195,47 +1300,21 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 //
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	dstBucket, dstPath := f.split(remote)
-	err := f.makeBucket(ctx, dstBucket)
-	if err != nil {
-		return nil, err
-	}
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	destBucketID, err := f.getBucketID(dstBucket)
-	if err != nil {
-		return nil, err
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          srcObj.id,
-		Name:              dstPath,
-		MetadataDirective: "COPY",
-		DestBucketID:      destBucketID,
-	}
-	var response api.FileInfo
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &response)
-		return f.shouldRetry(resp, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	o := &Object{
+	// Temporary Object under construction
+	dstObj := &Object{
 		fs:     f,
 		remote: remote,
 	}
-	err = o.decodeMetaDataFileInfo(&response)
+	err := f.copy(ctx, dstObj, srcObj, nil)
 	if err != nil {
 		return nil, err
 	}
-	return o, nil
+	return dstObj, nil
 }
 
 // Hashes returns the supported hash sets.
@@ -1245,7 +1324,7 @@ func (f *Fs) Hashes() hash.Set {
 
 // getDownloadAuthorization returns authorization token for downloading
 // without account.
-func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization string, err error) {
+func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string) (authorization string, err error) {
 	validDurationInSeconds := time.Duration(f.opt.DownloadAuthorizationDuration).Nanoseconds() / 1e9
 	if validDurationInSeconds <= 0 || validDurationInSeconds > 604800 {
 		return "", errors.New("--b2-download-auth-duration must be between 1 sec and 1 week")
@@ -1253,7 +1332,7 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri
 	if !f.hasPermission("shareFiles") {
 		return "", errors.New("sharing a file link requires the shareFiles permission")
 	}
-	bucketID, err := f.getBucketID(bucket)
+	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
 		return "", err
 	}
@@ -1263,13 +1342,13 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri
 	}
 	var request = api.GetDownloadAuthorizationRequest{
 		BucketID:               bucketID,
-		FileNamePrefix:         path.Join(f.root, remote),
+		FileNamePrefix:         f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
 		ValidDurationInSeconds: validDurationInSeconds,
 	}
 	var response api.GetDownloadAuthorizationResponse
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &response)
-		return f.shouldRetry(resp, err)
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return "", errors.Wrap(err, "failed to get download authorization")
@@ -1278,7 +1357,7 @@ func (f *Fs) getDownloadAuthorization(bucket, remote string) (authorization stri
 }
 
 // PublicLink returns a link for downloading without account
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	bucket, bucketPath := f.split(remote)
 	var RootURL string
 	if f.opt.DownloadURL == "" {
@@ -1301,12 +1380,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 	}
 	absPath := "/" + bucketPath
 	link = RootURL + "/file/" + urlEncode(bucket) + absPath
-	bucketType, err := f.getbucketType(bucket)
+	bucketType, err := f.getbucketType(ctx, bucket)
 	if err != nil {
 		return "", err
 	}
 	if bucketType == "allPrivate" || bucketType == "snapshot" {
-		AuthorizationToken, err := f.getDownloadAuthorization(bucket, remote)
+		AuthorizationToken, err := f.getDownloadAuthorization(ctx, bucket, remote)
 		if err != nil {
 			return "", err
 		}
@@ -1355,6 +1434,21 @@ func (o *Object) Size() int64 {
 	return o.size
 }
 
+// Clean the SHA1
+//
+// Make sure it is lower case
+//
+// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
+// Some tools (eg Cyberduck) use this
+func cleanSHA1(sha1 string) (out string) {
+	out = strings.ToLower(sha1)
+	const unverified = "unverified:"
+	if strings.HasPrefix(out, unverified) {
+		out = out[len(unverified):]
+	}
+	return out
+}
+
 // decodeMetaDataRaw sets the metadata from the data passed in
 //
 // Sets
@@ -1370,6 +1464,7 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if o.sha1 == "" || o.sha1 == "none" {
 		o.sha1 = Info[sha1Key]
 	}
+	o.sha1 = cleanSHA1(o.sha1)
 	o.size = Size
 	// Use the UploadTimestamp if can't get file info
 	o.modTime = time.Time(UploadTimestamp)
@@ -1453,7 +1548,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 // timeString returns modTime as the number of milliseconds
 // elapsed since January 1, 1970 UTC as a decimal string.
 func timeString(modTime time.Time) string {
-	return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
+	return strconv.FormatInt(modTime.UnixNano()/1e6, 10)
 }
 
 // parseTimeString converts a decimal string number of milliseconds
@@ -1468,7 +1563,7 @@ func (o *Object) parseTimeString(timeString string) (err error) {
 		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
 		return nil
 	}
-	o.modTime = time.Unix(unixMilliseconds/1E3, (unixMilliseconds%1E3)*1E6).UTC()
+	o.modTime = time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC()
 	return nil
 }
 
@@ -1490,28 +1585,10 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if err != nil {
 		return err
 	}
-	_, bucketPath := o.split()
 	info.Info[timeKey] = timeString(modTime)
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          o.id,
-		Name:              bucketPath, // copy to same name
-		MetadataDirective: "REPLACE",
-		ContentType:       info.ContentType,
-		Info:              info.Info,
-	}
-	var response api.FileInfo
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(&opts, &request, &response)
-		return o.fs.shouldRetry(resp, err)
-	})
-	if err != nil {
-		return err
-	}
-	return o.decodeMetaDataFileInfo(&response)
+
+	// Copy to the same name, overwriting the metadata only
+	return o.fs.copy(ctx, o, o, info)
 }
 
 // Storable returns if this object is storable
@@ -1600,12 +1677,12 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
 	} else {
 		bucket, bucketPath := o.split()
-		opts.Path += "/file/" + urlEncode(bucket) + "/" + urlEncode(bucketPath)
+		opts.Path += "/file/" + urlEncode(o.fs.opt.Enc.FromStandardName(bucket)) + "/" + urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath))
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
-		return o.fs.shouldRetry(resp, err)
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		return o.fs.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open for download")
@@ -1627,6 +1704,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			o.sha1 = resp.Header.Get(sha1InfoHeader)
 			fs.Debugf(o, "Reading sha1 from info - %q", o.sha1)
 		}
+		o.sha1 = cleanSHA1(o.sha1)
 	}
 	// Don't check length or hash on partial content
 	if resp.StatusCode == http.StatusPartialContent {
@@ -1686,7 +1764,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	if size == -1 {
 		// Check if the file is large enough for a chunked upload (needs to be at least two chunks)
-		buf := o.fs.getUploadBlock()
+		buf := o.fs.getBuf(false)
+
 		n, err := io.ReadFull(in, buf)
 		if err == nil {
 			bufReader := bufio.NewReader(in)
@@ -1696,26 +1775,28 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		if err == nil {
 			fs.Debugf(o, "File is big enough for chunked streaming")
-			up, err := o.fs.newLargeUpload(ctx, o, in, src)
+			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 			if err != nil {
-				o.fs.putUploadBlock(buf)
+				o.fs.putBuf(buf, false)
 				return err
 			}
-			return up.Stream(buf)
+			// NB Stream returns the buffer and token
+			return up.Stream(ctx, buf)
 		} else if err == io.EOF || err == io.ErrUnexpectedEOF {
 			fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
-			defer o.fs.putUploadBlock(buf)
+			defer o.fs.putBuf(buf, false)
 			size = int64(n)
 			in = bytes.NewReader(buf[:n])
 		} else {
+			o.fs.putBuf(buf, false)
 			return err
 		}
 	} else if size > int64(o.fs.opt.UploadCutoff) {
-		up, err := o.fs.newLargeUpload(ctx, o, in, src)
+		up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 		if err != nil {
 			return err
 		}
-		return up.Upload()
+		return up.Upload(ctx)
 	}
 
 	modTime := src.ModTime(ctx)
@@ -1729,7 +1810,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 
 	// Get upload URL
-	upload, err := o.fs.getUploadURL(bucket)
+	upload, err := o.fs.getUploadURL(ctx, bucket)
 	if err != nil {
 		return err
 	}
@@ -1795,9 +1876,10 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Method:  "POST",
 		RootURL: upload.UploadURL,
 		Body:    in,
+		Options: options,
 		ExtraHeaders: map[string]string{
 			"Authorization":  upload.AuthorizationToken,
-			"X-Bz-File-Name": urlEncode(bucketPath),
+			"X-Bz-File-Name": urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath)),
 			"Content-Type":   fs.MimeType(ctx, src),
 			sha1Header:       calculatedSha1,
 			timeHeader:       timeString(modTime),
@@ -1807,8 +1889,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var response api.FileInfo
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(&opts, nil, &response)
-		retry, err := o.fs.shouldRetry(resp, err)
+		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &response)
+		retry, err := o.fs.shouldRetry(ctx, resp, err)
 		// On retryable error clear UploadURL
 		if retry {
 			fs.Debugf(o, "Clearing upload URL because of error: %v", err)
@@ -1829,9 +1911,9 @@ func (o *Object) Remove(ctx context.Context) error {
 		return errNotWithVersions
 	}
 	if o.fs.opt.HardDelete {
-		return o.fs.deleteByID(o.id, bucketPath)
+		return o.fs.deleteByID(ctx, o.id, bucketPath)
 	}
-	return o.fs.hide(bucket, bucketPath)
+	return o.fs.hide(ctx, bucket, bucketPath)
 }
 
 // MimeType of an Object if known, "" otherwise

backend/b2/upload.go

@@ -20,7 +20,9 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
 )
 
 type hashAppendingReader struct {
@@ -68,20 +70,26 @@ func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader {
 
 // largeUpload is used to control the upload of large files which need chunking
 type largeUpload struct {
-	f        *Fs                             // parent Fs
-	o        *Object                         // object being uploaded
-	in       io.Reader                       // read the data from here
-	wrap     accounting.WrapFn               // account parts being transferred
-	id       string                          // ID of the file being uploaded
-	size     int64                           // total size
-	parts    int64                           // calculated number of parts, if known
-	sha1s    []string                        // slice of SHA1s for each part
-	uploadMu sync.Mutex                      // lock for upload variable
-	uploads  []*api.GetUploadPartURLResponse // result of get upload URL calls
+	f         *Fs                             // parent Fs
+	o         *Object                         // object being uploaded
+	doCopy    bool                            // doing copy rather than upload
+	what      string                          // text name of operation for logs
+	in        io.Reader                       // read the data from here
+	wrap      accounting.WrapFn               // account parts being transferred
+	id        string                          // ID of the file being uploaded
+	size      int64                           // total size
+	parts     int64                           // calculated number of parts, if known
+	sha1s     []string                        // slice of SHA1s for each part
+	uploadMu  sync.Mutex                      // lock for upload variable
+	uploads   []*api.GetUploadPartURLResponse // result of get upload URL calls
+	chunkSize int64                           // chunk size to use
+	src       *Object                         // if copying, object we are reading from
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) {
+//
+// If newInfo is set then metadata from that will be used instead of reading it from src
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	remote := o.remote
 	size := src.Size()
 	parts := int64(0)
@@ -89,8 +97,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
-		parts = size / int64(o.fs.opt.ChunkSize)
-		if size%int64(o.fs.opt.ChunkSize) != 0 {
+		parts = size / int64(chunkSize)
+		if size%int64(chunkSize) != 0 {
 			parts++
 		}
 		if parts > maxParts {
@@ -99,50 +107,61 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		sha1SliceSize = parts
 	}
 
-	modTime := src.ModTime(ctx)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_start_large_file",
 	}
 	bucket, bucketPath := o.split()
-	bucketID, err := f.getBucketID(bucket)
+	bucketID, err := f.getBucketID(ctx, bucket)
 	if err != nil {
 		return nil, err
 	}
 	var request = api.StartLargeFileRequest{
-		BucketID:    bucketID,
-		Name:        bucketPath,
-		ContentType: fs.MimeType(ctx, src),
-		Info: map[string]string{
-			timeKey: timeString(modTime),
-		},
+		BucketID: bucketID,
+		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
-	// Set the SHA1 if known
-	if !o.fs.opt.DisableCheckSum {
-		if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
-			request.Info[sha1Key] = calculatedSha1
+	if newInfo == nil {
+		modTime := src.ModTime(ctx)
+		request.ContentType = fs.MimeType(ctx, src)
+		request.Info = map[string]string{
+			timeKey: timeString(modTime),
 		}
+		// Set the SHA1 if known
+		if !o.fs.opt.DisableCheckSum || doCopy {
+			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
+				request.Info[sha1Key] = calculatedSha1
+			}
+		}
+	} else {
+		request.ContentType = newInfo.ContentType
+		request.Info = newInfo.Info
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, &request, &response)
-		return f.shouldRetry(resp, err)
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return nil, err
 	}
+	up = &largeUpload{
+		f:         f,
+		o:         o,
+		doCopy:    doCopy,
+		what:      "upload",
+		id:        response.ID,
+		size:      size,
+		parts:     parts,
+		sha1s:     make([]string, sha1SliceSize),
+		chunkSize: int64(chunkSize),
+	}
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
-	in, wrap := accounting.UnWrap(in)
-	up = &largeUpload{
-		f:     f,
-		o:     o,
-		in:    in,
-		wrap:  wrap,
-		id:    response.ID,
-		size:  size,
-		parts: parts,
-		sha1s: make([]string, sha1SliceSize),
+	if doCopy {
+		up.what = "copy"
+		up.src = src.(*Object)
+	} else {
+		up.in, up.wrap = accounting.UnWrap(in)
 	}
 	return up, nil
 }
@@ -150,7 +169,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 // getUploadURL returns the upload info with the UploadURL and the AuthorizationToken
 //
 // This should be returned with returnUploadURL when finished
-func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err error) {
+func (up *largeUpload) getUploadURL(ctx context.Context) (upload *api.GetUploadPartURLResponse, err error) {
 	up.uploadMu.Lock()
 	defer up.uploadMu.Unlock()
 	if len(up.uploads) == 0 {
@@ -162,8 +181,8 @@ func (up *largeUpload) getUploadURL() (upload *api.GetUploadPartURLResponse, err
 			ID: up.id,
 		}
 		err := up.f.pacer.Call(func() (bool, error) {
-			resp, err := up.f.srv.CallJSON(&opts, &request, &upload)
-			return up.f.shouldRetry(resp, err)
+			resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &upload)
+			return up.f.shouldRetry(ctx, resp, err)
 		})
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to get upload URL")
@@ -184,20 +203,13 @@ func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) {
 	up.uploadMu.Unlock()
 }
 
-// clearUploadURL clears the current UploadURL and the AuthorizationToken
-func (up *largeUpload) clearUploadURL() {
-	up.uploadMu.Lock()
-	up.uploads = nil
-	up.uploadMu.Unlock()
-}
-
 // Transfer a chunk
-func (up *largeUpload) transferChunk(part int64, body []byte) error {
+func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error {
 	err := up.f.pacer.Call(func() (bool, error) {
 		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
 
 		// Get upload URL
-		upload, err := up.getUploadURL()
+		upload, err := up.getUploadURL(ctx)
 		if err != nil {
 			return false, err
 		}
@@ -241,8 +253,8 @@ func (up *largeUpload) transferChunk(part int64, body []byte) error {
 
 		var response api.UploadPartResponse
 
-		resp, err := up.f.srv.CallJSON(&opts, nil, &response)
-		retry, err := up.f.shouldRetry(resp, err)
+		resp, err := up.f.srv.CallJSON(ctx, &opts, nil, &response)
+		retry, err := up.f.shouldRetry(ctx, resp, err)
 		if err != nil {
 			fs.Debugf(up.o, "Error sending chunk %d (retry=%v): %v: %#v", part, retry, err, err)
 		}
@@ -263,9 +275,41 @@ func (up *largeUpload) transferChunk(part int64, body []byte) error {
 	return err
 }
 
+// Copy a chunk
+func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
+	err := up.f.pacer.Call(func() (bool, error) {
+		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
+		opts := rest.Opts{
+			Method: "POST",
+			Path:   "/b2_copy_part",
+		}
+		offset := (part - 1) * up.chunkSize // where we are in the source file
+		var request = api.CopyPartRequest{
+			SourceID:    up.src.id,
+			LargeFileID: up.id,
+			PartNumber:  part,
+			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
+		}
+		var response api.UploadPartResponse
+		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
+		retry, err := up.f.shouldRetry(ctx, resp, err)
+		if err != nil {
+			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
+		}
+		up.sha1s[part-1] = response.SHA1
+		return retry, err
+	})
+	if err != nil {
+		fs.Debugf(up.o, "Error copying chunk %d: %v", part, err)
+	} else {
+		fs.Debugf(up.o, "Done copying chunk %d", part)
+	}
+	return err
+}
+
 // finish closes off the large upload
-func (up *largeUpload) finish() error {
-	fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
+func (up *largeUpload) finish(ctx context.Context) error {
+	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_finish_large_file",
@@ -276,8 +320,8 @@ func (up *largeUpload) finish() error {
 	}
 	var response api.FileInfo
 	err := up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(&opts, &request, &response)
-		return up.f.shouldRetry(resp, err)
+		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
+		return up.f.shouldRetry(ctx, resp, err)
 	})
 	if err != nil {
 		return err
@@ -286,7 +330,8 @@ func (up *largeUpload) finish() error {
 }
 
 // cancel aborts the large upload
-func (up *largeUpload) cancel() error {
+func (up *largeUpload) cancel(ctx context.Context) error {
+	fs.Debugf(up.o, "Cancelling large file %s", up.what)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_cancel_large_file",
@@ -296,142 +341,142 @@ func (up *largeUpload) cancel() error {
 	}
 	var response api.CancelLargeFileResponse
 	err := up.f.pacer.Call(func() (bool, error) {
-		resp, err := up.f.srv.CallJSON(&opts, &request, &response)
-		return up.f.shouldRetry(resp, err)
+		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
+		return up.f.shouldRetry(ctx, resp, err)
 	})
-	return err
-}
-
-func (up *largeUpload) managedTransferChunk(wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
-	wg.Add(1)
-	go func(part int64, buf []byte) {
-		defer wg.Done()
-		defer up.f.putUploadBlock(buf)
-		err := up.transferChunk(part, buf)
-		if err != nil {
-			select {
-			case errs <- err:
-			default:
-			}
-		}
-	}(part, buf)
-}
-
-func (up *largeUpload) finishOrCancelOnError(err error, errs chan error) error {
-	if err == nil {
-		select {
-		case err = <-errs:
-		default:
-		}
-	}
 	if err != nil {
-		fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
-		cancelErr := up.cancel()
-		if cancelErr != nil {
-			fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
-		}
-		return err
+		fs.Errorf(up.o, "Failed to cancel large file %s: %v", up.what, err)
 	}
-	return up.finish()
+	return err
 }
 
 // Stream uploads the chunks from the input, starting with a required initial
 // chunk. Assumes the file size is unknown and will upload until the input
 // reaches EOF.
-func (up *largeUpload) Stream(initialUploadBlock []byte) (err error) {
+//
+// Note that initialUploadBlock must be returned to f.putBuf()
+func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
-	errs := make(chan error, 1)
-	hasMoreParts := true
-	var wg sync.WaitGroup
-
-	// Transfer initial chunk
+	var (
+		g, gCtx      = errgroup.WithContext(ctx)
+		hasMoreParts = true
+	)
 	up.size = int64(len(initialUploadBlock))
-	up.managedTransferChunk(&wg, errs, 1, initialUploadBlock)
+	g.Go(func() error {
+		for part := int64(1); hasMoreParts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			var buf []byte
+			if part == 1 {
+				buf = initialUploadBlock
+			} else {
+				buf = up.f.getBuf(false)
+			}
 
-outer:
-	for part := int64(2); hasMoreParts; part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+			// Fail fast, in case an errgroup managed function returns an error
+			// gCtx is cancelled. There is no point in uploading all the other parts.
+			if gCtx.Err() != nil {
+				up.f.putBuf(buf, false)
+				return nil
+			}
+
+			// Read the chunk
+			var n int
+			if part == 1 {
+				n = len(buf)
+			} else {
+				n, err = io.ReadFull(up.in, buf)
+				if err == io.ErrUnexpectedEOF {
+					fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
+					buf = buf[:n]
+					hasMoreParts = false
+				} else if err == io.EOF {
+					fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
+					up.f.putBuf(buf, false)
+					return nil
+				} else if err != nil {
+					// other kinds of errors indicate failure
+					up.f.putBuf(buf, false)
+					return err
+				}
+			}
+
+			// Keep stats up to date
+			up.parts = part
+			up.size += int64(n)
+			if part > maxParts {
+				up.f.putBuf(buf, false)
+				return errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+			}
+
+			part := part // for the closure
+			g.Go(func() (err error) {
+				defer up.f.putBuf(buf, false)
+				return up.transferChunk(gCtx, part, buf)
+			})
 		}
-
-		// Get a block of memory
-		buf := up.f.getUploadBlock()
-
-		// Read the chunk
-		var n int
-		n, err = io.ReadFull(up.in, buf)
-		if err == io.ErrUnexpectedEOF {
-			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
-			buf = buf[:n]
-			hasMoreParts = false
-			err = nil
-		} else if err == io.EOF {
-			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
-			up.f.putUploadBlock(buf)
-			err = nil
-			break outer
-		} else if err != nil {
-			// other kinds of errors indicate failure
-			up.f.putUploadBlock(buf)
-			break outer
-		}
-
-		// Keep stats up to date
-		up.parts = part
-		up.size += int64(n)
-		if part > maxParts {
-			err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
-			break outer
-		}
-
-		// Transfer the chunk
-		up.managedTransferChunk(&wg, errs, part, buf)
+		return nil
+	})
+	err = g.Wait()
+	if err != nil {
+		return err
 	}
-	wg.Wait()
 	up.sha1s = up.sha1s[:up.parts]
-
-	return up.finishOrCancelOnError(err, errs)
+	return up.finish(ctx)
 }
 
 // Upload uploads the chunks from the input
-func (up *largeUpload) Upload() error {
-	fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
-	remaining := up.size
-	errs := make(chan error, 1)
-	var wg sync.WaitGroup
-	var err error
-outer:
-	for part := int64(1); part <= up.parts; part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+func (up *largeUpload) Upload(ctx context.Context) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
+	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
+	var (
+		g, gCtx   = errgroup.WithContext(ctx)
+		remaining = up.size
+	)
+	g.Go(func() error {
+		for part := int64(1); part <= up.parts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			buf := up.f.getBuf(up.doCopy)
+
+			// Fail fast, in case an errgroup managed function returns an error
+			// gCtx is cancelled. There is no point in uploading all the other parts.
+			if gCtx.Err() != nil {
+				up.f.putBuf(buf, up.doCopy)
+				return nil
+			}
+
+			reqSize := remaining
+			if reqSize >= up.chunkSize {
+				reqSize = up.chunkSize
+			}
+
+			if !up.doCopy {
+				// Read the chunk
+				buf = buf[:reqSize]
+				_, err = io.ReadFull(up.in, buf)
+				if err != nil {
+					up.f.putBuf(buf, up.doCopy)
+					return err
+				}
+			}
+
+			part := part // for the closure
+			g.Go(func() (err error) {
+				defer up.f.putBuf(buf, up.doCopy)
+				if !up.doCopy {
+					err = up.transferChunk(gCtx, part, buf)
+				} else {
+					err = up.copyChunk(gCtx, part, reqSize)
+				}
+				return err
+			})
+			remaining -= reqSize
 		}
-
-		reqSize := remaining
-		if reqSize >= int64(up.f.opt.ChunkSize) {
-			reqSize = int64(up.f.opt.ChunkSize)
-		}
-
-		// Get a block of memory
-		buf := up.f.getUploadBlock()[:reqSize]
-
-		// Read the chunk
-		_, err = io.ReadFull(up.in, buf)
-		if err != nil {
-			up.f.putUploadBlock(buf)
-			break outer
-		}
-
-		// Transfer the chunk
-		up.managedTransferChunk(&wg, errs, part, buf)
-		remaining -= reqSize
+		return nil
+	})
+	err = g.Wait()
+	if err != nil {
+		return err
 	}
-	wg.Wait()
-
-	return up.finishOrCancelOnError(err, errs)
+	return up.finish(ctx)
 }

backend/box/api/types.go

@@ -202,3 +202,43 @@ type CommitUpload struct {
 		ContentModifiedAt Time `json:"content_modified_at"`
 	} `json:"attributes"`
 }
+
+// ConfigJSON defines the shape of a box config.json
+type ConfigJSON struct {
+	BoxAppSettings AppSettings `json:"boxAppSettings"`
+	EnterpriseID   string      `json:"enterpriseID"`
+}
+
+// AppSettings defines the shape of the boxAppSettings within box config.json
+type AppSettings struct {
+	ClientID     string  `json:"clientID"`
+	ClientSecret string  `json:"clientSecret"`
+	AppAuth      AppAuth `json:"appAuth"`
+}
+
+// AppAuth defines the shape of the appAuth within boxAppSettings in config.json
+type AppAuth struct {
+	PublicKeyID string `json:"publicKeyID"`
+	PrivateKey  string `json:"privateKey"`
+	Passphrase  string `json:"passphrase"`
+}
+
+// User is returned from /users/me
+type User struct {
+	Type          string    `json:"type"`
+	ID            string    `json:"id"`
+	Name          string    `json:"name"`
+	Login         string    `json:"login"`
+	CreatedAt     time.Time `json:"created_at"`
+	ModifiedAt    time.Time `json:"modified_at"`
+	Language      string    `json:"language"`
+	Timezone      string    `json:"timezone"`
+	SpaceAmount   int64     `json:"space_amount"`
+	SpaceUsed     int64     `json:"space_used"`
+	MaxUploadSize int64     `json:"max_upload_size"`
+	Status        string    `json:"status"`
+	JobTitle      string    `json:"job_title"`
+	Phone         string    `json:"phone"`
+	Address       string    `json:"address"`
+	AvatarURL     string    `json:"avatar_url"`
+}

backend/box/box.go

@@ -11,8 +11,12 @@ package box
 
 import (
 	"context"
+	"crypto/rsa"
+	"encoding/json"
+	"encoding/pem"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -21,6 +25,12 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
+	"github.com/rclone/rclone/lib/jwtutil"
+
+	"github.com/youmark/pkcs8"
+
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
@@ -29,12 +39,14 @@ import (
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/jws"
 )
 
 const (
@@ -42,13 +54,13 @@ const (
 	rcloneEncryptedClientSecret = "sYbJYm99WB8jzeaLPU0OPDMJKIkZvD2qOn3SyEMfiJr03RdtDt3xcZEIudRhbIDL"
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
-	decayConstant               = 2   // bigger for slower decay, exponential
-	rootID                      = "0" // ID of root folder is always this
+	decayConstant               = 2 // bigger for slower decay, exponential
 	rootURL                     = "https://api.box.com/2.0"
 	uploadURL                   = "https://upload.box.com/api/2.0"
 	listChunks                  = 1000     // chunk size to read directory listings
 	minUploadCutoff             = 50000000 // upload cutoff can be no lower than this
 	defaultUploadCutoff         = 50 * 1024 * 1024
+	tokenURL                    = "https://api.box.com/oauth2/token"
 )
 
 // Globals
@@ -73,9 +85,19 @@ func init() {
 		Description: "Box",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("box", name, m, oauthConfig)
-			if err != nil {
-				log.Fatalf("Failed to configure token: %v", err)
+			jsonFile, ok := m.Get("box_config_file")
+			boxSubType, boxSubTypeOk := m.Get("box_sub_type")
+			var err error
+			if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
+				err = refreshJWTToken(jsonFile, boxSubType, name, m)
+				if err != nil {
+					log.Fatalf("Failed to configure token with jwt authentication: %v", err)
+				}
+			} else {
+				err = oauthutil.Config("box", name, m, oauthConfig, nil)
+				if err != nil {
+					log.Fatalf("Failed to configure token with oauth authentication: %v", err)
+				}
 			}
 		},
 		Options: []fs.Option{{
@@ -84,6 +106,24 @@ func init() {
 		}, {
 			Name: config.ConfigClientSecret,
 			Help: "Box App Client Secret\nLeave blank normally.",
+		}, {
+			Name:     "root_folder_id",
+			Help:     "Fill in for rclone to use a non root folder as its starting point.",
+			Default:  "0",
+			Advanced: true,
+		}, {
+			Name: "box_config_file",
+			Help: "Box App config.json location\nLeave blank normally." + env.ShellExpandHelp,
+		}, {
+			Name:    "box_sub_type",
+			Default: "user",
+			Examples: []fs.OptionExample{{
+				Value: "user",
+				Help:  "Rclone should act on behalf of a user",
+			}, {
+				Value: "enterprise",
+				Help:  "Rclone should act on behalf of a service account",
+			}},
 		}, {
 			Name:     "upload_cutoff",
 			Help:     "Cutoff for switching to multipart upload (>= 50MB).",
@@ -94,14 +134,119 @@ func init() {
 			Help:     "Max number of times to try committing a multipart file.",
 			Default:  100,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// From https://developer.box.com/docs/error-codes#section-400-bad-request :
+			// > Box only supports file or folder names that are 255 characters or less.
+			// > File names containing non-printable ascii, "/" or "\", names with leading
+			// > or trailing spaces, and the special names “.” and “..” are also unsupported.
+			//
+			// Testing revealed names with leading spaces work fine.
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
+func refreshJWTToken(jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
+	jsonFile = env.ShellExpand(jsonFile)
+	boxConfig, err := getBoxConfig(jsonFile)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+	}
+	privateKey, err := getDecryptedPrivateKey(boxConfig)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+	}
+	claims, err := getClaims(boxConfig, boxSubType)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+	}
+	signingHeaders := getSigningHeaders(boxConfig)
+	queryParams := getQueryParams(boxConfig)
+	client := fshttp.NewClient(fs.Config)
+	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
+	return err
+}
+
+func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
+	file, err := ioutil.ReadFile(configFile)
+	if err != nil {
+		return nil, errors.Wrap(err, "box: failed to read Box config")
+	}
+	err = json.Unmarshal(file, &boxConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "box: failed to parse Box config")
+	}
+	return boxConfig, nil
+}
+
+func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimSet, err error) {
+	val, err := jwtutil.RandomHex(20)
+	if err != nil {
+		return nil, errors.Wrap(err, "box: failed to generate random string for jti")
+	}
+
+	claims = &jws.ClaimSet{
+		Iss: boxConfig.BoxAppSettings.ClientID,
+		Sub: boxConfig.EnterpriseID,
+		Aud: tokenURL,
+		Exp: time.Now().Add(time.Second * 45).Unix(),
+		PrivateClaims: map[string]interface{}{
+			"box_sub_type": boxSubType,
+			"aud":          tokenURL,
+			"jti":          val,
+		},
+	}
+
+	return claims, nil
+}
+
+func getSigningHeaders(boxConfig *api.ConfigJSON) *jws.Header {
+	signingHeaders := &jws.Header{
+		Algorithm: "RS256",
+		Typ:       "JWT",
+		KeyID:     boxConfig.BoxAppSettings.AppAuth.PublicKeyID,
+	}
+
+	return signingHeaders
+}
+
+func getQueryParams(boxConfig *api.ConfigJSON) map[string]string {
+	queryParams := map[string]string{
+		"client_id":     boxConfig.BoxAppSettings.ClientID,
+		"client_secret": boxConfig.BoxAppSettings.ClientSecret,
+	}
+
+	return queryParams
+}
+
+func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err error) {
+
+	block, rest := pem.Decode([]byte(boxConfig.BoxAppSettings.AppAuth.PrivateKey))
+	if len(rest) > 0 {
+		return nil, errors.Wrap(err, "box: extra data included in private key")
+	}
+
+	rsaKey, err := pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(boxConfig.BoxAppSettings.AppAuth.Passphrase))
+	if err != nil {
+		return nil, errors.Wrap(err, "box: failed to decrypt private key")
+	}
+
+	return rsaKey.(*rsa.PrivateKey), nil
+}
+
 // Options defines the configuration for this backend
 type Options struct {
-	UploadCutoff  fs.SizeSuffix `config:"upload_cutoff"`
-	CommitRetries int           `config:"commit_retries"`
+	UploadCutoff  fs.SizeSuffix        `config:"upload_cutoff"`
+	CommitRetries int                  `config:"commit_retries"`
+	Enc           encoder.MultiEncoder `config:"encoding"`
+	RootFolderID  string               `config:"root_folder_id"`
 }
 
 // Fs represents a remote box
@@ -153,7 +298,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an box 'url'
+// parsePath parses a box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -181,22 +326,10 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// substitute reserved characters for box
-func replaceReservedChars(x string) string {
-	// Backslash for FULLWIDTH REVERSE SOLIDUS
-	return strings.Replace(x, "\\", "＼", -1)
-}
-
-// restore reserved characters for box
-func restoreReservedChars(x string) string {
-	// FULLWIDTH REVERSE SOLIDUS for Backslash
-	return strings.Replace(x, "＼", "\\", -1)
-}
-
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -204,7 +337,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		return nil, err
 	}
 
-	found, err := f.listAll(directoryID, false, true, func(item *api.Item) bool {
+	found, err := f.listAll(ctx, directoryID, false, true, func(item *api.Item) bool {
 		if item.Name == leaf {
 			info = item
 			return true
@@ -271,13 +404,27 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// Renew the token in the background
-	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.readMetaDataForPath(ctx, "")
-		return err
-	})
+	jsonFile, ok := m.Get("box_config_file")
+	boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 
-	// Get rootID
+	// If using box config.json and JWT, renewing should just refresh the token and
+	// should do so whether there are uploads pending or not.
+	if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+			err := refreshJWTToken(jsonFile, boxSubType, name, m)
+			return err
+		})
+		f.tokenRenewer.Start()
+	} else {
+		// Renew the token in the background
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+			_, err := f.readMetaDataForPath(ctx, "")
+			return err
+		})
+	}
+
+	// Get rootFolderID
+	rootID := f.opt.RootFolderID
 	f.dirCache = dircache.New(root, rootID, f)
 
 	// Find the current root
@@ -352,7 +499,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // FindLeaf finds a directory of name leaf in the folder with ID pathID
 func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut string, found bool, err error) {
 	// Find the leaf in pathID
-	found, err = f.listAll(pathID, true, false, func(item *api.Item) bool {
+	found, err = f.listAll(ctx, pathID, true, false, func(item *api.Item) bool {
 		if item.Name == leaf {
 			pathIDOut = item.ID
 			return true
@@ -380,13 +527,13 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Parameters: fieldsValue(),
 	}
 	mkdir := api.CreateFolder{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
 			ID: pathID,
 		},
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &mkdir, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -408,7 +555,7 @@ type listAllFn func(*api.Item) bool
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
-func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	opts := rest.Opts{
 		Method:     "GET",
 		Path:       "/folders/" + dirID + "/items",
@@ -423,7 +570,7 @@ OUTER:
 		var result api.FolderItems
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(&opts, nil, &result)
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 			return shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -446,7 +593,7 @@ OUTER:
 			if item.ItemStatus != api.ItemStatusActive {
 				continue
 			}
-			item.Name = restoreReservedChars(item.Name)
+			item.Name = f.opt.Enc.ToStandardName(item.Name)
 			if fn(item) {
 				found = true
 				break OUTER
@@ -470,16 +617,12 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
 	}
 	var iErr error
-	_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool {
+	_, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
 		remote := path.Join(dir, info.Name)
 		if info.Type == api.ItemTypeFolder {
 			// cache the directory ID for later lookups
@@ -514,7 +657,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -570,25 +713,19 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
 // deleteObject removes an object by ID
-func (f *Fs) deleteObject(id string) error {
+func (f *Fs) deleteObject(ctx context.Context, id string) error {
 	opts := rest.Opts{
 		Method:     "DELETE",
 		Path:       "/files/" + id,
 		NoResponse: true,
 	}
 	return f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.Call(&opts)
+		resp, err := f.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 }
@@ -601,10 +738,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -619,7 +752,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	opts.Parameters.Set("recursive", strconv.FormatBool(!check))
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.Call(&opts)
+		resp, err = f.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -682,9 +815,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Path:       "/files/" + srcObj.id + "/copy",
 		Parameters: fieldsValue(),
 	}
-	replacedLeaf := replaceReservedChars(leaf)
 	copyFile := api.CopyFile{
-		Name: replacedLeaf,
+		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
 			ID: directoryID,
 		},
@@ -692,7 +824,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var resp *http.Response
 	var info *api.Item
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &copyFile, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &copyFile, &info)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -715,7 +847,7 @@ func (f *Fs) Purge(ctx context.Context) error {
 }
 
 // move a file or folder
-func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err error) {
+func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (info *api.Item, err error) {
 	// Move the object
 	opts := rest.Opts{
 		Method:     "PUT",
@@ -723,14 +855,14 @@ func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err e
 		Parameters: fieldsValue(),
 	}
 	move := api.UpdateFileMove{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
 			ID: directoryID,
 		},
 	}
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &move, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -739,6 +871,30 @@ func (f *Fs) move(endpoint, id, leaf, directoryID string) (info *api.Item, err e
 	return info, nil
 }
 
+// About gets quota information
+func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/users/me",
+	}
+	var user api.User
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read user info")
+	}
+	// FIXME max upload size would be useful to use in Update
+	usage = &fs.Usage{
+		Used:  fs.NewUsageValue(user.SpaceUsed),                    // bytes in use
+		Total: fs.NewUsageValue(user.SpaceAmount),                  // bytes total
+		Free:  fs.NewUsageValue(user.SpaceAmount - user.SpaceUsed), // bytes free
+	}
+	return usage, nil
+}
+
 // Move src to this remote using server side move operations.
 //
 // This is stored with the remote path given
@@ -762,7 +918,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Do the move
-	info, err := f.move("/files/", srcObj.id, leaf, directoryID)
+	info, err := f.move(ctx, "/files/", srcObj.id, leaf, directoryID)
 	if err != nil {
 		return nil, err
 	}
@@ -788,64 +944,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	_, err = f.move("/folders/", srcID, leaf, directoryID)
+	_, err = f.move(ctx, "/folders/", srcID, dstLeaf, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -854,7 +960,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	id, err := f.dirCache.FindDir(ctx, remote, false)
 	var opts rest.Opts
 	if err == nil {
@@ -887,12 +993,72 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
 	var info api.Item
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &shareLink, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &shareLink, &info)
 		return shouldRetry(resp, err)
 	})
 	return info.SharedLink.URL, err
 }
 
+// deletePermanently permenently deletes a trashed file
+func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
+	opts := rest.Opts{
+		Method:     "DELETE",
+		NoResponse: true,
+	}
+	if itemType == api.ItemTypeFile {
+		opts.Path = "/files/" + id + "/trash"
+	} else {
+		opts.Path = "/folders/" + id + "/trash"
+	}
+	return f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.Call(ctx, &opts)
+		return shouldRetry(resp, err)
+	})
+}
+
+// CleanUp empties the trash
+func (f *Fs) CleanUp(ctx context.Context) (err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/folders/trash/items",
+		Parameters: url.Values{
+			"fields": []string{"type", "id"},
+		},
+	}
+	opts.Parameters.Set("limit", strconv.Itoa(listChunks))
+	offset := 0
+	for {
+		opts.Parameters.Set("offset", strconv.Itoa(offset))
+
+		var result api.FolderItems
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(resp, err)
+		})
+		if err != nil {
+			return errors.Wrap(err, "couldn't list trash")
+		}
+		for i := range result.Entries {
+			item := &result.Entries[i]
+			if item.Type == api.ItemTypeFolder || item.Type == api.ItemTypeFile {
+				err := f.deletePermanently(ctx, item.Type, item.ID)
+				if err != nil {
+					return errors.Wrap(err, "failed to delete file")
+				}
+			} else {
+				fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)
+				continue
+			}
+		}
+		offset += result.Limit
+		if offset >= result.TotalCount {
+			break
+		}
+	}
+	return
+}
+
 // DirCacheFlush resets the directory cache - used in testing as an
 // optional interface
 func (f *Fs) DirCacheFlush() {
@@ -924,11 +1090,6 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// srvPath returns a path for use in server
-func (o *Object) srvPath() string {
-	return replaceReservedChars(o.fs.rootSlash() + o.remote)
-}
-
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.SHA1 {
@@ -1006,7 +1167,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	}
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(&opts, &update, &info)
+		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
 		return shouldRetry(resp, err)
 	})
 	return info, err
@@ -1039,7 +1200,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		Options: options,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1051,9 +1212,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // upload does a single non-multipart upload
 //
 // This is recommended for less than 50 MB of content
-func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Time) (err error) {
+func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
 	upload := api.UploadFile{
-		Name:              replaceReservedChars(leaf),
+		Name:              o.fs.opt.Enc.FromStandardName(leaf),
 		ContentModifiedAt: api.Time(modTime),
 		ContentCreatedAt:  api.Time(modTime),
 		Parent: api.Parent{
@@ -1070,6 +1231,7 @@ func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Tim
 		MultipartContentName:  "contents",
 		MultipartFileName:     upload.Name,
 		RootURL:               uploadURL,
+		Options:               options,
 	}
 	// If object has an ID then it is existing so create a new version
 	if o.id != "" {
@@ -1078,7 +1240,7 @@ func (o *Object) upload(in io.Reader, leaf, directoryID string, modTime time.Tim
 		opts.Path = "/files/content"
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(&opts, &upload, &result)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, &upload, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1104,23 +1266,23 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
 
 	// Upload with simple or multipart
 	if size <= int64(o.fs.opt.UploadCutoff) {
-		err = o.upload(in, leaf, directoryID, modTime)
+		err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
 	} else {
-		err = o.uploadMultipart(in, leaf, directoryID, size, modTime)
+		err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
 	}
 	return err
 }
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	return o.fs.deleteObject(o.id)
+	return o.fs.deleteObject(ctx, o.id)
 }
 
 // ID returns the ID of the Object if known, or "" if not
@@ -1134,10 +1296,12 @@ var (
 	_ fs.Purger          = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/box/upload.go

@@ -4,6 +4,7 @@ package box
 
 import (
 	"bytes"
+	"context"
 	"crypto/sha1"
 	"encoding/base64"
 	"encoding/json"
@@ -18,11 +19,12 @@ import (
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
 )
 
 // createUploadSession creates an upload session for the object
-func (o *Object) createUploadSession(leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
+func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID string, size int64) (response *api.UploadSessionResponse, err error) {
 	opts := rest.Opts{
 		Method:  "POST",
 		Path:    "/files/upload_sessions",
@@ -37,11 +39,11 @@ func (o *Object) createUploadSession(leaf, directoryID string, size int64) (resp
 	} else {
 		opts.Path = "/files/upload_sessions"
 		request.FolderID = directoryID
-		request.FileName = replaceReservedChars(leaf)
+		request.FileName = o.fs.opt.Enc.FromStandardName(leaf)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(&opts, &request, &response)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, &response)
 		return shouldRetry(resp, err)
 	})
 	return
@@ -53,7 +55,7 @@ func sha1Digest(digest []byte) string {
 }
 
 // uploadPart uploads a part in an upload session
-func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) {
+func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
 	chunkSize := int64(len(chunk))
 	sha1sum := sha1.Sum(chunk)
 	opts := rest.Opts{
@@ -63,6 +65,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b
 		ContentType:   "application/octet-stream",
 		ContentLength: &chunkSize,
 		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
+		Options:       options,
 		ExtraHeaders: map[string]string{
 			"Digest": sha1Digest(sha1sum[:]),
 		},
@@ -70,7 +73,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts.Body = wrap(bytes.NewReader(chunk))
-		resp, err = o.fs.srv.CallJSON(&opts, nil, &response)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &response)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -80,7 +83,7 @@ func (o *Object) uploadPart(SessionID string, offset, totalSize int64, chunk []b
 }
 
 // commitUpload finishes an upload session
-func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
+func (o *Object) commitUpload(ctx context.Context, SessionID string, parts []api.Part, modTime time.Time, sha1sum []byte) (result *api.FolderItems, err error) {
 	opts := rest.Opts{
 		Method:  "POST",
 		Path:    "/files/upload_sessions/" + SessionID + "/commit",
@@ -104,7 +107,7 @@ func (o *Object) commitUpload(SessionID string, parts []api.Part, modTime time.T
 outer:
 	for tries = 0; tries < maxTries; tries++ {
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.srv.CallJSON(&opts, &request, nil)
+			resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 			if err != nil {
 				return shouldRetry(resp, err)
 			}
@@ -154,7 +157,7 @@ outer:
 }
 
 // abortUpload cancels an upload session
-func (o *Object) abortUpload(SessionID string) (err error) {
+func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error) {
 	opts := rest.Opts{
 		Method:     "DELETE",
 		Path:       "/files/upload_sessions/" + SessionID,
@@ -163,16 +166,16 @@ func (o *Object) abortUpload(SessionID string) (err error) {
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	return err
 }
 
 // uploadMultipart uploads a file using multipart upload
-func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) {
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
 	// Create upload session
-	session, err := o.createUploadSession(leaf, directoryID, size)
+	session, err := o.createUploadSession(ctx, leaf, directoryID, size)
 	if err != nil {
 		return errors.Wrap(err, "multipart upload create session failed")
 	}
@@ -180,15 +183,13 @@ func (o *Object) uploadMultipart(in io.Reader, leaf, directoryID string, size in
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize))
 
 	// Cancel the session if something went wrong
-	defer func() {
-		if err != nil {
-			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-			cancelErr := o.abortUpload(session.ID)
-			if cancelErr != nil {
-				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
-			}
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(o, "Cancelling multipart upload: %v", err)
+		cancelErr := o.abortUpload(ctx, session.ID)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-	}()
+	})()
 
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
@@ -235,7 +236,7 @@ outer:
 			defer wg.Done()
 			defer o.fs.uploadToken.Put()
 			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
-			partResponse, err := o.uploadPart(session.ID, position, size, buf, wrap)
+			partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
 			if err != nil {
 				err = errors.Wrap(err, "multipart upload failed to upload part")
 				select {
@@ -263,7 +264,7 @@ outer:
 	}
 
 	// Finalise the upload session
-	result, err := o.commitUpload(session.ID, parts, modTime, hash.Sum(nil))
+	result, err := o.commitUpload(ctx, session.ID, parts, modTime, hash.Sum(nil))
 	if err != nil {
 		return errors.Wrap(err, "multipart upload failed to finalize")
 	}

backend/cache/cache.go

@@ -65,6 +65,7 @@ func init() {
 		Name:        "cache",
 		Description: "Cache a remote",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to cache.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
@@ -86,7 +87,7 @@ func init() {
 			Advanced: true,
 		}, {
 			Name:     "plex_insecure",
-			Help:     "Skip all certificate verifications when connecting to the Plex server",
+			Help:     "Skip all certificate verification when connecting to the Plex server",
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -338,7 +339,7 @@ func parseRootPath(path string) (string, error) {
 	return strings.Trim(path, "/"), nil
 }
 
-// NewFs constructs a Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -519,9 +520,6 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// override only those features that use a temp fs and it doesn't support them
 	//f.features.ChangeNotify = f.ChangeNotify
 	if f.opt.TempWritePath != "" {
-		if f.tempFs.Features().Copy == nil {
-			f.features.Copy = nil
-		}
 		if f.tempFs.Features().Move == nil {
 			f.features.Move = nil
 		}
@@ -1532,6 +1530,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs())
 		return nil, fs.ErrorCantCopy
 	}
+	if f.opt.TempWritePath != "" && src.Fs() == f.tempFs {
+		return nil, fs.ErrorCantCopy
+	}
 	// the source must be a cached object or we abort
 	srcObj, ok := src.(*Object)
 	if !ok {
@@ -1828,6 +1829,19 @@ func (f *Fs) isRootInPath(p string) bool {
 	return strings.HasPrefix(p, f.Root()+"/")
 }
 
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
+	do := f.Fs.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	for _, dir := range dirs {
+		_ = f.cache.RemoveDir(dir.Remote())
+	}
+	return do(ctx, dirs)
+}
+
 // DirCacheFlush flushes the dir cache
 func (f *Fs) DirCacheFlush() {
 	_ = f.cache.RemoveDir("")
@@ -1882,6 +1896,31 @@ func (f *Fs) Disconnect(ctx context.Context) error {
 	return do(ctx)
 }
 
+var commandHelp = []fs.CommandHelp{
+	{
+		Name:  "stats",
+		Short: "Print stats on the cache backend in JSON format.",
+	},
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+	switch name {
+	case "stats":
+		return f.Stats()
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs             = (*Fs)(nil)
@@ -1899,4 +1938,6 @@ var (
 	_ fs.Abouter        = (*Fs)(nil)
 	_ fs.UserInfoer     = (*Fs)(nil)
 	_ fs.Disconnecter   = (*Fs)(nil)
+	_ fs.Commander      = (*Fs)(nil)
+	_ fs.MergeDirser    = (*Fs)(nil)
 )

backend/cache/cache_internal_test.go

@@ -1,4 +1,5 @@
 // +build !plan9
+// +build !race
 
 package cache_test
 
@@ -17,7 +18,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
@@ -299,6 +300,7 @@ func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {
 }
 
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
+	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -342,6 +344,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 }
 
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
+	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -1009,6 +1012,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		return nil, nil
 	}
 
+	// Config to pass to NewFs
+	m := configmap.Simple{}
+	for k, v := range r.runDefaultCfgMap {
+		m.Set(k, v)
+	}
+	for k, v := range flags {
+		m.Set(k, v)
+	}
+
 	// if the remote doesn't exist, create a new one with a local one for it
 	// identify which is the cache remote (it can be wrapped by a crypt too)
 	rootIsCrypt := false
@@ -1017,8 +1029,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		localRemote := remote + "-local"
 		config.FileSet(localRemote, "type", "local")
 		config.FileSet(localRemote, "nounc", "true")
-		config.FileSet(remote, "type", "cache")
-		config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
+		m.Set("type", "cache")
+		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
 		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
@@ -1028,8 +1040,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		if remoteType != "cache" {
 			if remoteType == "crypt" {
 				rootIsCrypt = true
-				config.FileSet(remote, "password", cryptPassword1)
-				config.FileSet(remote, "password2", cryptPassword2)
+				m.Set("password", cryptPassword1)
+				m.Set("password2", cryptPassword2)
 			}
 			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
@@ -1055,14 +1067,6 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 
 	fs.Config.LowLevelRetries = 1
 
-	m := configmap.Simple{}
-	for k, v := range r.runDefaultCfgMap {
-		m.Set(k, v)
-	}
-	for k, v := range flags {
-		m.Set(k, v)
-	}
-
 	// Instantiate root
 	if purge {
 		boltDb.PurgeTempUploads()
@@ -1139,23 +1143,6 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	return f
 }
 
-func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string {
-	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
-	// create some rand test data
-	testData := randStringBytes(int(size))
-
-	r.writeRemoteBytes(t, f, remote, testData)
-	return remote
-}
-
-func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object {
-	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
-	// create some rand test data
-	testData := randStringBytes(int(size))
-
-	return r.writeObjectBytes(t, f, remote, testData)
-}
-
 func (r *run) writeRemoteString(t *testing.T, f fs.Fs, remote, content string) {
 	r.writeRemoteBytes(t, f, remote, []byte(content))
 }
@@ -1344,26 +1331,6 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 	return l, err
 }
 
-func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string {
-	var err error
-	var l []string
-	if r.useMount {
-		var list []os.FileInfo
-		list, err = ioutil.ReadDir(path.Join(r.mntDir, remote))
-		for _, ll := range list {
-			l = append(l, ll.Name())
-		}
-	} else {
-		var list fs.DirEntries
-		list, err = f.List(context.Background(), remote)
-		for _, ll := range list {
-			l = append(l, ll.Remote())
-		}
-	}
-	require.NoError(t, err)
-	return l
-}
-
 func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
 	in, err := os.Open(src)
 	if err != nil {

backend/cache/cache_mount_other_test.go

@@ -0,0 +1,21 @@
+// +build !linux !go1.13
+// +build !darwin !go1.13
+// +build !freebsd !go1.13
+// +build !windows
+// +build !race
+
+package cache_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	panic("mountFs not defined for this platform")
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	panic("unmountFs not defined for this platform")
+}

backend/cache/cache_mount_unix_test.go

@@ -1,4 +1,5 @@
-// +build !plan9,!windows
+// +build linux,go1.13 darwin,go1.13 freebsd,go1.13
+// +build !race
 
 package cache_test
 

backend/cache/cache_mount_windows_test.go

@@ -1,4 +1,5 @@
 // +build windows
+// +build !race
 
 package cache_test
 

backend/cache/cache_test.go

@@ -1,6 +1,7 @@
 // Test Cache filesystem interface
 
 // +build !plan9
+// +build !race
 
 package cache_test
 
@@ -17,7 +18,8 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
+		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_upload_test.go

@@ -1,4 +1,5 @@
 // +build !plan9
+// +build !race
 
 package cache_test
 

backend/cache/directory.go

@@ -101,15 +101,6 @@ func (d *Directory) abs() string {
 	return cleanPath(path.Join(d.Dir, d.Name))
 }
 
-// parentRemote returns the absolute path parent remote
-func (d *Directory) parentRemote() string {
-	absPath := d.abs()
-	if absPath == "" {
-		return ""
-	}
-	return cleanPath(path.Dir(absPath))
-}
-
 // ModTime returns the cached ModTime
 func (d *Directory) ModTime(ctx context.Context) time.Time {
 	return time.Unix(0, d.CacheModTime)

backend/cache/object.go

@@ -24,15 +24,16 @@ const (
 type Object struct {
 	fs.Object `json:"-"`
 
-	ParentFs      fs.Fs                `json:"-"`        // parent fs
-	CacheFs       *Fs                  `json:"-"`        // cache fs
-	Name          string               `json:"name"`     // name of the directory
-	Dir           string               `json:"dir"`      // abs path of the object
-	CacheModTime  int64                `json:"modTime"`  // modification or creation time - IsZero for unknown
-	CacheSize     int64                `json:"size"`     // size of directory and contents or -1 if unknown
-	CacheStorable bool                 `json:"storable"` // says whether this object can be stored
-	CacheType     string               `json:"cacheType"`
-	CacheTs       time.Time            `json:"cacheTs"`
+	ParentFs      fs.Fs     `json:"-"`        // parent fs
+	CacheFs       *Fs       `json:"-"`        // cache fs
+	Name          string    `json:"name"`     // name of the directory
+	Dir           string    `json:"dir"`      // abs path of the object
+	CacheModTime  int64     `json:"modTime"`  // modification or creation time - IsZero for unknown
+	CacheSize     int64     `json:"size"`     // size of directory and contents or -1 if unknown
+	CacheStorable bool      `json:"storable"` // says whether this object can be stored
+	CacheType     string    `json:"cacheType"`
+	CacheTs       time.Time `json:"cacheTs"`
+	cacheHashesMu sync.Mutex
 	CacheHashes   map[hash.Type]string // all supported hashes cached
 
 	refreshMutex sync.Mutex
@@ -103,7 +104,9 @@ func (o *Object) updateData(ctx context.Context, source fs.Object) {
 	o.CacheSize = source.Size()
 	o.CacheStorable = source.Storable()
 	o.CacheTs = time.Now()
+	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
+	o.cacheHashesMu.Unlock()
 }
 
 // Fs returns its FS info
@@ -268,7 +271,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	o.CacheModTime = src.ModTime(ctx).UnixNano()
 	o.CacheSize = src.Size()
+	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
+	o.cacheHashesMu.Unlock()
 	o.CacheTs = time.Now()
 	o.persist()
 
@@ -309,11 +314,12 @@ func (o *Object) Remove(ctx context.Context) error {
 // since it might or might not be called, this is lazy loaded
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	_ = o.refresh(ctx)
+	o.cacheHashesMu.Lock()
 	if o.CacheHashes == nil {
 		o.CacheHashes = make(map[hash.Type]string)
 	}
-
 	cachedHash, found := o.CacheHashes[ht]
+	o.cacheHashesMu.Unlock()
 	if found {
 		return cachedHash, nil
 	}
@@ -324,7 +330,9 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	o.cacheHashesMu.Lock()
 	o.CacheHashes[ht] = liveHash
+	o.cacheHashesMu.Unlock()
 
 	o.persist()
 	fs.Debugf(o, "object hash cached: %v", liveHash)

backend/cache/storage_persistent.go

@@ -16,10 +16,10 @@ import (
 	"sync"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
+	bolt "go.etcd.io/bbolt"
 )
 
 // Constants
@@ -767,31 +767,6 @@ func (b *Persistent) iterateBuckets(buk *bolt.Bucket, bucketFn func(name string)
 	return err
 }
 
-func (b *Persistent) dumpRoot() string {
-	var itBuckets func(buk *bolt.Bucket) map[string]interface{}
-
-	itBuckets = func(buk *bolt.Bucket) map[string]interface{} {
-		m := make(map[string]interface{})
-		c := buk.Cursor()
-		for k, v := c.First(); k != nil; k, v = c.Next() {
-			if v == nil {
-				buk2 := buk.Bucket(k)
-				m[string(k)] = itBuckets(buk2)
-			} else {
-				m[string(k)] = "-"
-			}
-		}
-		return m
-	}
-	var mm map[string]interface{}
-	_ = b.db.View(func(tx *bolt.Tx) error {
-		mm = itBuckets(tx.Bucket([]byte(RootBucket)))
-		return nil
-	})
-	raw, _ := json.MarshalIndent(mm, "", "  ")
-	return string(raw)
-}
-
 // addPendingUpload adds a new file to the pending queue of uploads
 func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
@@ -1005,15 +980,6 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 	})
 }
 
-// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
-// TO BE USED IN TESTING ONLY
-func (b *Persistent) SetPendingUploadToStarted(remote string) error {
-	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
-		item.Started = true
-		return nil
-	})
-}
-
 // ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue
 func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
@@ -1061,19 +1027,6 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 	})
 }
 
-// PurgeTempUploads will remove all the pending uploads from the queue
-// TO BE USED IN TESTING ONLY
-func (b *Persistent) PurgeTempUploads() {
-	b.tempQueueMux.Lock()
-	defer b.tempQueueMux.Unlock()
-
-	_ = b.db.Update(func(tx *bolt.Tx) error {
-		_ = tx.DeleteBucket([]byte(tempBucket))
-		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
-		return nil
-	})
-}
-
 // Close should be called when the program ends gracefully
 func (b *Persistent) Close() {
 	b.cleanupMux.Lock()

backend/cache/utils_test.go

@@ -0,0 +1,23 @@
+package cache
+
+import bolt "go.etcd.io/bbolt"
+
+// PurgeTempUploads will remove all the pending uploads from the queue
+func (b *Persistent) PurgeTempUploads() {
+	b.tempQueueMux.Lock()
+	defer b.tempQueueMux.Unlock()
+
+	_ = b.db.Update(func(tx *bolt.Tx) error {
+		_ = tx.DeleteBucket([]byte(tempBucket))
+		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
+		return nil
+	})
+}
+
+// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
+func (b *Persistent) SetPendingUploadToStarted(remote string) error {
+	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
+		item.Started = true
+		return nil
+	})
+}

backend/chunker/chunker_internal_test.go

@@ -0,0 +1,691 @@
+package chunker
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Command line flags
+var (
+	UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads")
+)
+
+// test that chunking does not break large uploads
+func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
+	t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
+		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
+			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
+			Size:    int64(kilobytes) * int64(fs.KibiByte),
+		})
+	})
+}
+
+// test chunk name parser
+func testChunkNameFormat(t *testing.T, f *Fs) {
+	saveOpt := f.opt
+	defer func() {
+		// restore original settings (f is pointer, f.opt is struct)
+		f.opt = saveOpt
+		_ = f.setChunkNameFormat(f.opt.NameFormat)
+	}()
+
+	assertFormat := func(pattern, wantDataFormat, wantCtrlFormat, wantNameRegexp string) {
+		err := f.setChunkNameFormat(pattern)
+		assert.NoError(t, err)
+		assert.Equal(t, wantDataFormat, f.dataNameFmt)
+		assert.Equal(t, wantCtrlFormat, f.ctrlNameFmt)
+		assert.Equal(t, wantNameRegexp, f.nameRegexp.String())
+	}
+
+	assertFormatValid := func(pattern string) {
+		err := f.setChunkNameFormat(pattern)
+		assert.NoError(t, err)
+	}
+
+	assertFormatInvalid := func(pattern string) {
+		err := f.setChunkNameFormat(pattern)
+		assert.Error(t, err)
+	}
+
+	assertMakeName := func(wantChunkName, mainName string, chunkNo int, ctrlType, xactID string) {
+		gotChunkName := ""
+		assert.NotPanics(t, func() {
+			gotChunkName = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
+		}, "makeChunkName(%q,%d,%q,%q) must not panic", mainName, chunkNo, ctrlType, xactID)
+		if gotChunkName != "" {
+			assert.Equal(t, wantChunkName, gotChunkName)
+		}
+	}
+
+	assertMakeNamePanics := func(mainName string, chunkNo int, ctrlType, xactID string) {
+		assert.Panics(t, func() {
+			_ = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
+		}, "makeChunkName(%q,%d,%q,%q) should panic", mainName, chunkNo, ctrlType, xactID)
+	}
+
+	assertParseName := func(fileName, wantMainName string, wantChunkNo int, wantCtrlType, wantXactID string) {
+		gotMainName, gotChunkNo, gotCtrlType, gotXactID := f.parseChunkName(fileName)
+		assert.Equal(t, wantMainName, gotMainName)
+		assert.Equal(t, wantChunkNo, gotChunkNo)
+		assert.Equal(t, wantCtrlType, gotCtrlType)
+		assert.Equal(t, wantXactID, gotXactID)
+	}
+
+	const newFormatSupported = false // support for patterns not starting with base name (*)
+
+	// valid formats
+	assertFormat(`*.rclone_chunk.###`, `%s.rclone_chunk.%03d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*.rclone_chunk.#`, `%s.rclone_chunk.%d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*_chunk_#####`, `%s_chunk_%05d`, `%s_chunk__%s`, `^(.+?)_chunk_(?:([0-9]{5,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*-chunk-#`, `%s-chunk-%d`, `%s-chunk-_%s`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*-chunk-#-%^$()[]{}.+-!?:\`, `%s-chunk-%d-%%^$()[]{}.+-!?:\`, `%s-chunk-_%s-%%^$()[]{}.+-!?:\`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))-%\^\$\(\)\[\]\{\}\.\+-!\?:\\(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	if newFormatSupported {
+		assertFormat(`_*-chunk-##,`, `_%s-chunk-%02d,`, `_%s-chunk-_%s,`, `^_(.+?)-chunk-(?:([0-9]{2,})|_([a-z][a-z0-9]{2,6})),(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	}
+
+	// invalid formats
+	assertFormatInvalid(`chunk-#`)
+	assertFormatInvalid(`*-chunk`)
+	assertFormatInvalid(`*-*-chunk-#`)
+	assertFormatInvalid(`*-chunk-#-#`)
+	assertFormatInvalid(`#-chunk-*`)
+	assertFormatInvalid(`*/#`)
+
+	assertFormatValid(`*#`)
+	assertFormatInvalid(`**#`)
+	assertFormatInvalid(`#*`)
+	assertFormatInvalid(``)
+	assertFormatInvalid(`-`)
+
+	// quick tests
+	if newFormatSupported {
+		assertFormat(`part_*_#`, `part_%s_%d`, `part_%s__%s`, `^part_(.+?)_(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9][0-9a-z]{3,8})\.\.tmp_([0-9]{10,13}))?$`)
+		f.opt.StartFrom = 1
+
+		assertMakeName(`part_fish_1`, "fish", 0, "", "")
+		assertParseName(`part_fish_43`, "fish", 42, "", "")
+		assertMakeName(`part_fish__locks`, "fish", -2, "locks", "")
+		assertParseName(`part_fish__locks`, "fish", -1, "locks", "")
+		assertMakeName(`part_fish__x2y`, "fish", -2, "x2y", "")
+		assertParseName(`part_fish__x2y`, "fish", -1, "x2y", "")
+		assertMakeName(`part_fish_3_0004`, "fish", 2, "", "4")
+		assertParseName(`part_fish_4_0005`, "fish", 3, "", "0005")
+		assertMakeName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -3, "blkinfo", "jj5fvo3wr")
+		assertParseName(`part_fish__blkinfo_zz9fvo3wr`, "fish", -1, "blkinfo", "zz9fvo3wr")
+
+		// old-style temporary suffix (parse only)
+		assertParseName(`part_fish_4..tmp_0000000011`, "fish", 3, "", "000b")
+		assertParseName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -1, "blkinfo", "jj5fvo3wr")
+	}
+
+	// prepare format for long tests
+	assertFormat(`*.chunk.###`, `%s.chunk.%03d`, `%s.chunk._%s`, `^(.+?)\.chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	f.opt.StartFrom = 2
+
+	// valid data chunks
+	assertMakeName(`fish.chunk.003`, "fish", 1, "", "")
+	assertParseName(`fish.chunk.003`, "fish", 1, "", "")
+	assertMakeName(`fish.chunk.021`, "fish", 19, "", "")
+	assertParseName(`fish.chunk.021`, "fish", 19, "", "")
+
+	// valid temporary data chunks
+	assertMakeName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
+	assertParseName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
+	assertMakeName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
+	assertParseName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
+	assertMakeName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
+	assertParseName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
+	assertMakeName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
+	assertParseName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
+
+	// valid temporary data chunks (old temporary suffix, only parse)
+	assertParseName(`fish.chunk.004..tmp_0000000047`, "fish", 2, "", "001b")
+	assertParseName(`fish.chunk.323..tmp_9994567890123`, "fish", 321, "", "3jjfvo3wr")
+
+	// parsing invalid data chunk names
+	assertParseName(`fish.chunk.3`, "", -1, "", "")
+	assertParseName(`fish.chunk.001`, "", -1, "", "")
+	assertParseName(`fish.chunk.21`, "", -1, "", "")
+	assertParseName(`fish.chunk.-21`, "", -1, "", "")
+
+	assertParseName(`fish.chunk.004abcd`, "", -1, "", "")        // missing underscore delimiter
+	assertParseName(`fish.chunk.004__1234`, "", -1, "", "")      // extra underscore delimiter
+	assertParseName(`fish.chunk.004_123`, "", -1, "", "")        // too short temporary suffix
+	assertParseName(`fish.chunk.004_1234567890`, "", -1, "", "") // too long temporary suffix
+	assertParseName(`fish.chunk.004_-1234`, "", -1, "", "")      // temporary suffix must be positive
+	assertParseName(`fish.chunk.004_123E`, "", -1, "", "")       // uppercase not allowed
+	assertParseName(`fish.chunk.004_12.3`, "", -1, "", "")       // punctuation not allowed
+
+	// parsing invalid data chunk names (old temporary suffix)
+	assertParseName(`fish.chunk.004.tmp_0000000021`, "", -1, "", "")
+	assertParseName(`fish.chunk.003..tmp_123456789`, "", -1, "", "")
+	assertParseName(`fish.chunk.003..tmp_012345678901234567890123456789`, "", -1, "", "")
+	assertParseName(`fish.chunk.323..tmp_12345678901234`, "", -1, "", "")
+	assertParseName(`fish.chunk.003..tmp_-1`, "", -1, "", "")
+
+	// valid control chunks
+	assertMakeName(`fish.chunk._info`, "fish", -1, "info", "")
+	assertMakeName(`fish.chunk._locks`, "fish", -2, "locks", "")
+	assertMakeName(`fish.chunk._blkinfo`, "fish", -3, "blkinfo", "")
+	assertMakeName(`fish.chunk._x2y`, "fish", -4, "x2y", "")
+
+	assertParseName(`fish.chunk._info`, "fish", -1, "info", "")
+	assertParseName(`fish.chunk._locks`, "fish", -1, "locks", "")
+	assertParseName(`fish.chunk._blkinfo`, "fish", -1, "blkinfo", "")
+	assertParseName(`fish.chunk._x2y`, "fish", -1, "x2y", "")
+
+	// valid temporary control chunks
+	assertMakeName(`fish.chunk._info_0001`, "fish", -1, "info", "1")
+	assertMakeName(`fish.chunk._locks_4321`, "fish", -2, "locks", "4321")
+	assertMakeName(`fish.chunk._uploads_abcd`, "fish", -3, "uploads", "abcd")
+	assertMakeName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -4, "blkinfo", "xyzabcdef")
+	assertMakeName(`fish.chunk._x2y_1aaa`, "fish", -5, "x2y", "1aaa")
+
+	assertParseName(`fish.chunk._info_0001`, "fish", -1, "info", "0001")
+	assertParseName(`fish.chunk._locks_4321`, "fish", -1, "locks", "4321")
+	assertParseName(`fish.chunk._uploads_9abc`, "fish", -1, "uploads", "9abc")
+	assertParseName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -1, "blkinfo", "xyzabcdef")
+	assertParseName(`fish.chunk._x2y_1aaa`, "fish", -1, "x2y", "1aaa")
+
+	// valid temporary control chunks (old temporary suffix, parse only)
+	assertParseName(`fish.chunk._info..tmp_0000000047`, "fish", -1, "info", "001b")
+	assertParseName(`fish.chunk._locks..tmp_0000054321`, "fish", -1, "locks", "15wx")
+	assertParseName(`fish.chunk._uploads..tmp_0000000000`, "fish", -1, "uploads", "0000")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123`, "fish", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._x2y..tmp_0000000000`, "fish", -1, "x2y", "0000")
+
+	// parsing invalid control chunk names
+	assertParseName(`fish.chunk.metadata`, "", -1, "", "") // must be prepended by underscore
+	assertParseName(`fish.chunk.info`, "", -1, "", "")
+	assertParseName(`fish.chunk.locks`, "", -1, "", "")
+	assertParseName(`fish.chunk.uploads`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._os`, "", -1, "", "")        // too short
+	assertParseName(`fish.chunk._metadata`, "", -1, "", "")  // too long
+	assertParseName(`fish.chunk._blockinfo`, "", -1, "", "") // way too long
+	assertParseName(`fish.chunk._4me`, "", -1, "", "")       // cannot start with digit
+	assertParseName(`fish.chunk._567`, "", -1, "", "")       // cannot be all digits
+	assertParseName(`fish.chunk._me_ta`, "", -1, "", "")     // punctuation not allowed
+	assertParseName(`fish.chunk._in-fo`, "", -1, "", "")
+	assertParseName(`fish.chunk._.bin`, "", -1, "", "")
+	assertParseName(`fish.chunk._.2xy`, "", -1, "", "")
+
+	// parsing invalid temporary control chunks
+	assertParseName(`fish.chunk._blkinfo1234`, "", -1, "", "")     // missing underscore delimiter
+	assertParseName(`fish.chunk._info__1234`, "", -1, "", "")      // extra underscore delimiter
+	assertParseName(`fish.chunk._info_123`, "", -1, "", "")        // too short temporary suffix
+	assertParseName(`fish.chunk._info_1234567890`, "", -1, "", "") // too long temporary suffix
+	assertParseName(`fish.chunk._info_-1234`, "", -1, "", "")      // temporary suffix must be positive
+	assertParseName(`fish.chunk._info_123E`, "", -1, "", "")       // uppercase not allowed
+	assertParseName(`fish.chunk._info_12.3`, "", -1, "", "")       // punctuation not allowed
+
+	assertParseName(`fish.chunk._locks..tmp_123456789`, "", -1, "", "")
+	assertParseName(`fish.chunk._meta..tmp_-1`, "", -1, "", "")
+	assertParseName(`fish.chunk._blockinfo..tmp_012345678901234567890123456789`, "", -1, "", "")
+
+	// short control chunk names: 3 letters ok, 1-2 letters not allowed
+	assertMakeName(`fish.chunk._ext`, "fish", -1, "ext", "")
+	assertParseName(`fish.chunk._int`, "fish", -1, "int", "")
+
+	assertMakeNamePanics("fish", -1, "in", "")
+	assertMakeNamePanics("fish", -1, "up", "4")
+	assertMakeNamePanics("fish", -1, "x", "")
+	assertMakeNamePanics("fish", -1, "c", "1z")
+
+	assertMakeName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0")
+	assertMakeName(`fish.chunk._ext_0026`, "fish", -1, "ext", "26")
+	assertMakeName(`fish.chunk._int_0abc`, "fish", -1, "int", "abc")
+	assertMakeName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
+	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+
+	assertParseName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0000")
+	assertParseName(`fish.chunk._ext_0026`, "fish", -1, "ext", "0026")
+	assertParseName(`fish.chunk._int_0abc`, "fish", -1, "int", "0abc")
+	assertParseName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
+	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+
+	// base file name can sometimes look like a valid chunk name
+	assertParseName(`fish.chunk.003.chunk.004`, "fish.chunk.003", 2, "", "")
+	assertParseName(`fish.chunk.003.chunk._info`, "fish.chunk.003", -1, "info", "")
+	assertParseName(`fish.chunk.003.chunk._Meta`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._info.chunk.004`, "fish.chunk._info", 2, "", "")
+	assertParseName(`fish.chunk._info.chunk._info`, "fish.chunk._info", -1, "info", "")
+	assertParseName(`fish.chunk._info.chunk._info.chunk._Meta`, "", -1, "", "")
+
+	// base file name looking like a valid chunk name (old temporary suffix)
+	assertParseName(`fish.chunk.003.chunk.005..tmp_0000000022`, "fish.chunk.003", 3, "", "000m")
+	assertParseName(`fish.chunk.003.chunk._x..tmp_0000054321`, "", -1, "", "")
+	assertParseName(`fish.chunk._info.chunk.005..tmp_0000000023`, "fish.chunk._info", 3, "", "000n")
+	assertParseName(`fish.chunk._info.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	assertParseName(`fish.chunk.003.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.003", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._info.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._info", -1, "blkinfo", "3jjfvo3wr")
+
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.004`, "fish.chunk.004..tmp_0000000021", 2, "", "")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.005..tmp_0000000025`, "fish.chunk.004..tmp_0000000021", 3, "", "000p")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._info`, "fish.chunk.004..tmp_0000000021", -1, "info", "")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.004..tmp_0000000021", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._Meta`, "", -1, "", "")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.004`, "fish.chunk._blkinfo..tmp_9994567890123", 2, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.005..tmp_0000000026`, "fish.chunk._blkinfo..tmp_9994567890123", 3, "", "000q")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "info", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._Meta`, "", -1, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.004`, "fish.chunk._blkinfo..tmp_1234567890123456789", 2, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.005..tmp_0000000022`, "fish.chunk._blkinfo..tmp_1234567890123456789", 3, "", "000m")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "info", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._Meta`, "", -1, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	// attempts to make invalid chunk names
+	assertMakeNamePanics("fish", -1, "", "")          // neither data nor control
+	assertMakeNamePanics("fish", 0, "info", "")       // both data and control
+	assertMakeNamePanics("fish", -1, "metadata", "")  // control type too long
+	assertMakeNamePanics("fish", -1, "blockinfo", "") // control type way too long
+	assertMakeNamePanics("fish", -1, "2xy", "")       // first digit not allowed
+	assertMakeNamePanics("fish", -1, "123", "")       // all digits not allowed
+	assertMakeNamePanics("fish", -1, "Meta", "")      // only lower case letters allowed
+	assertMakeNamePanics("fish", -1, "in-fo", "")     // punctuation not allowed
+	assertMakeNamePanics("fish", -1, "_info", "")
+	assertMakeNamePanics("fish", -1, "info_", "")
+	assertMakeNamePanics("fish", -2, ".bind", "")
+	assertMakeNamePanics("fish", -2, "bind.", "")
+
+	assertMakeNamePanics("fish", -1, "", "1")          // neither data nor control
+	assertMakeNamePanics("fish", 0, "info", "23")      // both data and control
+	assertMakeNamePanics("fish", -1, "metadata", "45") // control type too long
+	assertMakeNamePanics("fish", -1, "blockinfo", "7") // control type way too long
+	assertMakeNamePanics("fish", -1, "2xy", "abc")     // first digit not allowed
+	assertMakeNamePanics("fish", -1, "123", "def")     // all digits not allowed
+	assertMakeNamePanics("fish", -1, "Meta", "mnk")    // only lower case letters allowed
+	assertMakeNamePanics("fish", -1, "in-fo", "xyz")   // punctuation not allowed
+	assertMakeNamePanics("fish", -1, "_info", "5678")
+	assertMakeNamePanics("fish", -1, "info_", "999")
+	assertMakeNamePanics("fish", -2, ".bind", "0")
+	assertMakeNamePanics("fish", -2, "bind.", "0")
+
+	assertMakeNamePanics("fish", 0, "", "1234567890") // temporary suffix too long
+	assertMakeNamePanics("fish", 0, "", "123F4")      // uppercase not allowed
+	assertMakeNamePanics("fish", 0, "", "123.")       // punctuation not allowed
+	assertMakeNamePanics("fish", 0, "", "_123")
+}
+
+func testSmallFileInternals(t *testing.T, f *Fs) {
+	const dir = "small"
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+	f.opt.FailHard = false
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+
+	checkSmallFileInternals := func(obj fs.Object) {
+		assert.NotNil(t, obj)
+		o, ok := obj.(*Object)
+		assert.True(t, ok)
+		assert.NotNil(t, o)
+		if o == nil {
+			return
+		}
+		switch {
+		case !f.useMeta:
+			// If meta format is "none", non-chunked file (even empty)
+			// internally is a single chunk without meta object.
+			assert.Nil(t, o.main)
+			assert.True(t, o.isComposite()) // sorry, sometimes a name is misleading
+			assert.Equal(t, 1, len(o.chunks))
+		case f.hashAll:
+			// Consistent hashing forces meta object on small files too
+			assert.NotNil(t, o.main)
+			assert.True(t, o.isComposite())
+			assert.Equal(t, 1, len(o.chunks))
+		default:
+			// normally non-chunked file is kept in the Object's main field
+			assert.NotNil(t, o.main)
+			assert.False(t, o.isComposite())
+			assert.Equal(t, 0, len(o.chunks))
+		}
+	}
+
+	checkContents := func(obj fs.Object, contents string) {
+		assert.NotNil(t, obj)
+		assert.Equal(t, int64(len(contents)), obj.Size())
+
+		r, err := obj.Open(ctx)
+		assert.NoError(t, err)
+		assert.NotNil(t, r)
+		if r == nil {
+			return
+		}
+		data, err := ioutil.ReadAll(r)
+		assert.NoError(t, err)
+		assert.Equal(t, contents, string(data))
+		_ = r.Close()
+	}
+
+	checkHashsum := func(obj fs.Object) {
+		var ht hash.Type
+		switch {
+		case !f.hashAll:
+			return
+		case f.useMD5:
+			ht = hash.MD5
+		case f.useSHA1:
+			ht = hash.SHA1
+		default:
+			return
+		}
+		// even empty files must have hashsum in consistent mode
+		sum, err := obj.Hash(ctx, ht)
+		assert.NoError(t, err)
+		assert.NotEqual(t, sum, "")
+	}
+
+	checkSmallFile := func(name, contents string) {
+		filename := path.Join(dir, name)
+		item := fstest.Item{Path: filename, ModTime: modTime}
+		_, put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
+		assert.NotNil(t, put)
+		checkSmallFileInternals(put)
+		checkContents(put, contents)
+		checkHashsum(put)
+
+		// objects returned by Put and NewObject must have similar structure
+		obj, err := f.NewObject(ctx, filename)
+		assert.NoError(t, err)
+		assert.NotNil(t, obj)
+		checkSmallFileInternals(obj)
+		checkContents(obj, contents)
+		checkHashsum(obj)
+
+		_ = obj.Remove(ctx)
+		_ = put.Remove(ctx) // for good
+	}
+
+	checkSmallFile("emptyfile", "")
+	checkSmallFile("smallfile", "Ok")
+}
+
+func testPreventCorruption(t *testing.T, f *Fs) {
+	if f.opt.ChunkSize > 50 {
+		t.Skip("this test requires small chunks")
+	}
+	const dir = "corrupted"
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+	f.opt.FailHard = true
+
+	contents := random.String(250)
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	const overlapMessage = "chunk overlap"
+
+	assertOverlapError := func(err error) {
+		assert.Error(t, err)
+		if err != nil {
+			assert.Contains(t, err.Error(), overlapMessage)
+		}
+	}
+
+	newFile := func(name string) fs.Object {
+		item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		require.NotNil(t, obj)
+		return obj
+	}
+	billyObj := newFile("billy")
+
+	billyChunkName := func(chunkNo int) string {
+		return f.makeChunkName(billyObj.Remote(), chunkNo, "", "")
+	}
+
+	err := f.Mkdir(ctx, billyChunkName(1))
+	assertOverlapError(err)
+
+	_, err = f.Move(ctx, newFile("silly1"), billyChunkName(2))
+	assert.Error(t, err)
+	assert.True(t, err == fs.ErrorCantMove || (err != nil && strings.Contains(err.Error(), overlapMessage)))
+
+	_, err = f.Copy(ctx, newFile("silly2"), billyChunkName(3))
+	assert.Error(t, err)
+	assert.True(t, err == fs.ErrorCantCopy || (err != nil && strings.Contains(err.Error(), overlapMessage)))
+
+	// accessing chunks in strict mode is prohibited
+	f.opt.FailHard = true
+	billyChunk4Name := billyChunkName(4)
+	billyChunk4, err := f.NewObject(ctx, billyChunk4Name)
+	assertOverlapError(err)
+
+	f.opt.FailHard = false
+	billyChunk4, err = f.NewObject(ctx, billyChunk4Name)
+	assert.NoError(t, err)
+	require.NotNil(t, billyChunk4)
+
+	f.opt.FailHard = true
+	_, err = f.Put(ctx, bytes.NewBufferString(contents), billyChunk4)
+	assertOverlapError(err)
+
+	// you can freely read chunks (if you have an object)
+	r, err := billyChunk4.Open(ctx)
+	assert.NoError(t, err)
+	var chunkContents []byte
+	assert.NotPanics(t, func() {
+		chunkContents, err = ioutil.ReadAll(r)
+		_ = r.Close()
+	})
+	assert.NoError(t, err)
+	assert.NotEqual(t, contents, string(chunkContents))
+
+	// but you can't change them
+	err = billyChunk4.Update(ctx, bytes.NewBufferString(contents), newFile("silly3"))
+	assertOverlapError(err)
+
+	// Remove isn't special, you can't corrupt files even if you have an object
+	err = billyChunk4.Remove(ctx)
+	assertOverlapError(err)
+
+	// recreate billy in case it was anyhow corrupted
+	willyObj := newFile("willy")
+	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", "")
+	f.opt.FailHard = false
+	willyChunk, err := f.NewObject(ctx, willyChunkName)
+	f.opt.FailHard = true
+	assert.NoError(t, err)
+	require.NotNil(t, willyChunk)
+
+	_, err = operations.Copy(ctx, f, willyChunk, willyChunkName, newFile("silly4"))
+	assertOverlapError(err)
+
+	// operations.Move will return error when chunker's Move refused
+	// to corrupt target file, but reverts to copy/delete method
+	// still trying to delete target chunk. Chunker must come to rescue.
+	_, err = operations.Move(ctx, f, willyChunk, willyChunkName, newFile("silly5"))
+	assertOverlapError(err)
+	r, err = willyChunk.Open(ctx)
+	assert.NoError(t, err)
+	assert.NotPanics(t, func() {
+		_, err = ioutil.ReadAll(r)
+		_ = r.Close()
+	})
+	assert.NoError(t, err)
+}
+
+func testChunkNumberOverflow(t *testing.T, f *Fs) {
+	if f.opt.ChunkSize > 50 {
+		t.Skip("this test requires small chunks")
+	}
+	const dir = "wreaked"
+	const wreakNumber = 10200300
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	contents := random.String(100)
+
+	newFile := func(f fs.Fs, name string) (fs.Object, string) {
+		filename := path.Join(dir, name)
+		item := fstest.Item{Path: filename, ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		require.NotNil(t, obj)
+		return obj, filename
+	}
+
+	f.opt.FailHard = false
+	file, fileName := newFile(f, "wreaker")
+	wreak, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", ""))
+
+	f.opt.FailHard = false
+	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
+	_, err := f.NewObject(ctx, fileName)
+	assert.Error(t, err)
+
+	f.opt.FailHard = true
+	_, err = f.List(ctx, dir)
+	assert.Error(t, err)
+	_, err = f.NewObject(ctx, fileName)
+	assert.Error(t, err)
+
+	f.opt.FailHard = false
+	_ = wreak.Remove(ctx)
+	_ = file.Remove(ctx)
+}
+
+func testMetadataInput(t *testing.T, f *Fs) {
+	const minChunkForTest = 50
+	if f.opt.ChunkSize < minChunkForTest {
+		t.Skip("this test requires chunks that fit metadata")
+	}
+
+	const dir = "usermeta"
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+	f.opt.FailHard = false
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+
+	putFile := func(f fs.Fs, name, contents, message string, check bool) fs.Object {
+		item := fstest.Item{Path: name, ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
+		assert.NotNil(t, obj, message)
+		return obj
+	}
+
+	runSubtest := func(contents, name string) {
+		description := fmt.Sprintf("file with %s metadata", name)
+		filename := path.Join(dir, name)
+		require.True(t, len(contents) > 2 && len(contents) < minChunkForTest, description+" test data is correct")
+
+		part := putFile(f.base, f.makeChunkName(filename, 0, "", ""), "oops", "", true)
+		_ = putFile(f, filename, contents, "upload "+description, false)
+
+		obj, err := f.NewObject(ctx, filename)
+		assert.NoError(t, err, "access "+description)
+		assert.NotNil(t, obj)
+		assert.Equal(t, int64(len(contents)), obj.Size(), "size "+description)
+
+		o, ok := obj.(*Object)
+		assert.NotNil(t, ok)
+		if o != nil {
+			assert.True(t, o.isComposite() && len(o.chunks) == 1, description+" is forced composite")
+			o = nil
+		}
+
+		defer func() {
+			_ = obj.Remove(ctx)
+			_ = part.Remove(ctx)
+		}()
+
+		r, err := obj.Open(ctx)
+		assert.NoError(t, err, "open "+description)
+		assert.NotNil(t, r, "open stream of "+description)
+		if err == nil && r != nil {
+			data, err := ioutil.ReadAll(r)
+			assert.NoError(t, err, "read all of "+description)
+			assert.Equal(t, contents, string(data), description+" contents is ok")
+			_ = r.Close()
+		}
+	}
+
+	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "")
+	require.NoError(t, err)
+	todaysMeta := string(metaData)
+	runSubtest(todaysMeta, "today")
+
+	pastMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":1`)
+	pastMeta = regexp.MustCompile(`"size":[0-9]+`).ReplaceAllLiteralString(pastMeta, `"size":0`)
+	runSubtest(pastMeta, "past")
+
+	futureMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":999`)
+	futureMeta = regexp.MustCompile(`"nchunks":[0-9]+`).ReplaceAllLiteralString(futureMeta, `"nchunks":0,"x":"y"`)
+	runSubtest(futureMeta, "future")
+}
+
+// InternalTest dispatches all internal tests
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("PutLarge", func(t *testing.T) {
+		if *UploadKilobytes <= 0 {
+			t.Skip("-upload-kilobytes is not set")
+		}
+		testPutLarge(t, f, *UploadKilobytes)
+	})
+	t.Run("ChunkNameFormat", func(t *testing.T) {
+		testChunkNameFormat(t, f)
+	})
+	t.Run("SmallFileInternals", func(t *testing.T) {
+		testSmallFileInternals(t, f)
+	})
+	t.Run("PreventCorruption", func(t *testing.T) {
+		testPreventCorruption(t, f)
+	})
+	t.Run("ChunkNumberOverflow", func(t *testing.T) {
+		testChunkNumberOverflow(t, f)
+	})
+	t.Run("MetadataInput", func(t *testing.T) {
+		testMetadataInput(t, f)
+	})
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -0,0 +1,58 @@
+// Test the Chunker filesystem interface
+package chunker_test
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/all" // for integration tests
+	"github.com/rclone/rclone/backend/chunker"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// Command line flags
+var (
+	// Invalid characters are not supported by some remotes, eg. Mailru.
+	// We enable testing with invalid characters when -remote is not set, so
+	// chunker overlays a local directory, but invalid characters are disabled
+	// by default when -remote is set, eg. when test_all runs backend tests.
+	// You can still test with invalid characters using the below flag.
+	UseBadChars = flag.Bool("bad-chars", false, "Set to test bad characters in file names when -remote is set")
+)
+
+// TestIntegration runs integration tests against a concrete remote
+// set by the -remote flag. If the flag is not set, it creates a
+// dynamic chunker overlay wrapping a local temporary directory.
+func TestIntegration(t *testing.T) {
+	opt := fstests.Opt{
+		RemoteName:               *fstest.RemoteName,
+		NilObject:                (*chunker.Object)(nil),
+		SkipBadWindowsCharacters: !*UseBadChars,
+		UnimplementableObjectMethods: []string{
+			"MimeType",
+			"GetTier",
+			"SetTier",
+		},
+		UnimplementableFsMethods: []string{
+			"PublicLink",
+			"OpenWriterAt",
+			"MergeDirs",
+			"DirCacheFlush",
+			"UserInfo",
+			"Disconnect",
+		},
+	}
+	if *fstest.RemoteName == "" {
+		name := "TestChunker"
+		opt.RemoteName = name + ":"
+		tempDir := filepath.Join(os.TempDir(), "rclone-chunker-test-standard")
+		opt.ExtraConfig = []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "chunker"},
+			{Name: name, Key: "remote", Value: tempDir},
+		}
+	}
+	fstests.Run(t, &opt)
+}

backend/crypt/cipher.go

@@ -71,30 +71,6 @@ type ReadSeekCloser interface {
 // OpenRangeSeek opens the file handle at the offset with the limit given
 type OpenRangeSeek func(ctx context.Context, offset, limit int64) (io.ReadCloser, error)
 
-// Cipher is used to swap out the encryption implementations
-type Cipher interface {
-	// EncryptFileName encrypts a file path
-	EncryptFileName(string) string
-	// DecryptFileName decrypts a file path, returns error if decrypt was invalid
-	DecryptFileName(string) (string, error)
-	// EncryptDirName encrypts a directory path
-	EncryptDirName(string) string
-	// DecryptDirName decrypts a directory path, returns error if decrypt was invalid
-	DecryptDirName(string) (string, error)
-	// EncryptData
-	EncryptData(io.Reader) (io.Reader, error)
-	// DecryptData
-	DecryptData(io.ReadCloser) (io.ReadCloser, error)
-	// DecryptDataSeek decrypt at a given position
-	DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error)
-	// EncryptedSize calculates the size of the data when encrypted
-	EncryptedSize(int64) int64
-	// DecryptedSize calculates the size of the data when decrypted
-	DecryptedSize(int64) (int64, error)
-	// NameEncryptionMode returns the used mode for name handling
-	NameEncryptionMode() NameEncryptionMode
-}
-
 // NameEncryptionMode is the type of file name encryption in use
 type NameEncryptionMode int
 
@@ -136,7 +112,8 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-type cipher struct {
+// Cipher defines an encoding and decoding cipher for the crypt backend
+type Cipher struct {
 	dataKey        [32]byte                  // Key for secretbox
 	nameKey        [32]byte                  // 16,24 or 32 bytes
 	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
@@ -148,8 +125,8 @@ type cipher struct {
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*cipher, error) {
-	c := &cipher{
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*Cipher, error) {
+	c := &Cipher{
 		mode:           mode,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
@@ -172,7 +149,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 //
 // Note that empty passsword makes all 0x00 keys which is used in the
 // tests.
-func (c *cipher) Key(password, salt string) (err error) {
+func (c *Cipher) Key(password, salt string) (err error) {
 	const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
 	var saltBytes = defaultSalt
 	if salt != "" {
@@ -196,33 +173,18 @@ func (c *cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *cipher) getBlock() []byte {
+func (c *Cipher) getBlock() []byte {
 	return c.buffers.Get().([]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *cipher) putBlock(buf []byte) {
+func (c *Cipher) putBlock(buf []byte) {
 	if len(buf) != blockSize {
 		panic("bad blocksize returned to pool")
 	}
 	c.buffers.Put(buf)
 }
 
-// check to see if the byte string is valid with no control characters
-// from 0x00 to 0x1F and is a valid UTF-8 string
-func checkValidString(buf []byte) error {
-	for i := range buf {
-		c := buf[i]
-		if c >= 0x00 && c < 0x20 || c == 0x7F {
-			return ErrorBadDecryptControlChar
-		}
-	}
-	if !utf8.Valid(buf) {
-		return ErrorBadDecryptUTF8
-	}
-	return nil
-}
-
 // encodeFileName encodes a filename using a modified version of
 // standard base32 as described in RFC4648
 //
@@ -255,13 +217,13 @@ func decodeFileName(in string) ([]byte, error) {
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
 // Rogaway.
 //
-// This makes for determinstic encryption which is what we want - the
+// This makes for deterministic encryption which is what we want - the
 // same filename must encrypt to the same thing.
 //
 // This means that
 //  * filenames with the same name will encrypt the same
 //  * filenames which start the same won't have a common prefix
-func (c *cipher) encryptSegment(plaintext string) string {
+func (c *Cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -271,7 +233,7 @@ func (c *cipher) encryptSegment(plaintext string) string {
 }
 
 // decryptSegment decrypts a path segment
-func (c *cipher) decryptSegment(ciphertext string) (string, error) {
+func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -294,15 +256,11 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	err = checkValidString(plaintext)
-	if err != nil {
-		return "", err
-	}
 	return string(plaintext), err
 }
 
 // Simple obfuscation routines
-func (c *cipher) obfuscateSegment(plaintext string) string {
+func (c *Cipher) obfuscateSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -389,7 +347,7 @@ func (c *cipher) obfuscateSegment(plaintext string) string {
 	return result.String()
 }
 
-func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
+func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -476,7 +434,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 }
 
 // encryptFileName encrypts a file path
-func (c *cipher) encryptFileName(in string) string {
+func (c *Cipher) encryptFileName(in string) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		// Skip directory name encryption if the user chose to
@@ -494,7 +452,7 @@ func (c *cipher) encryptFileName(in string) string {
 }
 
 // EncryptFileName encrypts a file path
-func (c *cipher) EncryptFileName(in string) string {
+func (c *Cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
 		return in + encryptedSuffix
 	}
@@ -502,7 +460,7 @@ func (c *cipher) EncryptFileName(in string) string {
 }
 
 // EncryptDirName encrypts a directory path
-func (c *cipher) EncryptDirName(in string) string {
+func (c *Cipher) EncryptDirName(in string) string {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in
 	}
@@ -510,7 +468,7 @@ func (c *cipher) EncryptDirName(in string) string {
 }
 
 // decryptFileName decrypts a file path
-func (c *cipher) decryptFileName(in string) (string, error) {
+func (c *Cipher) decryptFileName(in string) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -533,7 +491,7 @@ func (c *cipher) decryptFileName(in string) (string, error) {
 }
 
 // DecryptFileName decrypts a file path
-func (c *cipher) DecryptFileName(in string) (string, error) {
+func (c *Cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
 		remainingLength := len(in) - len(encryptedSuffix)
 		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
@@ -545,14 +503,15 @@ func (c *cipher) DecryptFileName(in string) (string, error) {
 }
 
 // DecryptDirName decrypts a directory path
-func (c *cipher) DecryptDirName(in string) (string, error) {
+func (c *Cipher) DecryptDirName(in string) (string, error) {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in, nil
 	}
 	return c.decryptFileName(in)
 }
 
-func (c *cipher) NameEncryptionMode() NameEncryptionMode {
+// NameEncryptionMode returns the encryption mode in use for names
+func (c *Cipher) NameEncryptionMode() NameEncryptionMode {
 	return c.mode
 }
 
@@ -600,7 +559,7 @@ func (n *nonce) increment() {
 	n.carry(0)
 }
 
-// add an uint64 to the nonce
+// add a uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
 	for i := 0; i < 8; i++ {
@@ -620,7 +579,7 @@ func (n *nonce) add(x uint64) {
 type encrypter struct {
 	mu       sync.Mutex
 	in       io.Reader
-	c        *cipher
+	c        *Cipher
 	nonce    nonce
 	buf      []byte
 	readBuf  []byte
@@ -630,7 +589,7 @@ type encrypter struct {
 }
 
 // newEncrypter creates a new file handle encrypting on the fly
-func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
+func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 	fh := &encrypter{
 		in:      in,
 		c:       c,
@@ -702,13 +661,19 @@ func (fh *encrypter) finish(err error) (int, error) {
 }
 
 // Encrypt data encrypts the data stream
-func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
+func (c *Cipher) encryptData(in io.Reader) (io.Reader, *encrypter, error) {
 	in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader
 	out, err := c.newEncrypter(in, nil)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return wrap(out), nil // and wrap the accounting back on
+	return wrap(out), out, nil // and wrap the accounting back on
+}
+
+// EncryptData encrypts the data stream
+func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
+	out, _, err := c.encryptData(in)
+	return out, err
 }
 
 // decrypter decrypts an io.ReaderCloser on the fly
@@ -717,7 +682,7 @@ type decrypter struct {
 	rc           io.ReadCloser
 	nonce        nonce
 	initialNonce nonce
-	c            *cipher
+	c            *Cipher
 	buf          []byte
 	readBuf      []byte
 	bufIndex     int
@@ -728,7 +693,7 @@ type decrypter struct {
 }
 
 // newDecrypter creates a new file handle decrypting on the fly
-func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
+func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	fh := &decrypter{
 		rc:      rc,
 		c:       c,
@@ -756,7 +721,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 }
 
 // newDecrypterSeek creates a new file handle decrypting on the fly
-func (c *cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
+func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
 	var rc io.ReadCloser
 	doRangeSeek := false
 	setLimit := false
@@ -1031,7 +996,7 @@ func (fh *decrypter) finishAndClose(err error) error {
 }
 
 // DecryptData decrypts the data stream
-func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
+func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 	out, err := c.newDecrypter(rc)
 	if err != nil {
 		return nil, err
@@ -1044,7 +1009,7 @@ func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 // The open function must return a ReadCloser opened to the offset supplied
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
-func (c *cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
+func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
 	out, err := c.newDecrypterSeek(ctx, open, offset, limit)
 	if err != nil {
 		return nil, err
@@ -1053,7 +1018,7 @@ func (c *cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset
 }
 
 // EncryptedSize calculates the size of the data when encrypted
-func (c *cipher) EncryptedSize(size int64) int64 {
+func (c *Cipher) EncryptedSize(size int64) int64 {
 	blocks, residue := size/blockDataSize, size%blockDataSize
 	encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
 	if residue != 0 {
@@ -1063,7 +1028,7 @@ func (c *cipher) EncryptedSize(size int64) int64 {
 }
 
 // DecryptedSize calculates the size of the data when decrypted
-func (c *cipher) DecryptedSize(size int64) (int64, error) {
+func (c *Cipher) DecryptedSize(size int64) (int64, error) {
 	size -= int64(fileHeaderSize)
 	if size < 0 {
 		return 0, ErrorEncryptedFileTooShort
@@ -1082,7 +1047,6 @@ func (c *cipher) DecryptedSize(size int64) (int64, error) {
 
 // check interfaces
 var (
-	_ Cipher         = (*cipher)(nil)
 	_ io.ReadCloser  = (*decrypter)(nil)
 	_ io.Seeker      = (*decrypter)(nil)
 	_ fs.RangeSeeker = (*decrypter)(nil)

backend/crypt/cipher_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -44,69 +45,6 @@ func TestNewNameEncryptionModeString(t *testing.T) {
 	assert.Equal(t, NameEncryptionMode(3).String(), "Unknown mode #3")
 }
 
-func TestValidString(t *testing.T) {
-	for _, test := range []struct {
-		in       string
-		expected error
-	}{
-		{"", nil},
-		{"\x01", ErrorBadDecryptControlChar},
-		{"a\x02", ErrorBadDecryptControlChar},
-		{"abc\x03", ErrorBadDecryptControlChar},
-		{"abc\x04def", ErrorBadDecryptControlChar},
-		{"\x05d", ErrorBadDecryptControlChar},
-		{"\x06def", ErrorBadDecryptControlChar},
-		{"\x07", ErrorBadDecryptControlChar},
-		{"\x08", ErrorBadDecryptControlChar},
-		{"\x09", ErrorBadDecryptControlChar},
-		{"\x0A", ErrorBadDecryptControlChar},
-		{"\x0B", ErrorBadDecryptControlChar},
-		{"\x0C", ErrorBadDecryptControlChar},
-		{"\x0D", ErrorBadDecryptControlChar},
-		{"\x0E", ErrorBadDecryptControlChar},
-		{"\x0F", ErrorBadDecryptControlChar},
-		{"\x10", ErrorBadDecryptControlChar},
-		{"\x11", ErrorBadDecryptControlChar},
-		{"\x12", ErrorBadDecryptControlChar},
-		{"\x13", ErrorBadDecryptControlChar},
-		{"\x14", ErrorBadDecryptControlChar},
-		{"\x15", ErrorBadDecryptControlChar},
-		{"\x16", ErrorBadDecryptControlChar},
-		{"\x17", ErrorBadDecryptControlChar},
-		{"\x18", ErrorBadDecryptControlChar},
-		{"\x19", ErrorBadDecryptControlChar},
-		{"\x1A", ErrorBadDecryptControlChar},
-		{"\x1B", ErrorBadDecryptControlChar},
-		{"\x1C", ErrorBadDecryptControlChar},
-		{"\x1D", ErrorBadDecryptControlChar},
-		{"\x1E", ErrorBadDecryptControlChar},
-		{"\x1F", ErrorBadDecryptControlChar},
-		{"\x20", nil},
-		{"\x7E", nil},
-		{"\x7F", ErrorBadDecryptControlChar},
-		{"£100", nil},
-		{`hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, nil},
-		{"£100", nil},
-		// Following tests from https://secure.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
-		{"a", nil},                                        // Valid ASCII
-		{"\xc3\xb1", nil},                                 // Valid 2 Octet Sequence
-		{"\xc3\x28", ErrorBadDecryptUTF8},                 // Invalid 2 Octet Sequence
-		{"\xa0\xa1", ErrorBadDecryptUTF8},                 // Invalid Sequence Identifier
-		{"\xe2\x82\xa1", nil},                             // Valid 3 Octet Sequence
-		{"\xe2\x28\xa1", ErrorBadDecryptUTF8},             // Invalid 3 Octet Sequence (in 2nd Octet)
-		{"\xe2\x82\x28", ErrorBadDecryptUTF8},             // Invalid 3 Octet Sequence (in 3rd Octet)
-		{"\xf0\x90\x8c\xbc", nil},                         // Valid 4 Octet Sequence
-		{"\xf0\x28\x8c\xbc", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 2nd Octet)
-		{"\xf0\x90\x28\xbc", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 3rd Octet)
-		{"\xf0\x28\x8c\x28", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 4th Octet)
-		{"\xf8\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8},     // Valid 5 Octet Sequence (but not Unicode!)
-		{"\xfc\xa1\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 6 Octet Sequence (but not Unicode!)
-	} {
-		actual := checkValidString([]byte(test.in))
-		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
-	}
-}
-
 func TestEncodeFileName(t *testing.T) {
 	for _, test := range []struct {
 		in       string
@@ -210,8 +148,6 @@ func TestDecryptSegment(t *testing.T) {
 		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
-		{c.encryptSegment("\x01"), ErrorBadDecryptControlChar},
-		{c.encryptSegment("\xc3\x28"), ErrorBadDecryptUTF8},
 	} {
 		actual, actualErr := c.decryptSegment(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))
@@ -705,16 +641,16 @@ var (
 
 // Test test infrastructure first!
 func TestRandomSource(t *testing.T) {
-	source := newRandomSource(1E8)
-	sink := newRandomSource(1E8)
+	source := newRandomSource(1e8)
+	sink := newRandomSource(1e8)
 	n, err := io.Copy(sink, source)
 	assert.NoError(t, err)
-	assert.Equal(t, int64(1E8), n)
+	assert.Equal(t, int64(1e8), n)
 
-	source = newRandomSource(1E8)
+	source = newRandomSource(1e8)
 	buf := make([]byte, 16)
 	_, _ = source.Read(buf)
-	sink = newRandomSource(1E8)
+	sink = newRandomSource(1e8)
 	_, err = io.Copy(sink, source)
 	assert.Error(t, err, "Error in stream")
 }
@@ -754,23 +690,23 @@ func testEncryptDecrypt(t *testing.T, bufSize int, copySize int64) {
 }
 
 func TestEncryptDecrypt1(t *testing.T) {
-	testEncryptDecrypt(t, 1, 1E7)
+	testEncryptDecrypt(t, 1, 1e7)
 }
 
 func TestEncryptDecrypt32(t *testing.T) {
-	testEncryptDecrypt(t, 32, 1E8)
+	testEncryptDecrypt(t, 32, 1e8)
 }
 
 func TestEncryptDecrypt4096(t *testing.T) {
-	testEncryptDecrypt(t, 4096, 1E8)
+	testEncryptDecrypt(t, 4096, 1e8)
 }
 
 func TestEncryptDecrypt65536(t *testing.T) {
-	testEncryptDecrypt(t, 65536, 1E8)
+	testEncryptDecrypt(t, 65536, 1e8)
 }
 
 func TestEncryptDecrypt65537(t *testing.T) {
-	testEncryptDecrypt(t, 65537, 1E8)
+	testEncryptDecrypt(t, 65537, 1e8)
 }
 
 var (
@@ -803,7 +739,7 @@ func TestEncryptData(t *testing.T) {
 	} {
 		c, err := newCipher(NameEncryptionStandard, "", "", true)
 		assert.NoError(t, err)
-		c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
+		c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
 		// Check encode works
 		buf := bytes.NewBuffer(test.in)
@@ -826,7 +762,7 @@ func TestEncryptData(t *testing.T) {
 func TestNewEncrypter(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
-	c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
+	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
 	z := &zeroes{}
 
@@ -849,23 +785,15 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
 
-	in := &errorReader{io.ErrUnexpectedEOF}
+	in := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1E6)
+	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(32), n)
 }
 
-type errorReader struct {
-	err error
-}
-
-func (er errorReader) Read(p []byte) (n int, err error) {
-	return 0, er.err
-}
-
 type closeDetector struct {
 	io.Reader
 	closed int
@@ -885,7 +813,7 @@ func (c *closeDetector) Close() error {
 func TestNewDecrypter(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
-	c.cryptoRand = newRandomSource(1E8) // nodge the crypto rand generator
+	c.cryptoRand = newRandomSource(1e8) // nodge the crypto rand generator
 
 	cd := newCloseDetector(bytes.NewBuffer(file0))
 	fh, err := c.newDecrypter(cd)
@@ -903,7 +831,7 @@ func TestNewDecrypter(t *testing.T) {
 		assert.Equal(t, 1, cd.closed)
 	}
 
-	er := &errorReader{errors.New("potato")}
+	er := &readers.ErrorReader{Err: errors.New("potato")}
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
@@ -929,14 +857,14 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
 
-	in2 := &errorReader{io.ErrUnexpectedEOF}
+	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
 	in := ioutil.NopCloser(io.MultiReader(in1, in2))
 
 	fh, err := c.newDecrypter(in)
 	assert.NoError(t, err)
 
-	n, err := io.CopyN(ioutil.Discard, fh, 1E6)
+	n, err := io.CopyN(ioutil.Discard, fh, 1e6)
 	assert.Equal(t, io.ErrUnexpectedEOF, err)
 	assert.Equal(t, int64(16), n)
 }
@@ -1001,7 +929,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 		assert.Equal(t, 0, n)
 	}
 
-	// Now try decoding it with a open/seek
+	// Now try decoding it with an open/seek
 	for _, offset := range trials {
 		for _, limit := range limits {
 			if offset+limit > len(plaintext) {
@@ -1183,7 +1111,7 @@ func TestDecrypterRead(t *testing.T) {
 
 	// Test producing an error on the file on Read the underlying file
 	in1 := bytes.NewBuffer(file1)
-	in2 := &errorReader{errors.New("potato")}
+	in2 := &readers.ErrorReader{Err: errors.New("potato")}
 	in := io.MultiReader(in1, in2)
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)

backend/crypt/crypt.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"path"
 	"strings"
 	"time"
 
@@ -25,6 +26,7 @@ func init() {
 		Name:        "crypt",
 		Description: "Encrypt/Decrypt a remote",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
@@ -35,19 +37,21 @@ func init() {
 			Default: "standard",
 			Examples: []fs.OptionExample{
 				{
-					Value: "off",
-					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
-				}, {
 					Value: "standard",
 					Help:  "Encrypt the filenames see the docs for the details.",
 				}, {
 					Value: "obfuscate",
 					Help:  "Very simple filename obfuscation.",
+				}, {
+					Value: "off",
+					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
 				},
 			},
 		}, {
-			Name:    "directory_name_encryption",
-			Help:    "Option to either encrypt directory names or leave them intact.",
+			Name: "directory_name_encryption",
+			Help: `Option to either encrypt directory names or leave them intact.
+
+NB If filename_encryption is "off" then this option will do nothing.`,
 			Default: true,
 			Examples: []fs.OptionExample{
 				{
@@ -63,10 +67,25 @@ func init() {
 			Name:       "password",
 			Help:       "Password or pass phrase for encryption.",
 			IsPassword: true,
+			Required:   true,
 		}, {
 			Name:       "password2",
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
 			IsPassword: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different crypt configs.
+
+Normally this option is not what you want, but if you have two crypts
+pointing to the same backend you can use it.
+
+This can be used, for example, to change file name encryption type
+without re-uploading all the data. Just make two crypt backends
+pointing to two different directories with the single changed
+parameter and use rclone move to move the files between the crypt
+remotes.`,
+			Advanced: true,
 		}, {
 			Name: "show_mapping",
 			Help: `For all files listed show how the names encrypt.
@@ -86,7 +105,7 @@ names, or for debugging purposes.`,
 }
 
 // newCipherForConfig constructs a Cipher for the given config name
-func newCipherForConfig(opt *Options) (Cipher, error) {
+func newCipherForConfig(opt *Options) (*Cipher, error) {
 	mode, err := NewNameEncryptionMode(opt.FilenameEncryption)
 	if err != nil {
 		return nil, err
@@ -113,7 +132,7 @@ func newCipherForConfig(opt *Options) (Cipher, error) {
 }
 
 // NewCipher constructs a Cipher for the given config
-func NewCipher(m configmap.Mapper) (Cipher, error) {
+func NewCipher(m configmap.Mapper) (*Cipher, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -143,6 +162,10 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
 	}
+	// Make sure to remove trailing . reffering to the current dir
+	if path.Base(rpath) == "." {
+		rpath = strings.TrimSuffix(rpath, ".")
+	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath))
 	wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig)
@@ -172,6 +195,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		CanHaveEmptyDirectories: true,
 		SetTier:                 true,
 		GetTier:                 true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -184,6 +208,7 @@ type Options struct {
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
+	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
 }
 
@@ -195,7 +220,7 @@ type Fs struct {
 	root     string
 	opt      Options
 	features *fs.Features // optional features
-	cipher   Cipher
+	cipher   *Cipher
 }
 
 // Name of the remote (as passed into NewFs)
@@ -232,7 +257,7 @@ func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
 	*entries = append(*entries, f.newObject(obj))
 }
 
-// Encrypt an directory file name to entries.
+// Encrypt a directory file name to entries.
 func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
@@ -319,7 +344,7 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
 	// Encrypt the data into wrappedIn
-	wrappedIn, err := f.cipher.EncryptData(in)
+	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
 		return nil, err
 	}
@@ -343,7 +368,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	}
 
 	// Transfer the data
-	o, err := put(ctx, wrappedIn, f.newObjectInfo(src), options...)
+	o, err := put(ctx, wrappedIn, f.newObjectInfo(src, encrypter.nonce), options...)
 	if err != nil {
 		return nil, err
 	}
@@ -496,11 +521,11 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	if do == nil {
 		return nil, errors.New("can't PutUnchecked")
 	}
-	wrappedIn, err := f.cipher.EncryptData(in)
+	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
 		return nil, err
 	}
-	o, err := do(ctx, wrappedIn, f.newObjectInfo(src))
+	o, err := do(ctx, wrappedIn, f.newObjectInfo(src, encrypter.nonce))
 	if err != nil {
 		return nil, err
 	}
@@ -553,6 +578,37 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 	return f.cipher.DecryptFileName(encryptedFileName)
 }
 
+// computeHashWithNonce takes the nonce and encrypts the contents of
+// src with it, and calculates the hash given by HashType on the fly
+//
+// Note that we break lots of encapsulation in this function.
+func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Object, hashType hash.Type) (hashStr string, err error) {
+	// Open the src for input
+	in, err := src.Open(ctx)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to open src")
+	}
+	defer fs.CheckClose(in, &err)
+
+	// Now encrypt the src with the nonce
+	out, err := f.cipher.newEncrypter(in, &nonce)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make encrypter")
+	}
+
+	// pipe into hash
+	m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make hasher")
+	}
+	_, err = io.Copy(m, out)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to hash data")
+	}
+
+	return m.Sums()[hashType], nil
+}
+
 // ComputeHash takes the nonce from o, and encrypts the contents of
 // src with it, and calculates the hash given by HashType on the fly
 //
@@ -564,7 +620,7 @@ func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType
 	if err != nil {
 		return "", errors.Wrap(err, "failed to open object to read nonce")
 	}
-	d, err := f.cipher.(*cipher).newDecrypter(in)
+	d, err := f.cipher.newDecrypter(in)
 	if err != nil {
 		_ = in.Close()
 		return "", errors.Wrap(err, "failed to open object to read nonce")
@@ -589,30 +645,7 @@ func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType
 		return "", errors.Wrap(err, "failed to close nonce read")
 	}
 
-	// Open the src for input
-	in, err = src.Open(ctx)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to open src")
-	}
-	defer fs.CheckClose(in, &err)
-
-	// Now encrypt the src with the nonce
-	out, err := f.cipher.(*cipher).newEncrypter(in, &nonce)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to make encrypter")
-	}
-
-	// pipe into hash
-	m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
-	if err != nil {
-		return "", errors.Wrap(err, "failed to make hasher")
-	}
-	_, err = io.Copy(m, out)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to hash data")
-	}
-
-	return m.Sums()[hashType], nil
+	return f.computeHashWithNonce(ctx, nonce, src, hashType)
 }
 
 // MergeDirs merges the contents of all the directories passed
@@ -639,7 +672,7 @@ func (f *Fs) DirCacheFlush() {
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	do := f.Fs.Features().PublicLink
 	if do == nil {
 		return "", errors.New("PublicLink not supported")
@@ -647,9 +680,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
 	o, err := f.NewObject(ctx, remote)
 	if err != nil {
 		// assume it is a directory
-		return do(ctx, f.cipher.EncryptDirName(remote))
+		return do(ctx, f.cipher.EncryptDirName(remote), expire, unlink)
 	}
-	return do(ctx, o.(*Object).Object.Remote())
+	return do(ctx, o.(*Object).Object.Remote(), expire, unlink)
 }
 
 // ChangeNotify calls the passed function with a path
@@ -684,6 +717,67 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 	do(ctx, wrappedNotifyFunc, pollIntervalChan)
 }
 
+var commandHelp = []fs.CommandHelp{
+	{
+		Name:  "encode",
+		Short: "Encode the given filename(s)",
+		Long: `This encodes the filenames given as arguments returning a list of
+strings of the encoded results.
+
+Usage Example:
+
+    rclone backend encode crypt: file1 [file2...]
+    rclone rc backend/command command=encode fs=crypt: file1 [file2...]
+`,
+	},
+	{
+		Name:  "decode",
+		Short: "Decode the given filename(s)",
+		Long: `This decodes the filenames given as arguments returning a list of
+strings of the decoded results. It will return an error if any of the
+inputs are invalid.
+
+Usage Example:
+
+    rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
+    rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
+`,
+	},
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "decode":
+		out := make([]string, 0, len(arg))
+		for _, encryptedFileName := range arg {
+			fileName, err := f.DecryptFileName(encryptedFileName)
+			if err != nil {
+				return out, errors.Wrap(err, fmt.Sprintf("Failed to decrypt : %s", encryptedFileName))
+			}
+			out = append(out, fileName)
+		}
+		return out, nil
+	case "encode":
+		out := make([]string, 0, len(arg))
+		for _, fileName := range arg {
+			encryptedFileName := f.EncryptFileName(fileName)
+			out = append(out, encryptedFileName)
+		}
+		return out, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -825,13 +919,15 @@ func (f *Fs) Disconnect(ctx context.Context) error {
 // This encrypts the remote name and adjusts the size
 type ObjectInfo struct {
 	fs.ObjectInfo
-	f *Fs
+	f     *Fs
+	nonce nonce
 }
 
-func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo {
+func (f *Fs) newObjectInfo(src fs.ObjectInfo, nonce nonce) *ObjectInfo {
 	return &ObjectInfo{
 		ObjectInfo: src,
 		f:          f,
+		nonce:      nonce,
 	}
 }
 
@@ -857,6 +953,23 @@ func (o *ObjectInfo) Size() int64 {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
+	var srcObj fs.Object
+	var ok bool
+	// Get the underlying object if there is one
+	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
+		// Prefer direct interface assertion
+	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
+		// Otherwise likely is an operations.OverrideRemote
+		srcObj = do.UnWrap()
+	} else {
+		return "", nil
+	}
+	// if this is wrapping a local object then we work out the hash
+	if srcObj.Fs().Features().IsLocal {
+		// Read the data and encrypt it to calculate the hash
+		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
+		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)
+	}
 	return "", nil
 }
 
@@ -895,6 +1008,7 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.Commander       = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)

backend/crypt/crypt_internal_test.go

@@ -0,0 +1,143 @@
+package crypt
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testWrapper struct {
+	fs.ObjectInfo
+}
+
+// UnWrap returns the Object that this Object is wrapping or nil if it
+// isn't wrapping anything
+func (o testWrapper) UnWrap() fs.Object {
+	if o, ok := o.ObjectInfo.(fs.Object); ok {
+		return o
+	}
+	return nil
+}
+
+// Create a temporary local fs to upload things from
+
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
+	localFs, err := fs.TemporaryLocalFs()
+	require.NoError(t, err)
+	cleanup = func() {
+		require.NoError(t, localFs.Rmdir(context.Background(), ""))
+	}
+	return localFs, cleanup
+}
+
+// Upload a file to a remote
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
+	inBuf := bytes.NewBufferString(contents)
+	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
+	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
+	obj, err := f.Put(context.Background(), inBuf, upSrc)
+	require.NoError(t, err)
+	cleanup = func() {
+		require.NoError(t, obj.Remove(context.Background()))
+	}
+	return obj, cleanup
+}
+
+// Test the ObjectInfo
+func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
+	var (
+		contents = random.String(100)
+		path     = "hash_test_object"
+		ctx      = context.Background()
+	)
+	if wrap {
+		path = "_wrap"
+	}
+
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
+
+	obj, cleanupObj := uploadFile(t, localFs, path, contents)
+	defer cleanupObj()
+
+	// encrypt the data
+	inBuf := bytes.NewBufferString(contents)
+	var outBuf bytes.Buffer
+	enc, err := f.cipher.newEncrypter(inBuf, nil)
+	require.NoError(t, err)
+	nonce := enc.nonce // read the nonce at the start
+	_, err = io.Copy(&outBuf, enc)
+	require.NoError(t, err)
+
+	var oi fs.ObjectInfo = obj
+	if wrap {
+		// wrap the object in an fs.ObjectUnwrapper if required
+		oi = testWrapper{oi}
+	}
+
+	// wrap the object in a crypt for upload using the nonce we
+	// saved from the encryptor
+	src := f.newObjectInfo(oi, nonce)
+
+	// Test ObjectInfo methods
+	assert.Equal(t, int64(outBuf.Len()), src.Size())
+	assert.Equal(t, f, src.Fs())
+	assert.NotEqual(t, path, src.Remote())
+
+	// Test ObjectInfo.Hash
+	wantHash := md5.Sum(outBuf.Bytes())
+	gotHash, err := src.Hash(ctx, hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
+}
+
+func testComputeHash(t *testing.T, f *Fs) {
+	var (
+		contents = random.String(100)
+		path     = "compute_hash_test"
+		ctx      = context.Background()
+		hashType = f.Fs.Hashes().GetOne()
+	)
+
+	if hashType == hash.None {
+		t.Skipf("%v: does not support hashes", f.Fs)
+	}
+
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
+
+	// Upload a file to localFs as a test object
+	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
+	defer cleanupLocalObj()
+
+	// Upload the same data to the remote Fs also
+	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
+	defer cleanupRemoteObj()
+
+	// Calculate the expected Hash of the remote object
+	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)
+	require.NoError(t, err)
+
+	// Test computed hash matches remote object hash
+	remoteObjHash, err := remoteObj.(*Object).Object.Hash(ctx, hashType)
+	require.NoError(t, err)
+	assert.Equal(t, remoteObjHash, computedHash)
+}
+
+// InternalTest is called by fstests.Run to extra tests
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("ObjectInfo", func(t *testing.T) { testObjectInfo(t, f, false) })
+	t.Run("ObjectInfoWrap", func(t *testing.T) { testObjectInfo(t, f, true) })
+	t.Run("ComputeHash", func(t *testing.T) { testComputeHash(t, f) })
+}

backend/drive/drive_internal_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
@@ -268,6 +269,98 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 	}
 }
 
+// TestIntegration/FsMkdir/FsPutFiles/Internal/Shortcuts
+func (f *Fs) InternalTestShortcuts(t *testing.T) {
+	const (
+		// from fstest/fstests/fstests.go
+		existingDir    = "hello? sausage"
+		existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
+		existingSubDir = "êé"
+	)
+	ctx := context.Background()
+	srcObj, err := f.NewObject(ctx, existingFile)
+	require.NoError(t, err)
+	srcHash, err := srcObj.Hash(ctx, hash.MD5)
+	require.NoError(t, err)
+	assert.NotEqual(t, "", srcHash)
+	t.Run("Errors", func(t *testing.T) {
+		_, err := f.makeShortcut(ctx, "", f, "")
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "can't be root")
+
+		_, err = f.makeShortcut(ctx, "notfound", f, "dst")
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "can't find source")
+
+		_, err = f.makeShortcut(ctx, existingFile, f, existingFile)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "not overwriting")
+		assert.Contains(t, err.Error(), "existing file")
+
+		_, err = f.makeShortcut(ctx, existingFile, f, existingDir)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "not overwriting")
+		assert.Contains(t, err.Error(), "existing directory")
+	})
+	t.Run("File", func(t *testing.T) {
+		dstObj, err := f.makeShortcut(ctx, existingFile, f, "shortcut.txt")
+		require.NoError(t, err)
+		require.NotNil(t, dstObj)
+		assert.Equal(t, "shortcut.txt", dstObj.Remote())
+		dstHash, err := dstObj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, srcHash, dstHash)
+		require.NoError(t, dstObj.Remove(ctx))
+	})
+	t.Run("Dir", func(t *testing.T) {
+		dstObj, err := f.makeShortcut(ctx, existingDir, f, "shortcutdir")
+		require.NoError(t, err)
+		require.Nil(t, dstObj)
+		entries, err := f.List(ctx, "shortcutdir")
+		require.NoError(t, err)
+		require.Equal(t, 1, len(entries))
+		require.Equal(t, "shortcutdir/"+existingSubDir, entries[0].Remote())
+		require.NoError(t, f.Rmdir(ctx, "shortcutdir"))
+	})
+	t.Run("Command", func(t *testing.T) {
+		_, err := f.Command(ctx, "shortcut", []string{"one"}, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "need exactly 2 arguments")
+
+		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
+			"target": "doesnotexistremote:",
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "couldn't find target")
+
+		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
+			"target": ".",
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "target is not a drive backend")
+
+		dstObjI, err := f.Command(ctx, "shortcut", []string{existingFile, "shortcut2.txt"}, map[string]string{
+			"target": fs.ConfigString(f),
+		})
+		require.NoError(t, err)
+		dstObj := dstObjI.(*Object)
+		assert.Equal(t, "shortcut2.txt", dstObj.Remote())
+		dstHash, err := dstObj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, srcHash, dstHash)
+		require.NoError(t, dstObj.Remove(ctx))
+
+		dstObjI, err = f.Command(ctx, "shortcut", []string{existingFile, "shortcut3.txt"}, nil)
+		require.NoError(t, err)
+		dstObj = dstObjI.(*Object)
+		assert.Equal(t, "shortcut3.txt", dstObj.Remote())
+		dstHash, err = dstObj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, srcHash, dstHash)
+		require.NoError(t, dstObj.Remove(ctx))
+	})
+}
+
 func (f *Fs) InternalTest(t *testing.T) {
 	// These tests all depend on each other so run them as nested tests
 	t.Run("DocumentImport", func(t *testing.T) {
@@ -282,6 +375,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 			})
 		})
 	})
+	t.Run("Shortcuts", f.InternalTestShortcuts)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/drive/upload.go

@@ -11,15 +11,15 @@
 package drive
 
 import (
+	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strconv"
 
-	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/readers"
@@ -50,7 +50,7 @@ type resumableUpload struct {
 }
 
 // Upload the io.Reader in of size bytes with contentType and info
-func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
+func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType, fileID, remote string, info *drive.File) (*drive.File, error) {
 	params := url.Values{
 		"alt":        {"json"},
 		"uploadType": {"resumable"},
@@ -81,18 +81,21 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string
 		if err != nil {
 			return false, err
 		}
+		req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 		googleapi.Expand(req.URL, map[string]string{
 			"fileId": fileID,
 		})
 		req.Header.Set("Content-Type", "application/json; charset=UTF-8")
 		req.Header.Set("X-Upload-Content-Type", contentType)
-		req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
+		if size >= 0 {
+			req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
+		}
 		res, err = f.client.Do(req)
 		if err == nil {
 			defer googleapi.CloseBody(res)
 			err = googleapi.CheckResponse(res)
 		}
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -106,60 +109,31 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType, fileID, remote string
 		MediaType:     contentType,
 		ContentLength: size,
 	}
-	return rx.Upload()
+	return rx.Upload(ctx)
 }
 
 // Make an http.Request for the range passed in
-func (rx *resumableUpload) makeRequest(start int64, body io.ReadSeeker, reqSize int64) *http.Request {
+func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io.ReadSeeker, reqSize int64) *http.Request {
 	req, _ := http.NewRequest("POST", rx.URI, body)
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	req.ContentLength = reqSize
+	totalSize := "*"
+	if rx.ContentLength >= 0 {
+		totalSize = strconv.FormatInt(rx.ContentLength, 10)
+	}
 	if reqSize != 0 {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize))
 	} else {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize))
 	}
 	req.Header.Set("Content-Type", rx.MediaType)
 	return req
 }
 
-// rangeRE matches the transfer status response from the server. $1 is
-// the last byte index uploaded.
-var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
-
-// Query drive for the amount transferred so far
-//
-// If error is nil, then start should be valid
-func (rx *resumableUpload) transferStatus() (start int64, err error) {
-	req := rx.makeRequest(0, nil, 0)
-	res, err := rx.f.client.Do(req)
-	if err != nil {
-		return 0, err
-	}
-	defer googleapi.CloseBody(res)
-	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
-		return rx.ContentLength, nil
-	}
-	if res.StatusCode != statusResumeIncomplete {
-		err = googleapi.CheckResponse(res)
-		if err != nil {
-			return 0, err
-		}
-		return 0, errors.Errorf("unexpected http return code %v", res.StatusCode)
-	}
-	Range := res.Header.Get("Range")
-	if m := rangeRE.FindStringSubmatch(Range); len(m) == 2 {
-		start, err = strconv.ParseInt(m[1], 10, 64)
-		if err == nil {
-			return start, nil
-		}
-	}
-	return 0, errors.Errorf("unable to parse range %q", Range)
-}
-
 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
-func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
+func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
 	_, _ = chunk.Seek(0, io.SeekStart)
-	req := rx.makeRequest(start, chunk, chunkSize)
+	req := rx.makeRequest(ctx, start, chunk, chunkSize)
 	res, err := rx.f.client.Do(req)
 	if err != nil {
 		return 599, err
@@ -192,23 +166,45 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 
 // Upload uploads the chunks from the input
 // It retries each chunk using the pacer and --low-level-retries
-func (rx *resumableUpload) Upload() (*drive.File, error) {
+func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 	start := int64(0)
 	var StatusCode int
 	var err error
 	buf := make([]byte, int(rx.f.opt.ChunkSize))
-	for start < rx.ContentLength {
-		reqSize := rx.ContentLength - start
-		if reqSize >= int64(rx.f.opt.ChunkSize) {
-			reqSize = int64(rx.f.opt.ChunkSize)
+	for finished := false; !finished; {
+		var reqSize int64
+		var chunk io.ReadSeeker
+		if rx.ContentLength >= 0 {
+			// If size known use repeatable reader for smoother bwlimit
+			if start >= rx.ContentLength {
+				break
+			}
+			reqSize = rx.ContentLength - start
+			if reqSize >= int64(rx.f.opt.ChunkSize) {
+				reqSize = int64(rx.f.opt.ChunkSize)
+			}
+			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
+		} else {
+			// If size unknown read into buffer
+			var n int
+			n, err = readers.ReadFill(rx.Media, buf)
+			if err == io.EOF {
+				// Send the last chunk with the correct ContentLength
+				// otherwise Google doesn't know we've finished
+				rx.ContentLength = start + int64(n)
+				finished = true
+			} else if err != nil {
+				return nil, err
+			}
+			reqSize = int64(n)
+			chunk = bytes.NewReader(buf[:reqSize])
 		}
-		chunk := readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
 
 		// Transfer the chunk
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
-			StatusCode, err = rx.transferChunk(start, chunk, reqSize)
-			again, err := shouldRetry(err)
+			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
+			again, err := rx.f.shouldRetry(err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
 				err = nil

backend/dropbox/dropbox.go

@@ -39,6 +39,7 @@ import (
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
 	"github.com/pkg/errors"
+	"github.com/rclone/rclone/backend/dropbox/dbhash"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -46,6 +47,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -102,16 +104,23 @@ var (
 	// A regexp matching path names for files Dropbox ignores
 	// See https://www.dropbox.com/en/help/145 - Ignored files
 	ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
+
+	// DbHashType is the hash.Type for Dropbox
+	DbHashType hash.Type
 )
 
 // Register with Fs
 func init() {
+	DbHashType = hash.RegisterHash("DropboxHash", 64, dbhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "dropbox",
 		Description: "Dropbox",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.ConfigNoOffline("dropbox", name, m, dropboxConfig)
+			opt := oauthutil.Options{
+				NoOffline: true,
+			}
+			err := oauthutil.Config("dropbox", name, m, dropboxConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -139,14 +148,28 @@ memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
 			Help:     "Impersonate this user when using a business account.",
 			Default:  "",
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \
+			// as invalid characters.
+			// Testing revealed names with trailing spaces and the DEL character don't work.
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDel |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize   fs.SizeSuffix `config:"chunk_size"`
-	Impersonate string        `config:"impersonate"`
+	ChunkSize   fs.SizeSuffix        `config:"chunk_size"`
+	Impersonate string               `config:"impersonate"`
+	Enc         encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote dropbox server
@@ -205,7 +228,11 @@ func shouldRetry(err error) (bool, error) {
 		return false, err
 	}
 	baseErrString := errors.Cause(err).Error()
-	// handle any official Retry-After header from Dropbox's SDK first
+	// First check for Insufficient Space
+	if strings.Contains(baseErrString, "insufficient_space") {
+		return false, fserrors.FatalError(err)
+	}
+	// Then handle any official Retry-After header from Dropbox's SDK
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
@@ -372,14 +399,15 @@ func (f *Fs) setRoot(root string) {
 // getMetadata gets the metadata for a file or directory
 func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
 	err = f.pacer.Call(func() (bool, error) {
-		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{Path: objPath})
+		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
+			Path: f.opt.Enc.FromStandardPath(objPath),
+		})
 		return shouldRetry(err)
 	})
 	if err != nil {
 		switch e := err.(type) {
 		case files.GetMetadataAPIError:
-			switch e.EndpointError.Path.Tag {
-			case files.LookupErrorNotFound:
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
 				notFound = true
 				err = nil
 			}
@@ -466,7 +494,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	for {
 		if !started {
 			arg := files.ListFolderArg{
-				Path:      root,
+				Path:      f.opt.Enc.FromStandardPath(root),
 				Recursive: false,
 			}
 			if root == "/" {
@@ -479,8 +507,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if err != nil {
 				switch e := err.(type) {
 				case files.ListFolderAPIError:
-					switch e.EndpointError.Path.Tag {
-					case files.LookupErrorNotFound:
+					if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
 						err = fs.ErrorDirNotFound
 					}
 				}
@@ -517,7 +544,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 			// Only the last element is reliably cased in PathDisplay
 			entryPath := metadata.PathDisplay
-			leaf := path.Base(entryPath)
+			leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
 			remote := path.Join(dir, leaf)
 			if folderInfo != nil {
 				d := fs.NewDir(remote, time.Now())
@@ -575,7 +602,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 	// create it
 	arg2 := files.CreateFolderArg{
-		Path: root,
+		Path: f.opt.Enc.FromStandardPath(root),
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.CreateFolderV2(&arg2)
@@ -601,6 +628,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return errors.Wrap(err, "Rmdir")
 	}
 
+	root = f.opt.Enc.FromStandardPath(root)
 	// check directory empty
 	arg := files.ListFolderArg{
 		Path:      root,
@@ -657,9 +685,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Copy
-	arg := files.RelocationArg{}
-	arg.FromPath = srcObj.remotePath()
-	arg.ToPath = dstObj.remotePath()
+	arg := files.RelocationArg{
+		RelocationPath: files.RelocationPath{
+			FromPath: f.opt.Enc.FromStandardPath(srcObj.remotePath()),
+			ToPath:   f.opt.Enc.FromStandardPath(dstObj.remotePath()),
+		},
+	}
 	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
@@ -691,7 +722,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 func (f *Fs) Purge(ctx context.Context) (err error) {
 	// Let dropbox delete the filesystem tree
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: f.slashRoot})
+		_, err = f.srv.DeleteV2(&files.DeleteArg{
+			Path: f.opt.Enc.FromStandardPath(f.slashRoot),
+		})
 		return shouldRetry(err)
 	})
 	return err
@@ -720,9 +753,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Do the move
-	arg := files.RelocationArg{}
-	arg.FromPath = srcObj.remotePath()
-	arg.ToPath = dstObj.remotePath()
+	arg := files.RelocationArg{
+		RelocationPath: files.RelocationPath{
+			FromPath: f.opt.Enc.FromStandardPath(srcObj.remotePath()),
+			ToPath:   f.opt.Enc.FromStandardPath(dstObj.remotePath()),
+		},
+	}
 	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
@@ -746,11 +782,17 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
-	absPath := "/" + path.Join(f.Root(), remote)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
+	absPath := f.opt.Enc.FromStandardPath(path.Join(f.slashRoot, remote))
 	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
 	createArg := sharing.CreateSharedLinkWithSettingsArg{
 		Path: absPath,
+		// FIXME this gives settings_error/not_authorized/.. errors
+		// and the expires setting isn't in the documentation so remove
+		// for now.
+		// Settings: &sharing.SharedLinkSettings{
+		// 	Expires: time.Now().Add(time.Duration(expire)).UTC().Round(time.Second),
+		// },
 	}
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
@@ -758,7 +800,8 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 		return shouldRetry(err)
 	})
 
-	if err != nil && strings.Contains(err.Error(), sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
+	if err != nil && strings.Contains(err.Error(),
+		sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
 		fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
 		listArg := sharing.ListSharedLinksArg{
 			Path:       absPath,
@@ -820,9 +863,12 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	// ...apparently not necessary
 
 	// Do the move
-	arg := files.RelocationArg{}
-	arg.FromPath = srcPath
-	arg.ToPath = dstPath
+	arg := files.RelocationArg{
+		RelocationPath: files.RelocationPath{
+			FromPath: f.opt.Enc.FromStandardPath(srcPath),
+			ToPath:   f.opt.Enc.FromStandardPath(dstPath),
+		},
+	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.MoveV2(&arg)
 		return shouldRetry(err)
@@ -863,7 +909,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.Dropbox)
+	return hash.Set(DbHashType)
 }
 
 // ------------------------------------------------------------
@@ -888,7 +934,7 @@ func (o *Object) Remote() string {
 
 // Hash returns the dropbox special hash
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.Dropbox {
+	if t != DbHashType {
 		return "", hash.ErrUnsupported
 	}
 	err := o.readMetaData()
@@ -977,7 +1023,10 @@ func (o *Object) Storable() bool {
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	fs.FixRangeOption(options, o.bytes)
 	headers := fs.OpenOptionHeaders(options)
-	arg := files.DownloadArg{Path: o.remotePath(), ExtraHeaders: headers}
+	arg := files.DownloadArg{
+		Path:         o.fs.opt.Enc.FromStandardPath(o.remotePath()),
+		ExtraHeaders: headers,
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, in, err = o.fs.srv.Download(&arg)
 		return shouldRetry(err)
@@ -986,7 +1035,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	switch e := err.(type) {
 	case files.DownloadAPIError:
 		// Don't attempt to retry copyright violation errors
-		if e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
+		if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
 			return nil, fserrors.NoRetryError(err)
 		}
 	}
@@ -1087,6 +1136,13 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 			return false, nil
 		}
 		entry, err = o.fs.srv.UploadSessionFinish(args, chunk)
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
+			}
+		}
 		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
@@ -1104,10 +1160,9 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	remote := o.remotePath()
 	if ignoredFiles.MatchString(remote) {
-		fs.Logf(o, "File name disallowed - not uploading")
-		return nil
+		return fserrors.NoRetryError(errors.Errorf("file name %q is disallowed - not uploading", path.Base(remote)))
 	}
-	commitInfo := files.NewCommitInfo(o.remotePath())
+	commitInfo := files.NewCommitInfo(o.fs.opt.Enc.FromStandardPath(o.remotePath()))
 	commitInfo.Mode.Tag = "overwrite"
 	// The Dropbox API only accepts timestamps in UTC with second precision.
 	commitInfo.ClientModified = src.ModTime(ctx).UTC().Round(time.Second)
@@ -1132,7 +1187,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // Remove an object
 func (o *Object) Remove(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{Path: o.remotePath()})
+		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
+			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
+		})
 		return shouldRetry(err)
 	})
 	return err

backend/fichier/api.go

@@ -17,6 +17,7 @@ import (
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	429, // Too Many Requests.
+	403, // Forbidden (may happen when request limit is exceeded)
 	500, // Internal Server Error
 	502, // Bad Gateway
 	503, // Service Unavailable
@@ -32,7 +33,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 
 var isAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString
 
-func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) {
+func (f *Fs) getDownloadToken(ctx context.Context, url string) (*GetTokenResponse, error) {
 	request := DownloadRequest{
 		URL:    url,
 		Single: 1,
@@ -44,7 +45,7 @@ func (f *Fs) getDownloadToken(url string) (*GetTokenResponse, error) {
 
 	var token GetTokenResponse
 	err := f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, &request, &token)
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, &token)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -72,7 +73,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
 
 	var sharedFiles SharedFolderResponse
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, nil, &sharedFiles)
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, &sharedFiles)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -88,7 +89,7 @@ func (f *Fs) listSharedFiles(ctx context.Context, id string) (entries fs.DirEntr
 	return entries, nil
 }
 
-func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) {
+func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesList, err error) {
 	// fs.Debugf(f, "Requesting files for dir `%s`", directoryID)
 	request := ListFilesRequest{
 		FolderID: directoryID,
@@ -101,17 +102,21 @@ func (f *Fs) listFiles(directoryID int) (filesList *FilesList, err error) {
 
 	filesList = &FilesList{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, &request, filesList)
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, filesList)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
 	}
+	for i := range filesList.Items {
+		item := &filesList.Items[i]
+		item.Filename = f.opt.Enc.ToStandardName(item.Filename)
+	}
 
 	return filesList, nil
 }
 
-func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error) {
+func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *FoldersList, err error) {
 	// fs.Debugf(f, "Requesting folders for id `%s`", directoryID)
 
 	request := ListFolderRequest{
@@ -125,12 +130,17 @@ func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error)
 
 	foldersList = &FoldersList{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, &request, foldersList)
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, foldersList)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list folders")
 	}
+	foldersList.Name = f.opt.Enc.ToStandardName(foldersList.Name)
+	for i := range foldersList.SubFolders {
+		folder := &foldersList.SubFolders[i]
+		folder.Name = f.opt.Enc.ToStandardName(folder.Name)
+	}
 
 	// fs.Debugf(f, "Got FoldersList for id `%s`", directoryID)
 
@@ -138,11 +148,6 @@ func (f *Fs) listFolders(directoryID int) (foldersList *FoldersList, err error)
 }
 
 func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
-
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -153,12 +158,12 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
 		return nil, err
 	}
 
-	files, err := f.listFiles(folderID)
+	files, err := f.listFiles(ctx, folderID)
 	if err != nil {
 		return nil, err
 	}
 
-	folders, err := f.listFolders(folderID)
+	folders, err := f.listFolders(ctx, folderID)
 	if err != nil {
 		return nil, err
 	}
@@ -166,7 +171,6 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
 	entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders))
 
 	for i, item := range files.Items {
-		item.Filename = restoreReservedChars(item.Filename)
 		entries[i] = f.newObjectFromFile(ctx, dir, item)
 	}
 
@@ -176,7 +180,6 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
 			return nil, err
 		}
 
-		folder.Name = restoreReservedChars(folder.Name)
 		fullPath := getRemote(dir, folder.Name)
 		folderID := strconv.Itoa(folder.ID)
 
@@ -205,8 +208,8 @@ func getRemote(dir, fileName string) string {
 	return dir + "/" + fileName
 }
 
-func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse, err error) {
-	name := replaceReservedChars(leaf)
+func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) {
+	name := f.opt.Enc.FromStandardName(leaf)
 	// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
 
 	request := MakeFolderRequest{
@@ -221,7 +224,7 @@ func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse
 
 	response = &MakeFolderResponse{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, &request, response)
+		resp, err := f.rest.CallJSON(ctx, &opts, &request, response)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -233,7 +236,7 @@ func (f *Fs) makeFolder(leaf string, folderID int) (response *MakeFolderResponse
 	return response, err
 }
 
-func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKResponse, err error) {
+func (f *Fs) removeFolder(ctx context.Context, name string, folderID int) (response *GenericOKResponse, err error) {
 	// fs.Debugf(f, "Removing folder with id `%s`", directoryID)
 
 	request := &RemoveFolderRequest{
@@ -248,7 +251,7 @@ func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKRespons
 	response = &GenericOKResponse{}
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.rest.CallJSON(&opts, request, response)
+		resp, err = f.rest.CallJSON(ctx, &opts, request, response)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -263,7 +266,7 @@ func (f *Fs) removeFolder(name string, folderID int) (response *GenericOKRespons
 	return response, nil
 }
 
-func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) {
+func (f *Fs) deleteFile(ctx context.Context, url string) (response *GenericOKResponse, err error) {
 	request := &RemoveFileRequest{
 		Files: []RmFile{
 			{url},
@@ -277,7 +280,7 @@ func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) {
 
 	response = &GenericOKResponse{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, request, response)
+		resp, err := f.rest.CallJSON(ctx, &opts, request, response)
 		return shouldRetry(resp, err)
 	})
 
@@ -290,7 +293,7 @@ func (f *Fs) deleteFile(url string) (response *GenericOKResponse, err error) {
 	return response, nil
 }
 
-func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) {
+func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse, err error) {
 	// fs.Debugf(f, "Requesting Upload node")
 
 	opts := rest.Opts{
@@ -301,7 +304,7 @@ func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) {
 
 	response = &GetUploadNodeResponse{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, nil, response)
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -313,10 +316,10 @@ func (f *Fs) getUploadNode() (response *GetUploadNodeResponse, err error) {
 	return response, err
 }
 
-func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) {
+func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string, options ...fs.OpenOption) (response *http.Response, err error) {
 	// fs.Debugf(f, "Uploading File `%s`", fileName)
 
-	fileName = replaceReservedChars(fileName)
+	fileName = f.opt.Enc.FromStandardName(fileName)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
 		return nil, errors.New("Invalid UploadID")
@@ -331,6 +334,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID,
 		NoResponse:           true,
 		Body:                 in,
 		ContentLength:        &size,
+		Options:              options,
 		MultipartContentName: "file[]",
 		MultipartFileName:    fileName,
 		MultipartParams: map[string][]string{
@@ -343,7 +347,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID,
 	}
 
 	err = f.pacer.CallNoRetry(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, nil, nil)
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, nil)
 		return shouldRetry(resp, err)
 	})
 
@@ -356,7 +360,7 @@ func (f *Fs) uploadFile(in io.Reader, size int64, fileName, folderID, uploadID,
 	return response, err
 }
 
-func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
+func (f *Fs) endUpload(ctx context.Context, uploadID string, nodeurl string) (response *EndFileUploadResponse, err error) {
 	// fs.Debugf(f, "Ending File Upload `%s`", uploadID)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
@@ -377,7 +381,7 @@ func (f *Fs) endUpload(uploadID string, nodeurl string) (response *EndFileUpload
 
 	response = &EndFileUploadResponse{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.rest.CallJSON(&opts, nil, response)
+		resp, err := f.rest.CallJSON(ctx, &opts, nil, response)
 		return shouldRetry(resp, err)
 	})
 

backend/fichier/fichier.go

@@ -11,21 +11,24 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 )
 
 const (
-	rootID        = "0"
-	apiBaseURL    = "https://api.1fichier.com/v1"
-	minSleep      = 334 * time.Millisecond // 3 API calls per second is recommended
-	maxSleep      = 5 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
+	rootID         = "0"
+	apiBaseURL     = "https://api.1fichier.com/v1"
+	minSleep       = 400 * time.Millisecond // api is extremely rate limited now
+	maxSleep       = 5 * time.Second
+	decayConstant  = 2 // bigger for slower decay, exponential
+	attackConstant = 0 // start with max sleep
 )
 
 func init() {
@@ -35,25 +38,48 @@ func init() {
 		Config: func(name string, config configmap.Mapper) {
 		},
 		NewFs: NewFs,
-		Options: []fs.Option{
-			{
-				Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
-				Name: "api_key",
-			},
-			{
-				Help:     "If you want to download a shared folder, add this parameter",
-				Name:     "shared_folder",
-				Required: false,
-				Advanced: true,
-			},
-		},
+		Options: []fs.Option{{
+			Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
+			Name: "api_key",
+		}, {
+			Help:     "If you want to download a shared folder, add this parameter",
+			Name:     "shared_folder",
+			Required: false,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Characters that need escaping
+			//
+			// 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+			// 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
+			// 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
+			// 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
+			// 		'\'': '＇', // FULLWIDTH APOSTROPHE
+			// 		'$':  '＄', // FULLWIDTH DOLLAR SIGN
+			// 		'`':  '｀', // FULLWIDTH GRAVE ACCENT
+			//
+			// Leading space and trailing space
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeSingleQuote |
+				encoder.EncodeBackQuote |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeLtGt |
+				encoder.EncodeDollar |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	APIKey       string `config:"api_key"`
-	SharedFolder string `config:"shared_folder"`
+	APIKey       string               `config:"api_key"`
+	SharedFolder string               `config:"shared_folder"`
+	Enc          encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs is the interface a cloud storage system must provide
@@ -61,9 +87,9 @@ type Fs struct {
 	root       string
 	name       string
 	features   *fs.Features
+	opt        Options
 	dirCache   *dircache.DirCache
 	baseClient *http.Client
-	options    *Options
 	pacer      *fs.Pacer
 	rest       *rest.Client
 }
@@ -74,7 +100,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	if err != nil {
 		return "", false, err
 	}
-	folders, err := f.listFolders(folderID)
+	folders, err := f.listFolders(ctx, folderID)
 	if err != nil {
 		return "", false, err
 	}
@@ -95,7 +121,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	if err != nil {
 		return "", err
 	}
-	resp, err := f.makeFolder(leaf, folderID)
+	resp, err := f.makeFolder(ctx, leaf, folderID)
 	if err != nil {
 		return "", err
 	}
@@ -141,8 +167,7 @@ func (f *Fs) Features() *fs.Features {
 //
 // On Windows avoid single character remote names as they can be mixed
 // up with drive letters.
-func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error) {
-	root := replaceReservedChars(rootleaf)
+func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) {
 	opt := new(Options)
 	err := configstruct.Set(config, opt)
 	if err != nil {
@@ -160,8 +185,8 @@ func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error)
 	f := &Fs{
 		name:       name,
 		root:       root,
-		options:    opt,
-		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		opt:        *opt,
+		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
 		baseClient: &http.Client{},
 	}
 
@@ -174,7 +199,7 @@ func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error)
 
 	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
 
-	f.rest.SetHeader("Authorization", "Bearer "+f.options.APIKey)
+	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
 
 	f.dirCache = dircache.New(root, rootID, f)
 
@@ -224,8 +249,8 @@ func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error)
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	if f.options.SharedFolder != "" {
-		return f.listSharedFiles(ctx, f.options.SharedFolder)
+	if f.opt.SharedFolder != "" {
+		return f.listSharedFiles(ctx, f.opt.SharedFolder)
 	}
 
 	dirContent, err := f.listDir(ctx, dir)
@@ -239,7 +264,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -251,7 +276,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 	if err != nil {
 		return nil, err
 	}
-	files, err := f.listFiles(folderID)
+	files, err := f.listFiles(ctx, folderID)
 	if err != nil {
 		return nil, err
 	}
@@ -273,7 +298,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Put in to the remote path with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Put should either
 // return an error or upload it properly (rather than e.g. calling panic).
 //
@@ -298,28 +323,28 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
-	if size > int64(100E9) {
+	if size > int64(100e9) {
 		return nil, errors.New("File too big, cant upload")
 	} else if size == 0 {
 		return nil, fs.ErrorCantUploadEmptyFiles
 	}
 
-	nodeResponse, err := f.getUploadNode()
+	nodeResponse, err := f.getUploadNode(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
 
-	_, err = f.uploadFile(in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL)
+	_, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...)
 	if err != nil {
 		return nil, err
 	}
 
-	fileUploadResponse, err := f.endUpload(nodeResponse.ID, nodeResponse.URL)
+	fileUploadResponse, err := f.endUpload(ctx, nodeResponse.ID, nodeResponse.URL)
 	if err != nil {
 		return nil, err
 	}
@@ -346,7 +371,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 			Date:        time.Now().Format("2006-01-02 15:04:05"),
 			Filename:    link.Filename,
 			Pass:        0,
-			Size:        int(fileSize),
+			Size:        fileSize,
 			URL:         link.Download,
 		},
 	}, nil
@@ -364,13 +389,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 //
 // Shouldn't return an error if it already exists
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -378,11 +397,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 //
 // Return an error if it doesn't exist or isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -393,7 +407,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return err
 	}
 
-	_, err = f.removeFolder(dir, folderID)
+	_, err = f.removeFolder(ctx, dir, folderID)
 	if err != nil {
 		return err
 	}

backend/fichier/object.go

@@ -43,7 +43,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	return int64(o.file.Size)
+	return o.file.Size
 }
 
 // Fs returns read only access to the Fs that this object is part of
@@ -74,8 +74,8 @@ func (o *Object) SetModTime(context.Context, time.Time) error {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	fs.FixRangeOption(options, int64(o.file.Size))
-	downloadToken, err := o.fs.getDownloadToken(o.file.URL)
+	fs.FixRangeOption(options, o.file.Size)
+	downloadToken, err := o.fs.getDownloadToken(ctx, o.file.URL)
 
 	if err != nil {
 		return nil, err
@@ -89,7 +89,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.rest.Call(&opts)
+		resp, err = o.fs.rest.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 
@@ -101,7 +101,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
@@ -131,7 +131,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 func (o *Object) Remove(ctx context.Context) error {
 	// fs.Debugf(f, "Removing file `%s` with url `%s`", o.file.Filename, o.file.URL)
 
-	_, err := o.fs.deleteFile(o.file.URL)
+	_, err := o.fs.deleteFile(ctx, o.file.URL)
 
 	if err != nil {
 		return err

backend/fichier/replace.go

@@ -1,71 +0,0 @@
-/*
-Translate file names for 1fichier
-
-1Fichier reserved characters
-
-The following characters are 1Fichier reserved characters, and can't
-be used in 1Fichier  folder and file names.
-
-*/
-
-package fichier
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// 1Fichier has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-		'\'': '＇', // FULLWIDTH APOSTROPHE
-		'$':  '＄', // FULLWIDTH DOLLAR SIGN
-		'`':  '｀', // FULLWIDTH GRAVE ACCENT
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	invCharMap           map[rune]rune
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// file names can't start with space either
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Replace reserved characters
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/fichier/replace_test.go

@@ -1,24 +0,0 @@
-package fichier
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{"\"'<>/\\$`", `＂＇＜＞/＼＄｀`},
-		{" leading space", "␠leading space"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/fichier/structs.go

@@ -69,7 +69,7 @@ type SharedFolderResponse []SharedFile
 type SharedFile struct {
 	Filename string `json:"filename"`
 	Link     string `json:"link"`
-	Size     int    `json:"size"`
+	Size     int64  `json:"size"`
 }
 
 // EndFileUploadResponse is the response structure of the corresponding request
@@ -93,7 +93,7 @@ type File struct {
 	Date        string `json:"date"`
 	Filename    string `json:"filename"`
 	Pass        int    `json:"pass"`
-	Size        int    `json:"size"`
+	Size        int64  `json:"size"`
 	URL         string `json:"url"`
 }
 

backend/ftp/ftp.go

@@ -14,10 +14,12 @@ import (
 	"github.com/jlaffaye/ftp"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 )
@@ -28,54 +30,82 @@ func init() {
 		Name:        "ftp",
 		Description: "FTP Connection",
 		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "host",
-				Help:     "FTP host to connect to",
-				Required: true,
-				Examples: []fs.OptionExample{{
-					Value: "ftp.example.com",
-					Help:  "Connect to ftp.example.com",
-				}},
-			}, {
-				Name: "user",
-				Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
-			}, {
-				Name: "port",
-				Help: "FTP port, leave blank to use default (21)",
-			}, {
-				Name:       "pass",
-				Help:       "FTP password",
-				IsPassword: true,
-				Required:   true,
-			}, {
-				Name:    "tls",
-				Help:    "Use FTP over TLS (Implicit)",
-				Default: false,
-			}, {
-				Name:     "concurrency",
-				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
-				Default:  0,
-				Advanced: true,
-			}, {
-				Name:     "no_check_certificate",
-				Help:     "Do not verify the TLS certificate of the server",
-				Default:  false,
-				Advanced: true,
-			},
-		},
+		Options: []fs.Option{{
+			Name:     "host",
+			Help:     "FTP host to connect to",
+			Required: true,
+			Examples: []fs.OptionExample{{
+				Value: "ftp.example.com",
+				Help:  "Connect to ftp.example.com",
+			}},
+		}, {
+			Name: "user",
+			Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
+		}, {
+			Name: "port",
+			Help: "FTP port, leave blank to use default (21)",
+		}, {
+			Name:       "pass",
+			Help:       "FTP password",
+			IsPassword: true,
+			Required:   true,
+		}, {
+			Name: "tls",
+			Help: `Use FTPS over TLS (Implicit)
+When using implicit FTP over TLS the client will connect using TLS
+right from the start, which in turn breaks the compatibility with
+non-TLS-aware servers. This is usually served over port 990 rather
+than port 21. Cannot be used in combination with explicit FTP.`,
+			Default: false,
+		}, {
+			Name: "explicit_tls",
+			Help: `Use FTP over TLS (Explicit)
+When using explicit FTP over TLS the client explicitly request
+security from the server in order to upgrade a plain text connection
+to an encrypted one. Cannot be used in combination with implicit FTP.`,
+			Default: false,
+		}, {
+			Name:     "concurrency",
+			Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
+			Default:  0,
+			Advanced: true,
+		}, {
+			Name:     "no_check_certificate",
+			Help:     "Do not verify the TLS certificate of the server",
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     "disable_epsv",
+			Help:     "Disable using EPSV even if server advertises support",
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// The FTP protocol can't handle trailing spaces (for instance
+			// pureftpd turns them into _)
+			//
+			// proftpd can't handle '*' in file names
+			// pureftpd can't handle '[', ']' or '*'
+			Default: (encoder.Display |
+				encoder.EncodeRightSpace),
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string `config:"host"`
-	User              string `config:"user"`
-	Pass              string `config:"pass"`
-	Port              string `config:"port"`
-	TLS               bool   `config:"tls"`
-	Concurrency       int    `config:"concurrency"`
-	SkipVerifyTLSCert bool   `config:"no_check_certificate"`
+	Host              string               `config:"host"`
+	User              string               `config:"user"`
+	Pass              string               `config:"pass"`
+	Port              string               `config:"port"`
+	TLS               bool                 `config:"tls"`
+	ExplicitTLS       bool                 `config:"explicit_tls"`
+	Concurrency       int                  `config:"concurrency"`
+	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
+	DisableEPSV       bool                 `config:"disable_epsv"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote FTP server
@@ -134,12 +164,24 @@ func (f *Fs) Features() *fs.Features {
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
 	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
-	if f.opt.TLS {
+	if f.opt.TLS && f.opt.ExplicitTLS {
+		fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+	} else if f.opt.TLS {
 		tlsConfig := &tls.Config{
 			ServerName:         f.opt.Host,
 			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
 		}
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
+	} else if f.opt.ExplicitTLS {
+		tlsConfig := &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
+		}
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
+	}
+	if f.opt.DisableEPSV {
+		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
 	}
 	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
 	if err != nil {
@@ -169,7 +211,11 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 	if c != nil {
 		return c, nil
 	}
-	return f.ftpConnection()
+	c, err = f.ftpConnection()
+	if err != nil && f.opt.Concurrency > 0 {
+		f.tokens.Put()
+	}
+	return c, err
 }
 
 // Return an FTP connection to the pool
@@ -182,7 +228,13 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	if f.opt.Concurrency > 0 {
 		defer f.tokens.Put()
 	}
+	if pc == nil {
+		return
+	}
 	c := *pc
+	if c == nil {
+		return
+	}
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
@@ -295,10 +347,37 @@ func translateErrorDir(err error) error {
 	return err
 }
 
+// entryToStandard converts an incoming ftp.Entry to Standard encoding
+func (f *Fs) entryToStandard(entry *ftp.Entry) {
+	// Skip . and .. as we don't want these encoded
+	if entry.Name == "." || entry.Name == ".." {
+		return
+	}
+	entry.Name = f.opt.Enc.ToStandardName(entry.Name)
+	entry.Target = f.opt.Enc.ToStandardPath(entry.Target)
+}
+
+// dirFromStandardPath returns dir in encoded form.
+func (f *Fs) dirFromStandardPath(dir string) string {
+	// Skip . and .. as we don't want these encoded
+	if dir == "." || dir == ".." {
+		return dir
+	}
+	return f.opt.Enc.FromStandardPath(dir)
+}
+
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
 	fullPath := path.Join(f.root, remote)
+	if fullPath == "" || fullPath == "." || fullPath == "/" {
+		// if root, assume exists and synthesize an entry
+		return &ftp.Entry{
+			Name: "",
+			Type: ftp.EntryTypeFolder,
+			Time: time.Now(),
+		}, nil
+	}
 	dir := path.Dir(fullPath)
 	base := path.Base(fullPath)
 
@@ -306,12 +385,13 @@ func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "findItem")
 	}
-	files, err := c.List(dir)
+	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
 	for _, file := range files {
+		f.entryToStandard(file)
 		if file.Name == base {
 			return file, nil
 		}
@@ -366,7 +446,7 @@ func (f *Fs) dirExists(remote string) (exists bool, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
+	// defer log.Trace(dir, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
 	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "list")
@@ -378,7 +458,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	resultchan := make(chan []*ftp.Entry, 1)
 	errchan := make(chan error, 1)
 	go func() {
-		result, err := c.List(path.Join(f.root, dir))
+		result, err := c.List(f.dirFromStandardPath(path.Join(f.root, dir)))
 		f.putFtpConnection(&c, err)
 		if err != nil {
 			errchan <- err
@@ -415,6 +495,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	for i := range files {
 		object := files[i]
+		f.entryToStandard(object)
 		newremote := path.Join(dir, object.Name)
 		switch object.Type {
 		case ftp.EntryTypeFolder:
@@ -484,19 +565,21 @@ func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "getInfo")
 	}
-	files, err := c.List(dir)
+	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
 
 	for i := range files {
-		if files[i].Name == base {
+		file := files[i]
+		f.entryToStandard(file)
+		if file.Name == base {
 			info := &FileInfo{
 				Name:    remote,
-				Size:    files[i].Size,
-				ModTime: files[i].Time,
-				IsDir:   files[i].Type == ftp.EntryTypeFolder,
+				Size:    file.Size,
+				ModTime: file.Time,
+				IsDir:   file.Type == ftp.EntryTypeFolder,
 			}
 			return info, nil
 		}
@@ -506,6 +589,7 @@ func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
 
 // mkdir makes the directory and parents using unrooted paths
 func (f *Fs) mkdir(abspath string) error {
+	abspath = path.Clean(abspath)
 	if abspath == "." || abspath == "/" {
 		return nil
 	}
@@ -527,7 +611,7 @@ func (f *Fs) mkdir(abspath string) error {
 	if connErr != nil {
 		return errors.Wrap(connErr, "mkdir")
 	}
-	err = c.MakeDir(abspath)
+	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
 	switch errX := err.(type) {
 	case *textproto.Error:
@@ -563,7 +647,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return errors.Wrap(translateErrorFile(err), "Rmdir")
 	}
-	err = c.RemoveDir(path.Join(f.root, dir))
+	err = c.RemoveDir(f.dirFromStandardPath(path.Join(f.root, dir)))
 	f.putFtpConnection(&c, err)
 	return translateErrorDir(err)
 }
@@ -584,8 +668,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, errors.Wrap(err, "Move")
 	}
 	err = c.Rename(
-		path.Join(srcObj.fs.root, srcObj.remote),
-		path.Join(f.root, remote),
+		f.opt.Enc.FromStandardPath(path.Join(srcObj.fs.root, srcObj.remote)),
+		f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
 	)
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -638,8 +722,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return errors.Wrap(err, "DirMove")
 	}
 	err = c.Rename(
-		srcPath,
-		dstPath,
+		f.dirFromStandardPath(srcPath),
+		f.dirFromStandardPath(dstPath),
 	)
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -725,19 +809,23 @@ func (f *ftpReadCloser) Close() error {
 	case <-timer.C:
 		// if timer fired assume no error but connection dead
 		fs.Errorf(f.f, "Timeout when waiting for connection Close")
+		f.f.putFtpConnection(nil, nil)
 		return nil
 	}
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
+		f.f.putFtpConnection(nil, nil)
 	} else {
 		f.f.putFtpConnection(&f.c, nil)
 	}
 	// mask the error if it was caused by a premature close
+	// NB StatusAboutToSend is to work around a bug in pureftpd
+	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
 	switch errX := err.(type) {
 	case *textproto.Error:
 		switch errX.Code {
-		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable:
+		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
 		}
 	}
@@ -765,7 +853,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
-	fd, err := c.RetrFrom(path, uint64(offset))
+	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
 	if err != nil {
 		o.fs.putFtpConnection(&c, err)
 		return nil, errors.Wrap(err, "open")
@@ -800,10 +888,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return errors.Wrap(err, "Update")
 	}
-	err = c.Stor(path, in)
+	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	if err != nil {
 		_ = c.Quit() // toss this connection to avoid sync errors
 		remove()
+		o.fs.putFtpConnection(nil, err)
 		return errors.Wrap(err, "update stor")
 	}
 	o.fs.putFtpConnection(&c, nil)
@@ -830,7 +919,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		if err != nil {
 			return errors.Wrap(err, "Remove")
 		}
-		err = c.Delete(path)
+		err = c.Delete(o.fs.opt.Enc.FromStandardPath(path))
 		o.fs.putFtpConnection(&c, err)
 	}
 	return err

backend/ftp/ftp_test.go

@@ -5,13 +5,44 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/ftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTP:",
+		RemoteName: "TestFTPProftpd:",
 		NilObject:  (*ftp.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPRclone:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPPureftpd:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+// func TestIntegration4(t *testing.T) {
+// 	if *fstest.RemoteName != "" {
+// 		t.Skip("skipping as -remote is set")
+// 	}
+// 	fstests.Run(t, &fstests.Opt{
+// 		RemoteName: "TestFTPVsftpd:",
+// 		NilObject:  (*ftp.Object)(nil),
+// 	})
+// }

backend/googlecloudstorage/googlecloudstorage.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"os"
 	"path"
 	"strings"
 	"time"
@@ -37,6 +36,8 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"golang.org/x/oauth2"
@@ -81,7 +82,7 @@ func init() {
 			if saFile != "" || saCreds != "" {
 				return
 			}
-			err := oauthutil.Config("google cloud storage", name, m, storageConfig)
+			err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -97,7 +98,7 @@ func init() {
 			Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",
 		}, {
 			Name: "service_account_file",
-			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
 			Name: "service_account_credentials",
 			Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
@@ -241,24 +242,35 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "COLDLINE",
 				Help:  "Coldline storage class",
+			}, {
+				Value: "ARCHIVE",
+				Help:  "Archive storage class",
 			}, {
 				Value: "DURABLE_REDUCED_AVAILABILITY",
 				Help:  "Durable reduced availability storage class",
 			}},
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.Base |
+				encoder.EncodeCrLf |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ProjectNumber             string `config:"project_number"`
-	ServiceAccountFile        string `config:"service_account_file"`
-	ServiceAccountCredentials string `config:"service_account_credentials"`
-	ObjectACL                 string `config:"object_acl"`
-	BucketACL                 string `config:"bucket_acl"`
-	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
-	Location                  string `config:"location"`
-	StorageClass              string `config:"storage_class"`
+	ProjectNumber             string               `config:"project_number"`
+	ServiceAccountFile        string               `config:"service_account_file"`
+	ServiceAccountCredentials string               `config:"service_account_credentials"`
+	ObjectACL                 string               `config:"object_acl"`
+	BucketACL                 string               `config:"bucket_acl"`
+	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`
+	Location                  string               `config:"location"`
+	StorageClass              string               `config:"storage_class"`
+	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote storage server
@@ -349,7 +361,8 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
 // split returns bucket and bucketPath from the object
@@ -374,6 +387,7 @@ func (f *Fs) setRoot(root string) {
 
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+	ctx := context.TODO()
 	var oAuthClient *http.Client
 
 	// Parse config into Options struct
@@ -391,7 +405,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, errors.Wrap(err, "error opening service account credentials file")
 		}
@@ -437,8 +451,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	if f.rootBucket != "" && f.rootDirectory != "" {
 		// Check to see if the object exists
+		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.Get(f.rootBucket, f.rootDirectory).Do()
+			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
 			return shouldRetry(err)
 		})
 		if err == nil {
@@ -457,7 +472,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object, error) {
+func (f *Fs) newObjectWithInfo(ctx context.Context, remote string, info *storage.Object) (fs.Object, error) {
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -465,7 +480,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object,
 	if info != nil {
 		o.setMetaData(info)
 	} else {
-		err := o.readMetaData() // reads info and meta, returning an error
+		err := o.readMetaData(ctx) // reads info and meta, returning an error
 		if err != nil {
 			return nil, err
 		}
@@ -476,7 +491,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *storage.Object) (fs.Object,
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(remote, nil)
+	return f.newObjectWithInfo(ctx, remote, nil)
 }
 
 // listFn is called from list to handle an object.
@@ -504,7 +519,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	for {
 		var objects *storage.Objects
 		err = f.pacer.Call(func() (bool, error) {
-			objects, err = list.Do()
+			objects, err = list.Context(ctx).Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -521,6 +536,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				if !strings.HasSuffix(remote, "/") {
 					continue
 				}
+				remote = f.opt.Enc.ToStandardPath(remote)
 				if !strings.HasPrefix(remote, prefix) {
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
@@ -536,12 +552,13 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			}
 		}
 		for _, object := range objects.Items {
-			if !strings.HasPrefix(object.Name, prefix) {
+			remote := f.opt.Enc.ToStandardPath(object.Name)
+			if !strings.HasPrefix(remote, prefix) {
 				fs.Logf(f, "Odd name received %q", object.Name)
 				continue
 			}
-			remote := object.Name[len(prefix):]
-			isDirectory := strings.HasSuffix(remote, "/")
+			remote = remote[len(prefix):]
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
@@ -563,12 +580,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 }
 
 // Convert a list item into a DirEntry
-func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory bool) (fs.DirEntry, error) {
+func (f *Fs) itemToDirEntry(ctx context.Context, remote string, object *storage.Object, isDirectory bool) (fs.DirEntry, error) {
 	if isDirectory {
 		d := fs.NewDir(remote, time.Time{}).SetSize(int64(object.Size))
 		return d, nil
 	}
-	o, err := f.newObjectWithInfo(remote, object)
+	o, err := f.newObjectWithInfo(ctx, remote, object)
 	if err != nil {
 		return nil, err
 	}
@@ -579,7 +596,7 @@ func (f *Fs) itemToDirEntry(remote string, object *storage.Object, isDirectory b
 func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) {
 	// List the objects
 	err = f.list(ctx, bucket, directory, prefix, addBucket, false, func(remote string, object *storage.Object, isDirectory bool) error {
-		entry, err := f.itemToDirEntry(remote, object, isDirectory)
+		entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
 		if err != nil {
 			return err
 		}
@@ -605,14 +622,14 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 	for {
 		var buckets *storage.Buckets
 		err = f.pacer.Call(func() (bool, error) {
-			buckets, err = listBuckets.Do()
+			buckets, err = listBuckets.Context(ctx).Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
 			return nil, err
 		}
 		for _, bucket := range buckets.Items {
-			d := fs.NewDir(bucket.Name, time.Time{})
+			d := fs.NewDir(f.opt.Enc.ToStandardName(bucket.Name), time.Time{})
 			entries = append(entries, d)
 		}
 		if buckets.NextPageToken == "" {
@@ -664,7 +681,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	list := walk.NewListRHelper(callback)
 	listR := func(bucket, directory, prefix string, addBucket bool) error {
 		return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *storage.Object, isDirectory bool) error {
-			entry, err := f.itemToDirEntry(remote, object, isDirectory)
+			entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
 			if err != nil {
 				return err
 			}
@@ -731,7 +748,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 		// List something from the bucket to see if it exists.  Doing it like this enables the use of a
 		// service account that only has the "Storage Object Admin" role.  See #2193 for details.
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.List(bucket).MaxResults(1).Do()
+			_, err = f.svc.Objects.List(bucket).MaxResults(1).Context(ctx).Do()
 			return shouldRetry(err)
 		})
 		if err == nil {
@@ -766,7 +783,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) (err error) {
 			if !f.opt.BucketPolicyOnly {
 				insertBucket.PredefinedAcl(f.opt.BucketACL)
 			}
-			_, err = insertBucket.Do()
+			_, err = insertBucket.Context(ctx).Do()
 			return shouldRetry(err)
 		})
 	}, nil)
@@ -783,7 +800,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	}
 	return f.cache.Remove(bucket, func() error {
 		return f.pacer.Call(func() (bool, error) {
-			err = f.svc.Buckets.Delete(bucket).Do()
+			err = f.svc.Buckets.Delete(bucket).Context(ctx).Do()
 			return shouldRetry(err)
 		})
 	})
@@ -824,7 +841,11 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	var newObject *storage.Object
 	err = f.pacer.Call(func() (bool, error) {
-		newObject, err = f.svc.Objects.Copy(srcBucket, srcPath, dstBucket, dstPath, nil).Do()
+		copyObject := f.svc.Objects.Copy(srcBucket, srcPath, dstBucket, dstPath, nil)
+		if !f.opt.BucketPolicyOnly {
+			copyObject.DestinationPredefinedAcl(f.opt.ObjectACL)
+		}
+		newObject, err = copyObject.Context(ctx).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -907,25 +928,33 @@ func (o *Object) setMetaData(info *storage.Object) {
 	}
 }
 
-// readMetaData gets the metadata if it hasn't already been fetched
-//
-// it also sets the info
-func (o *Object) readMetaData() (err error) {
-	if !o.modTime.IsZero() {
-		return nil
-	}
+// readObjectInfo reads the definition for an object
+func (o *Object) readObjectInfo(ctx context.Context) (object *storage.Object, err error) {
 	bucket, bucketPath := o.split()
-	var object *storage.Object
 	err = o.fs.pacer.Call(func() (bool, error) {
-		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Do()
+		object, err = o.fs.svc.Objects.Get(bucket, bucketPath).Context(ctx).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
 		if gErr, ok := err.(*googleapi.Error); ok {
 			if gErr.Code == http.StatusNotFound {
-				return fs.ErrorObjectNotFound
+				return nil, fs.ErrorObjectNotFound
 			}
 		}
+		return nil, err
+	}
+	return object, nil
+}
+
+// readMetaData gets the metadata if it hasn't already been fetched
+//
+// it also sets the info
+func (o *Object) readMetaData(ctx context.Context) (err error) {
+	if !o.modTime.IsZero() {
+		return nil
+	}
+	object, err := o.readObjectInfo(ctx)
+	if err != nil {
 		return err
 	}
 	o.setMetaData(object)
@@ -937,7 +966,7 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime(ctx context.Context) time.Time {
-	err := o.readMetaData()
+	err := o.readMetaData(ctx)
 	if err != nil {
 		// fs.Logf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -954,16 +983,27 @@ func metadataFromModTime(modTime time.Time) map[string]string {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) (err error) {
-	// This only adds metadata so will perserve other metadata
-	bucket, bucketPath := o.split()
-	object := storage.Object{
-		Bucket:   bucket,
-		Name:     bucketPath,
-		Metadata: metadataFromModTime(modTime),
+	// read the complete existing object first
+	object, err := o.readObjectInfo(ctx)
+	if err != nil {
+		return err
 	}
+	// Add the mtime to the existing metadata
+	mtime := modTime.Format(timeFormatOut)
+	if object.Metadata == nil {
+		object.Metadata = make(map[string]string, 1)
+	}
+	object.Metadata[metaMtime] = mtime
+	// Copy the object to itself to update the metadata
+	// Using PATCH requires too many permissions
+	bucket, bucketPath := o.split()
 	var newObject *storage.Object
 	err = o.fs.pacer.Call(func() (bool, error) {
-		newObject, err = o.fs.svc.Objects.Patch(bucket, bucketPath, &object).Do()
+		copyObject := o.fs.svc.Objects.Copy(bucket, bucketPath, bucket, bucketPath, object)
+		if !o.fs.opt.BucketPolicyOnly {
+			copyObject.DestinationPredefinedAcl(o.fs.opt.ObjectACL)
+		}
+		newObject, err = copyObject.Context(ctx).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -984,6 +1024,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	fs.FixRangeOption(options, o.bytes)
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	var res *http.Response
@@ -1025,13 +1066,40 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentType: fs.MimeType(ctx, src),
 		Metadata:    metadataFromModTime(modTime),
 	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			object.CacheControl = value
+		case "content-disposition":
+			object.ContentDisposition = value
+		case "content-encoding":
+			object.ContentEncoding = value
+		case "content-language":
+			object.ContentLanguage = value
+		case "content-type":
+			object.ContentType = value
+		default:
+			const googMetaPrefix = "x-goog-meta-"
+			if strings.HasPrefix(lowerKey, googMetaPrefix) {
+				metaKey := lowerKey[len(googMetaPrefix):]
+				object.Metadata[metaKey] = value
+			} else {
+				fs.Errorf(o, "Don't know how to set key %q on upload", key)
+			}
+		}
+	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		insertObject := o.fs.svc.Objects.Insert(bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)
 		if !o.fs.opt.BucketPolicyOnly {
 			insertObject.PredefinedAcl(o.fs.opt.ObjectACL)
 		}
-		newObject, err = insertObject.Do()
+		newObject, err = insertObject.Context(ctx).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -1046,7 +1114,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 func (o *Object) Remove(ctx context.Context) (err error) {
 	bucket, bucketPath := o.split()
 	err = o.fs.pacer.Call(func() (bool, error) {
-		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Do()
+		err = o.fs.svc.Objects.Delete(bucket, bucketPath).Context(ctx).Do()
 		return shouldRetry(err)
 	})
 	return err

backend/googlephotos/albums_test.go

@@ -30,7 +30,7 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a1)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
+		"one": {a1},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1": a1,
@@ -39,7 +39,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"one": a1,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one"},
+		"": {"one"},
 	}, albums.path)
 
 	a2 := &api.Album{
@@ -49,8 +49,8 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a2)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
-		"two": []*api.Album{a2},
+		"one": {a1},
+		"two": {a2},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1": a1,
@@ -61,7 +61,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"two": a2,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one", "two"},
+		"": {"one", "two"},
 	}, albums.path)
 
 	// Add a duplicate
@@ -72,8 +72,8 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a2a)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
-		"two": []*api.Album{a2, a2a},
+		"one": {a1},
+		"two": {a2, a2a},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":  a1,
@@ -86,7 +86,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one", "two {2}", "two {2a}"},
+		"": {"one", "two {2}", "two {2a}"},
 	}, albums.path)
 
 	// Add a sub directory
@@ -97,9 +97,9 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":    a1,
@@ -114,8 +114,8 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	// Add a weird path
@@ -126,10 +126,10 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a0)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"{0}":     []*api.Album{a0},
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"{0}":     {a0},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"0":    a0,
@@ -146,8 +146,8 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}", "{0}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}", "{0}"},
+		"one": {"sub"},
 	}, albums.path)
 }
 
@@ -181,9 +181,9 @@ func TestAlbumsDel(t *testing.T) {
 	albums.add(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":    a1,
@@ -198,16 +198,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a1)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"2":    a2,
@@ -220,16 +220,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a2)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"2a":   a2a,
@@ -240,16 +240,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a2a)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1sub": a1sub,
@@ -258,16 +258,16 @@ func TestAlbumsDel(t *testing.T) {
 		"one/sub": a1sub,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one"},
-		"one": []string{"sub"},
+		"":    {"one"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{}, albums.byID)
 	assert.Equal(t, map[string]*api.Album{}, albums.byTitle)

backend/googlephotos/api/types.go

@@ -17,7 +17,7 @@ type Error struct {
 	Details ErrorDetails `json:"error"`
 }
 
-// Error statisfies error interface
+// Error satisfies error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Details.Message, e.Details.Code, e.Details.Status)
 }

backend/googlephotos/googlephotos.go

@@ -96,7 +96,7 @@ func init() {
 			}
 
 			// Do the oauth
-			err = oauthutil.Config("google photos", name, m, oauthConfig)
+			err = oauthutil.Config("google photos", name, m, oauthConfig, nil)
 			if err != nil {
 				golog.Fatalf("Failed to configure token: %v", err)
 			}
@@ -134,14 +134,20 @@ rclone mount needs to know the size of files in advance of reading
 them, so setting this flag when using rclone mount is recommended if
 you want to read the media.`,
 			Advanced: true,
+		}, {
+			Name:     "start_year",
+			Default:  2000,
+			Help:     `Year limits the photos to be downloaded to those which are uploaded after the given year`,
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ReadOnly bool `config:"read_only"`
-	ReadSize bool `config:"read_size"`
+	ReadOnly  bool `config:"read_only"`
+	ReadSize  bool `config:"read_size"`
+	StartYear int  `config:"start_year"`
 }
 
 // Fs represents a remote storage server
@@ -202,6 +208,11 @@ func (f *Fs) dirTime() time.Time {
 	return f.startTime
 }
 
+// startYear returns the start year
+func (f *Fs) startYear() int {
+	return f.opt.StartYear
+}
+
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	429, // Too Many Requests.
@@ -224,6 +235,10 @@ func errorHandler(resp *http.Response) error {
 	if err != nil {
 		body = nil
 	}
+	// Google sends 404 messages as images so be prepared for that
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
+		body = []byte("Image not found or broken")
+	}
 	var e = api.Error{
 		Details: api.ErrorDetails{
 			Code:    resp.StatusCode,
@@ -290,7 +305,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 }
 
 // fetchEndpoint gets the openid endpoint named from the Google config
-func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) {
+func (f *Fs) fetchEndpoint(ctx context.Context, name string) (endpoint string, err error) {
 	// Get openID config without auth
 	opts := rest.Opts{
 		Method:  "GET",
@@ -298,7 +313,7 @@ func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) {
 	}
 	var openIDconfig map[string]interface{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.unAuth.CallJSON(&opts, nil, &openIDconfig)
+		resp, err := f.unAuth.CallJSON(ctx, &opts, nil, &openIDconfig)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -316,7 +331,7 @@ func (f *Fs) fetchEndpoint(name string) (endpoint string, err error) {
 
 // UserInfo fetches info about the current user with oauth2
 func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err error) {
-	endpoint, err := f.fetchEndpoint("userinfo_endpoint")
+	endpoint, err := f.fetchEndpoint(ctx, "userinfo_endpoint")
 	if err != nil {
 		return nil, err
 	}
@@ -327,7 +342,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
 		RootURL: endpoint,
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, nil, &userInfo)
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &userInfo)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -338,7 +353,7 @@ func (f *Fs) UserInfo(ctx context.Context) (userInfo map[string]string, err erro
 
 // Disconnect kills the token and refresh token
 func (f *Fs) Disconnect(ctx context.Context) (err error) {
-	endpoint, err := f.fetchEndpoint("revocation_endpoint")
+	endpoint, err := f.fetchEndpoint(ctx, "revocation_endpoint")
 	if err != nil {
 		return err
 	}
@@ -358,7 +373,7 @@ func (f *Fs) Disconnect(ctx context.Context) (err error) {
 	}
 	var res interface{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(&opts, nil, &res)
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &res)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -423,7 +438,7 @@ func findID(name string) string {
 
 // list the albums into an internal cache
 // FIXME cache invalidation
-func (f *Fs) listAlbums(shared bool) (all *albums, err error) {
+func (f *Fs) listAlbums(ctx context.Context, shared bool) (all *albums, err error) {
 	f.albumsMu.Lock()
 	defer f.albumsMu.Unlock()
 	all, ok := f.albums[shared]
@@ -445,7 +460,7 @@ func (f *Fs) listAlbums(shared bool) (all *albums, err error) {
 		var result api.ListAlbums
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(&opts, nil, &result)
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 			return shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -482,7 +497,7 @@ type listFn func(remote string, object *api.MediaItem, isDirectory bool) error
 // dir is the starting directory, "" for root
 //
 // Set recurse to read sub directories
-func (f *Fs) list(filter api.SearchFilter, fn listFn) (err error) {
+func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err error) {
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/mediaItems:search",
@@ -494,7 +509,7 @@ func (f *Fs) list(filter api.SearchFilter, fn listFn) (err error) {
 		var result api.MediaItems
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(&opts, &filter, &result)
+			resp, err = f.srv.CallJSON(ctx, &opts, &filter, &result)
 			return shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -543,7 +558,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, remote string, item *api.MediaI
 // listDir lists a single directory
 func (f *Fs) listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error) {
 	// List the objects
-	err = f.list(filter, func(remote string, item *api.MediaItem, isDirectory bool) error {
+	err = f.list(ctx, filter, func(remote string, item *api.MediaItem, isDirectory bool) error {
 		entry, err := f.itemToDirEntry(ctx, prefix+remote, item, isDirectory)
 		if err != nil {
 			return err
@@ -638,7 +653,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 	var result api.Album
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, request, &result)
+		resp, err = f.srv.CallJSON(ctx, &opts, request, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -654,7 +669,7 @@ func (f *Fs) createAlbum(ctx context.Context, albumTitle string) (album *api.Alb
 func (f *Fs) getOrCreateAlbum(ctx context.Context, albumTitle string) (album *api.Album, err error) {
 	f.createMu.Lock()
 	defer f.createMu.Unlock()
-	albums, err := f.listAlbums(false)
+	albums, err := f.listAlbums(ctx, false)
 	if err != nil {
 		return nil, err
 	}
@@ -708,7 +723,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 		return err
 	}
 	albumTitle := match[1]
-	allAlbums, err := f.listAlbums(false)
+	allAlbums, err := f.listAlbums(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -773,7 +788,7 @@ func (o *Object) Size() int64 {
 		RootURL: o.downloadURL(),
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -824,7 +839,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 		var item api.MediaItem
 		var resp *http.Response
 		err = o.fs.pacer.Call(func() (bool, error) {
-			resp, err = o.fs.srv.CallJSON(&opts, nil, &item)
+			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &item)
 			return shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -901,7 +916,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		Options: options,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -943,8 +958,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Upload the media item in exchange for an UploadToken
 	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/uploads",
+		Method:  "POST",
+		Path:    "/uploads",
+		Options: options,
 		ExtraHeaders: map[string]string{
 			"X-Goog-Upload-File-Name": fileName,
 			"X-Goog-Upload-Protocol":  "raw",
@@ -954,9 +970,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var token []byte
 	var resp *http.Response
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		if err != nil {
-			_ = resp.Body.Close()
 			return shouldRetry(resp, err)
 		}
 		token, err = rest.ReadBody(resp)
@@ -987,7 +1002,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	var result api.BatchCreateResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(&opts, request, &result)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, request, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1004,7 +1019,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Add upload to internal storage
 	if pattern.isUpload {
+		o.fs.uploadedMu.Lock()
 		o.fs.uploaded.AddEntry(o)
+		o.fs.uploadedMu.Unlock()
 	}
 	return nil
 }
@@ -1030,7 +1047,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(&opts, &request, nil)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, &request, nil)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {

backend/googlephotos/googlephotos_test.go

@@ -12,6 +12,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -26,17 +27,6 @@ const (
 	fileNameUpload = "rclone-test-image2.jpg"
 )
 
-// Wrapper to override the remote for an object
-type overrideRemoteObject struct {
-	fs.Object
-	remote string
-}
-
-// Remote returns the overridden remote name
-func (o *overrideRemoteObject) Remote() string {
-	return o.remote
-}
-
 func TestIntegration(t *testing.T) {
 	ctx := context.Background()
 	fstest.Initialise()
@@ -66,7 +56,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, &overrideRemoteObject{srcObj, remote})
+			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -231,7 +221,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, &overrideRemoteObject{srcObj, remote})
+		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()

backend/googlephotos/pattern.go

@@ -20,9 +20,10 @@ import (
 // file pattern parsing
 type lister interface {
 	listDir(ctx context.Context, prefix string, filter api.SearchFilter) (entries fs.DirEntries, err error)
-	listAlbums(shared bool) (all *albums, err error)
+	listAlbums(ctx context.Context, shared bool) (all *albums, err error)
 	listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error)
 	dirTime() time.Time
+	startYear() int
 }
 
 // dirPattern describes a single directory pattern
@@ -52,6 +53,7 @@ var patterns = dirPatterns{
 				fs.NewDir(prefix+"album", f.dirTime()),
 				fs.NewDir(prefix+"shared-album", f.dirTime()),
 				fs.NewDir(prefix+"upload", f.dirTime()),
+				fs.NewDir(prefix+"feature", f.dirTime()),
 			}, nil
 		},
 	},
@@ -189,6 +191,28 @@ var patterns = dirPatterns{
 		re:     `^shared-album/(.+?)/([^/]+)$`,
 		isFile: true,
 	},
+	{
+		re: `^feature$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			return fs.DirEntries{
+				fs.NewDir(prefix+"favorites", f.dirTime()),
+			}, nil
+		},
+	},
+	{
+		re: `^feature/favorites$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			filter := featureFilter(ctx, f, match)
+			if err != nil {
+				return nil, err
+			}
+			return f.listDir(ctx, prefix, filter)
+		},
+	},
+	{
+		re:     `^feature/favorites/([^/]+)$`,
+		isFile: true,
+	},
 }.mustCompile()
 
 // mustCompile compiles the regexps in the dirPatterns
@@ -200,7 +224,7 @@ func (ds dirPatterns) mustCompile() dirPatterns {
 	return ds
 }
 
-// match finds the path passed in in the matching structure and
+// match finds the path passed in the matching structure and
 // returns the parameters and a pointer to the match, or nil.
 func (ds dirPatterns) match(root string, itemPath string, isFile bool) (match []string, prefix string, pattern *dirPattern) {
 	itemPath = strings.Trim(itemPath, "/")
@@ -222,11 +246,10 @@ func (ds dirPatterns) match(root string, itemPath string, isFile bool) (match []
 	return nil, "", nil
 }
 
-// Return the years from 2000 to today
-// FIXME make configurable?
+// Return the years from startYear to today
 func years(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
 	currentYear := f.dirTime().Year()
-	for year := 2000; year <= currentYear; year++ {
+	for year := f.startYear(); year <= currentYear; year++ {
 		entries = append(entries, fs.NewDir(prefix+fmt.Sprint(year), f.dirTime()))
 	}
 	return entries, nil
@@ -290,13 +313,31 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 	return sf, nil
 }
 
+// featureFilter creates a filter for the Feature enum
+//
+// The API only supports one feature, FAVORITES, so hardcode that feature
+//
+// https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
+func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {
+	sf = api.SearchFilter{
+		Filters: &api.Filters{
+			FeatureFilter: &api.FeatureFilter{
+				IncludedFeatures: []string{
+					"FAVORITES",
+				},
+			},
+		},
+	}
+	return sf
+}
+
 // Turns an albumPath into entries
 //
 // These can either be synthetic directory entries if the album path
 // is a prefix of another album, or actual files, or a combination of
 // the two.
 func albumsToEntries(ctx context.Context, f lister, shared bool, prefix string, albumPath string) (entries fs.DirEntries, err error) {
-	albums, err := f.listAlbums(shared)
+	albums, err := f.listAlbums(ctx, shared)
 	if err != nil {
 		return nil, err
 	}

backend/googlephotos/pattern_test.go

@@ -44,7 +44,7 @@ func (f *testLister) listDir(ctx context.Context, prefix string, filter api.Sear
 }
 
 // mock listAlbums for testing
-func (f *testLister) listAlbums(shared bool) (all *albums, err error) {
+func (f *testLister) listAlbums(ctx context.Context, shared bool) (all *albums, err error) {
 	return f.albums, nil
 }
 
@@ -59,6 +59,11 @@ func (f *testLister) dirTime() time.Time {
 	return startTime
 }
 
+// mock startYear for testing
+func (f *testLister) startYear() int {
+	return 2000
+}
+
 func TestPatternMatch(t *testing.T) {
 	for testNumber, test := range []struct {
 		// input
@@ -150,6 +155,38 @@ func TestPatternMatch(t *testing.T) {
 			wantPrefix:  "file.jpg/",
 			wantPattern: &patterns[5],
 		},
+		{
+			root:        "",
+			itemPath:    "feature",
+			isFile:      false,
+			wantMatch:   []string{"feature"},
+			wantPrefix:  "feature/",
+			wantPattern: &patterns[23],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature",
+			itemPath:    "favorites",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "favorites/",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "file.jpg",
+			isFile:      true,
+			wantMatch:   []string{"feature/favorites/file.jpg", "file.jpg"},
+			wantPrefix:  "file.jpg/",
+			wantPattern: &patterns[25],
+		},
 	} {
 		t.Run(fmt.Sprintf("#%d,root=%q,itemPath=%q,isFile=%v", testNumber, test.root, test.itemPath, test.isFile), func(t *testing.T) {
 			gotMatch, gotPrefix, gotPattern := patterns.match(test.root, test.itemPath, test.isFile)

backend/http/http.go

@@ -13,6 +13,7 @@ import (
 	"path"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -77,6 +78,26 @@ Note that this may cause rclone to confuse genuine HTML files with
 directories.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_head",
+			Help: `Don't use HEAD requests to find file sizes in dir listing
+
+If your site is being very slow to load then you can try this option.
+Normally rclone does a HEAD request for each potential file in a
+directory listing to:
+
+- find its size
+- check it really exists
+- check to see if it is a directory
+
+If you set this option, rclone will not do the HEAD request.  This will mean
+
+- directory listings are much quicker
+- rclone won't have the times or sizes of any files
+- some files that don't exist may be in the listing
+`,
+			Default:  false,
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -86,6 +107,7 @@ directories.`,
 type Options struct {
 	Endpoint string          `config:"url"`
 	NoSlash  bool            `config:"no_slash"`
+	NoHead   bool            `config:"no_head"`
 	Headers  fs.CommaSepList `config:"headers"`
 }
 
@@ -124,6 +146,7 @@ func statusError(res *http.Response, err error) error {
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+	ctx := context.TODO()
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -162,6 +185,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		// check to see if points to a file
 		req, err := http.NewRequest("HEAD", u.String(), nil)
 		if err == nil {
+			req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 			addHeaders(req, opt)
 			res, err := noRedir.Do(req)
 			err = statusError(res, err)
@@ -237,7 +261,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 		fs:     f,
 		remote: remote,
 	}
-	err := o.stat()
+	err := o.stat(ctx)
 	if err != nil {
 		return nil, err
 	}
@@ -355,7 +379,7 @@ func (f *Fs) addHeaders(req *http.Request) {
 }
 
 // Read the directory passed in
-func (f *Fs) readDir(dir string) (names []string, err error) {
+func (f *Fs) readDir(ctx context.Context, dir string) (names []string, err error) {
 	URL := f.url(dir)
 	u, err := url.Parse(URL)
 	if err != nil {
@@ -369,6 +393,7 @@ func (f *Fs) readDir(dir string) (names []string, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "readDir failed")
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	f.addHeaders(req)
 	res, err := f.httpClient.Do(req)
 	if err == nil {
@@ -408,34 +433,53 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if !strings.HasSuffix(dir, "/") && dir != "" {
 		dir += "/"
 	}
-	names, err := f.readDir(dir)
+	names, err := f.readDir(ctx, dir)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error listing %q", dir)
 	}
+	var (
+		entriesMu sync.Mutex // to protect entries
+		wg        sync.WaitGroup
+		in        = make(chan string, fs.Config.Checkers)
+	)
+	add := func(entry fs.DirEntry) {
+		entriesMu.Lock()
+		entries = append(entries, entry)
+		entriesMu.Unlock()
+	}
+	for i := 0; i < fs.Config.Checkers; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for remote := range in {
+				file := &Object{
+					fs:     f,
+					remote: remote,
+				}
+				switch err := file.stat(ctx); err {
+				case nil:
+					add(file)
+				case fs.ErrorNotAFile:
+					// ...found a directory not a file
+					add(fs.NewDir(remote, timeUnset))
+				default:
+					fs.Debugf(remote, "skipping because of error: %v", err)
+				}
+			}
+		}()
+	}
 	for _, name := range names {
 		isDir := name[len(name)-1] == '/'
 		name = strings.TrimRight(name, "/")
 		remote := path.Join(dir, name)
 		if isDir {
-			dir := fs.NewDir(remote, timeUnset)
-			entries = append(entries, dir)
+			add(fs.NewDir(remote, timeUnset))
 		} else {
-			file := &Object{
-				fs:     f,
-				remote: remote,
-			}
-			switch err = file.stat(); err {
-			case nil:
-				entries = append(entries, file)
-			case fs.ErrorNotAFile:
-				// ...found a directory not a file
-				dir := fs.NewDir(remote, timeUnset)
-				entries = append(entries, dir)
-			default:
-				fs.Debugf(remote, "skipping because of error: %v", err)
-			}
+			in <- remote
 		}
 	}
+	close(in)
+	wg.Wait()
 	return entries, nil
 }
 
@@ -492,12 +536,19 @@ func (o *Object) url() string {
 }
 
 // stat updates the info field in the Object
-func (o *Object) stat() error {
+func (o *Object) stat(ctx context.Context) error {
+	if o.fs.opt.NoHead {
+		o.size = -1
+		o.modTime = timeUnset
+		o.contentType = fs.MimeType(ctx, o)
+		return nil
+	}
 	url := o.url()
 	req, err := http.NewRequest("HEAD", url, nil)
 	if err != nil {
 		return errors.Wrap(err, "stat failed")
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	o.fs.addHeaders(req)
 	res, err := o.fs.httpClient.Do(req)
 	if err == nil && res.StatusCode == http.StatusNotFound {
@@ -546,6 +597,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	if err != nil {
 		return nil, errors.Wrap(err, "Open failed")
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 
 	// Add optional headers
 	for k, v := range fs.OpenOptionHeaders(options) {

backend/http/http_internal_test.go

@@ -166,8 +166,7 @@ func TestNewObject(t *testing.T) {
 	require.NoError(t, err)
 	tFile := fi.ModTime()
 
-	dt, ok := fstest.CheckTimeEqualWithPrecision(tObj, tFile, time.Second)
-	assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", o.Remote(), dt, time.Second, tObj, tFile, time.Second))
+	fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
 
 	// check object not found
 	o, err = f.NewObject(context.Background(), "not found.txt")

backend/hubic/auth.go

@@ -1,6 +1,7 @@
 package hubic
 
 import (
+	"context"
 	"net/http"
 	"time"
 
@@ -20,13 +21,13 @@ func newAuth(f *Fs) *auth {
 	}
 }
 
-// Request constructs a http.Request for authentication
+// Request constructs an http.Request for authentication
 //
 // returns nil for not needed
 func (a *auth) Request(*swift.Connection) (r *http.Request, err error) {
 	const retries = 10
 	for try := 1; try <= retries; try++ {
-		err = a.f.getCredentials()
+		err = a.f.getCredentials(context.TODO())
 		if err == nil {
 			break
 		}

backend/hubic/hubic.go

@@ -7,6 +7,7 @@ package hubic
 // to be revisted after some actual experience.
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -38,7 +39,7 @@ var (
 	// Description of how to auth for this app
 	oauthConfig = &oauth2.Config{
 		Scopes: []string{
-			"credentials.r", // Read Openstack credentials
+			"credentials.r", // Read OpenStack credentials
 		},
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  "https://api.hubic.com/oauth/auth/",
@@ -57,7 +58,7 @@ func init() {
 		Description: "Hubic",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("hubic", name, m, oauthConfig)
+			err := oauthutil.Config("hubic", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -75,8 +76,8 @@ func init() {
 // credentials is the JSON returned from the Hubic API to read the
 // OpenStack credentials
 type credentials struct {
-	Token    string `json:"token"`    // Openstack token
-	Endpoint string `json:"endpoint"` // Openstack endpoint
+	Token    string `json:"token"`    // OpenStack token
+	Endpoint string `json:"endpoint"` // OpenStack endpoint
 	Expires  string `json:"expires"`  // Expires date - eg "2015-11-09T14:24:56+01:00"
 }
 
@@ -115,11 +116,12 @@ func (f *Fs) String() string {
 // getCredentials reads the OpenStack Credentials using the Hubic API
 //
 // The credentials are read into the Fs
-func (f *Fs) getCredentials() (err error) {
+func (f *Fs) getCredentials(ctx context.Context) (err error) {
 	req, err := http.NewRequest("GET", "https://api.hubic.com/1.0/account/credentials", nil)
 	if err != nil {
 		return err
 	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	resp, err := f.client.Do(req)
 	if err != nil {
 		return err

backend/jottacloud/api/types.go

@@ -46,13 +46,57 @@ func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 // APIString returns Time string in Jottacloud API format
 func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
 
+// LoginToken is struct representing the login token generated in the WebUI
+type LoginToken struct {
+	Username      string `json:"username"`
+	Realm         string `json:"realm"`
+	WellKnownLink string `json:"well_known_link"`
+	AuthToken     string `json:"auth_token"`
+}
+
+// WellKnown contains some configuration parameters for setting up endpoints
+type WellKnown struct {
+	Issuer                                     string   `json:"issuer"`
+	AuthorizationEndpoint                      string   `json:"authorization_endpoint"`
+	TokenEndpoint                              string   `json:"token_endpoint"`
+	TokenIntrospectionEndpoint                 string   `json:"token_introspection_endpoint"`
+	UserinfoEndpoint                           string   `json:"userinfo_endpoint"`
+	EndSessionEndpoint                         string   `json:"end_session_endpoint"`
+	JwksURI                                    string   `json:"jwks_uri"`
+	CheckSessionIframe                         string   `json:"check_session_iframe"`
+	GrantTypesSupported                        []string `json:"grant_types_supported"`
+	ResponseTypesSupported                     []string `json:"response_types_supported"`
+	SubjectTypesSupported                      []string `json:"subject_types_supported"`
+	IDTokenSigningAlgValuesSupported           []string `json:"id_token_signing_alg_values_supported"`
+	UserinfoSigningAlgValuesSupported          []string `json:"userinfo_signing_alg_values_supported"`
+	RequestObjectSigningAlgValuesSupported     []string `json:"request_object_signing_alg_values_supported"`
+	ResponseNodesSupported                     []string `json:"response_modes_supported"`
+	RegistrationEndpoint                       string   `json:"registration_endpoint"`
+	TokenEndpointAuthMethodsSupported          []string `json:"token_endpoint_auth_methods_supported"`
+	TokenEndpointAuthSigningAlgValuesSupported []string `json:"token_endpoint_auth_signing_alg_values_supported"`
+	ClaimsSupported                            []string `json:"claims_supported"`
+	ClaimTypesSupported                        []string `json:"claim_types_supported"`
+	ClaimsParameterSupported                   bool     `json:"claims_parameter_supported"`
+	ScopesSupported                            []string `json:"scopes_supported"`
+	RequestParameterSupported                  bool     `json:"request_parameter_supported"`
+	RequestURIParameterSupported               bool     `json:"request_uri_parameter_supported"`
+	CodeChallengeMethodsSupported              []string `json:"code_challenge_methods_supported"`
+	TLSClientCertificateBoundAccessTokens      bool     `json:"tls_client_certificate_bound_access_tokens"`
+	IntrospectionEndpoint                      string   `json:"introspection_endpoint"`
+}
+
 // TokenJSON is the struct representing the HTTP response from OAuth2
 // providers returning a token in JSON form.
 type TokenJSON struct {
-	AccessToken  string `json:"access_token"`
-	TokenType    string `json:"token_type"`
-	RefreshToken string `json:"refresh_token"`
-	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
+	AccessToken      string `json:"access_token"`
+	ExpiresIn        int32  `json:"expires_in"` // at least PayPal returns string, while most return number
+	RefreshExpiresIn int32  `json:"refresh_expires_in"`
+	RefreshToken     string `json:"refresh_token"`
+	TokenType        string `json:"token_type"`
+	IDToken          string `json:"id_token"`
+	NotBeforePolicy  int32  `json:"not-before-policy"`
+	SessionState     string `json:"session_state"`
+	Scope            string `json:"scope"`
 }
 
 // JSON structures returned by new API
@@ -120,6 +164,12 @@ type CustomerInfo struct {
 	IOSHash           string      `json:"ios_hash"`
 }
 
+// TrashResponse is returned when emptying the Trash
+type TrashResponse struct {
+	Folders int64 `json:"folders"`
+	Files   int64 `json:"files"`
+}
+
 // XML structures returned by the old API
 
 // Flag is a hacky type for checking if an attribute is present

backend/jottacloud/replace.go

@@ -1,77 +0,0 @@
-/*
-Translate file names for JottaCloud adapted from OneDrive
-
-
-The following characters are JottaCloud reserved characters, and can't
-be used in JottaCloud folder and file names.
-
-  jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"
-
-
-*/
-
-package jottacloud
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// Onedrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		'*':  '＊', // FULLWIDTH ASTERISK
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'?':  '？', // FULLWIDTH QUESTION MARK
-		':':  '：', // FULLWIDTH COLON
-		';':  '；', // FULLWIDTH SEMICOLON
-		'|':  '｜', // FULLWIDTH VERTICAL LINE
-		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	invCharMap           map[rune]rune
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-	fixEndingWithSpace   = regexp.MustCompile(` (/|$)`)
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// Filenames can't start with space
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Filenames can't end with space
-	in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/jottacloud/replace_test.go

@@ -1,28 +0,0 @@
-package jottacloud
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{`\*<>?:;|"`, `＼＊＜＞？：；｜＂`},
-		{`\*<>?:;|"\*<>?:;|"`, `＼＊＜＞？：；｜＂＼＊＜＞？：；｜＂`},
-		{" leading space", "␠leading space"},
-		{"trailing space ", "trailing space␠"},
-		{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
-		{"trailing space /trailing space /trailing space ", "trailing space␠/trailing space␠/trailing space␠"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/koofr/koofr.go

@@ -12,10 +12,13 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 
 	httpclient "github.com/koofr/go-httpclient"
 	koofrclient "github.com/koofr/go-koofrclient"
@@ -27,49 +30,56 @@ func init() {
 		Name:        "koofr",
 		Description: "Koofr",
 		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "endpoint",
-				Help:     "The Koofr API endpoint to use",
-				Default:  "https://app.koofr.net",
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "mountid",
-				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
-				Required: false,
-				Default:  "",
-				Advanced: true,
-			}, {
-				Name:     "setmtime",
-				Help:     "Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend.",
-				Default:  true,
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "user",
-				Help:     "Your Koofr user name",
-				Required: true,
-			}, {
-				Name:       "password",
-				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
-				IsPassword: true,
-				Required:   true,
-			},
-		},
+		Options: []fs.Option{{
+			Name:     "endpoint",
+			Help:     "The Koofr API endpoint to use",
+			Default:  "https://app.koofr.net",
+			Required: true,
+			Advanced: true,
+		}, {
+			Name:     "mountid",
+			Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
+			Required: false,
+			Default:  "",
+			Advanced: true,
+		}, {
+			Name:     "setmtime",
+			Help:     "Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend.",
+			Default:  true,
+			Required: true,
+			Advanced: true,
+		}, {
+			Name:     "user",
+			Help:     "Your Koofr user name",
+			Required: true,
+		}, {
+			Name:       "password",
+			Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
+			IsPassword: true,
+			Required:   true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
+		}},
 	})
 }
 
 // Options represent the configuration of the Koofr backend
 type Options struct {
-	Endpoint string `config:"endpoint"`
-	MountID  string `config:"mountid"`
-	User     string `config:"user"`
-	Password string `config:"password"`
-	SetMTime bool   `config:"setmtime"`
+	Endpoint string               `config:"endpoint"`
+	MountID  string               `config:"mountid"`
+	User     string               `config:"user"`
+	Password string               `config:"password"`
+	SetMTime bool                 `config:"setmtime"`
+	Enc      encoder.MultiEncoder `config:"encoding"`
 }
 
-// A Fs is a representation of a remote Koofr Fs
+// An Fs is a representation of a remote Koofr Fs
 type Fs struct {
 	name     string
 	mountID  string
@@ -240,9 +250,9 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
 }
 
-// fullPath constructs a full, absolute path from a Fs root relative path,
+// fullPath constructs a full, absolute path from an Fs root relative path,
 func (f *Fs) fullPath(part string) string {
-	return path.Join("/", f.root, part)
+	return f.opt.Enc.FromStandardPath(path.Join("/", f.root, part))
 }
 
 // NewFs constructs a new filesystem given a root path and configuration options
@@ -256,7 +266,9 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	if err != nil {
 		return nil, err
 	}
-	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
+	httpClient := httpclient.New()
+	httpClient.Client = fshttp.NewClient(fs.Config)
+	client := koofrclient.NewKoofrClientWithHTTPClient(opt.Endpoint, httpClient)
 	basicAuth := fmt.Sprintf("Basic %s",
 		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
 	client.HTTPClient.Headers.Set("Authorization", basicAuth)
@@ -293,7 +305,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		}
 		return nil, errors.New("Failed to find mount " + opt.MountID)
 	}
-	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
+	rootFile, err := f.client.FilesInfo(f.mountID, f.opt.Enc.FromStandardPath("/"+f.root))
 	if err == nil && rootFile.Type != "dir" {
 		f.root = dir(f.root)
 		err = fs.ErrorIsFile
@@ -311,13 +323,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	entries = make([]fs.DirEntry, len(files))
 	for i, file := range files {
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name))
 		if file.Type == "dir" {
-			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
+			entries[i] = fs.NewDir(remote, time.Unix(0, 0))
 		} else {
 			entries[i] = &Object{
 				fs:     f,
 				info:   file,
-				remote: path.Join(dir, file.Name),
+				remote: remote,
 			}
 		}
 	}
@@ -408,7 +421,7 @@ func translateErrorsObject(err error) error {
 }
 
 // mkdir creates a directory at the given remote path. Creates ancestors if
-// neccessary
+// necessary
 func (f *Fs) mkdir(fullPath string) error {
 	if fullPath == "/" {
 		return nil
@@ -590,7 +603,7 @@ func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link,
 }
 
 // PublicLink creates a public link to the remote path
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
 	if err != nil {
 		return "", translateErrorsDir(err)

backend/local/encode_darwin.go

@@ -0,0 +1,11 @@
+//+build darwin
+
+package local
+
+import "github.com/rclone/rclone/lib/encoder"
+
+// This is the encoding used by the local backend for macOS
+//
+// macOS can't store invalid UTF-8, it converts them into %XX encoding
+const defaultEnc = (encoder.Base |
+	encoder.EncodeInvalidUtf8)

backend/local/encode_other.go

@@ -0,0 +1,8 @@
+//+build !windows,!darwin
+
+package local
+
+import "github.com/rclone/rclone/lib/encoder"
+
+// This is the encoding used by the local backend for non windows platforms
+const defaultEnc = encoder.Base

backend/local/encode_windows.go

@@ -0,0 +1,33 @@
+//+build windows
+
+package local
+
+import "github.com/rclone/rclone/lib/encoder"
+
+// This is the encoding used by the local backend for windows platforms
+//
+// List of replaced characters:
+//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+//   : (colon)         -> '：' // FULLWIDTH COLON
+//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+//
+// Additionally names can't end with a period (.) or space ( ).
+// List of replaced characters:
+//   . (period)        -> '．' // FULLWIDTH FULL STOP
+//     (space)         -> '␠' // SYMBOL FOR SPACE
+//
+// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
+//
+// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
+const defaultEnc = (encoder.Base |
+	encoder.EncodeWin |
+	encoder.EncodeBackSlash |
+	encoder.EncodeCtl |
+	encoder.EncodeRightSpace |
+	encoder.EncodeRightPeriod |
+	encoder.EncodeInvalidUtf8)

backend/local/local.go

@@ -20,10 +20,12 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
 )
@@ -39,6 +41,7 @@ func init() {
 		Name:        "local",
 		Description: "Local Disk",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name: "nounc",
 			Help: "Disable UNC (long path names) conversion on Windows",
@@ -87,7 +90,24 @@ are being uploaded and aborts with a message which starts "can't copy
 
 However on some file systems this modification time check may fail (eg
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
-check can be disabled with this flag.`,
+check can be disabled with this flag.
+
+If this flag is set, rclone will use its best efforts to transfer a
+file which is being updated. If the file is only having things
+appended to it (eg a log) then rclone will transfer the log file with
+the size it had the first time rclone saw it.
+
+If the file is being modified throughout (not just appended to) then
+the transfer may fail with a hash check failure.
+
+In detail, once the file has had stat() called on it for the first
+time we:
+
+- Only transfer the size that stat gave
+- Only checksum the size that stat gave
+- Don't update the stat info for the file
+
+`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -115,6 +135,21 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_sparse",
+			Help: `Disable sparse files for multi-thread downloads
+
+On Windows platforms rclone will make sparse files when doing
+multi-thread downloads. This avoids long pauses on large files where
+the OS zeros the file. However sparse files may be undesirable as they
+cause disk fragmentation and can be slow to work with.`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default:  defaultEnc,
 		}},
 	}
 	fs.Register(fsi)
@@ -122,15 +157,17 @@ to override the default choice.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks    bool `config:"copy_links"`
-	TranslateSymlinks bool `config:"links"`
-	SkipSymlinks      bool `config:"skip_links"`
-	NoUTFNorm         bool `config:"no_unicode_normalization"`
-	NoCheckUpdated    bool `config:"no_check_updated"`
-	NoUNC             bool `config:"nounc"`
-	OneFileSystem     bool `config:"one_file_system"`
-	CaseSensitive     bool `config:"case_sensitive"`
-	CaseInsensitive   bool `config:"case_insensitive"`
+	FollowSymlinks    bool                 `config:"copy_links"`
+	TranslateSymlinks bool                 `config:"links"`
+	SkipSymlinks      bool                 `config:"skip_links"`
+	NoUTFNorm         bool                 `config:"no_unicode_normalization"`
+	NoCheckUpdated    bool                 `config:"no_check_updated"`
+	NoUNC             bool                 `config:"nounc"`
+	OneFileSystem     bool                 `config:"one_file_system"`
+	CaseSensitive     bool                 `config:"case_sensitive"`
+	CaseInsensitive   bool                 `config:"case_insensitive"`
+	NoSparse          bool                 `config:"no_sparse"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -142,19 +179,19 @@ type Fs struct {
 	dev         uint64              // device number of root node
 	precisionOk sync.Once           // Whether we need to read the precision
 	precision   time.Duration       // precision of local filesystem
-	wmu         sync.Mutex          // used for locking access to 'warned'.
+	warnedMu    sync.Mutex          // used for locking access to 'warned'.
 	warned      map[string]struct{} // whether we have warned about this string
+
 	// do os.Lstat or os.Stat
 	lstat          func(name string) (os.FileInfo, error)
-	dirNames       *mapper    // directory name mapping
 	objectHashesMu sync.Mutex // global lock for Object.hashes
 }
 
 // Object represents a local filesystem object
 type Object struct {
 	fs             *Fs    // The Fs this object is part of
-	remote         string // The remote path - properly UTF-8 encoded - for rclone
-	path           string // The local path - may not be properly UTF-8 encoded - for OS
+	remote         string // The remote path (encoded path)
+	path           string // The local path (OS path)
 	size           int64  // file metadata - always present
 	mode           os.FileMode
 	modTime        time.Time
@@ -183,14 +220,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	f := &Fs{
-		name:     name,
-		opt:      *opt,
-		warned:   make(map[string]struct{}),
-		dev:      devUnset,
-		lstat:    os.Lstat,
-		dirNames: newMapper(),
+		name:   name,
+		opt:    *opt,
+		warned: make(map[string]struct{}),
+		dev:    devUnset,
+		lstat:  os.Lstat,
 	}
-	f.root = f.cleanPath(root)
+	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
 		CaseInsensitive:         f.caseInsensitive(),
 		CanHaveEmptyDirectories: true,
@@ -235,12 +271,12 @@ func (f *Fs) Name() string {
 
 // Root of the remote (as passed into NewFs)
 func (f *Fs) Root() string {
-	return f.root
+	return f.opt.Enc.ToStandardPath(filepath.ToSlash(f.root))
 }
 
 // String converts this Fs to a string
 func (f *Fs) String() string {
-	return fmt.Sprintf("Local file system at %s", f.root)
+	return fmt.Sprintf("Local file system at %s", f.Root())
 }
 
 // Features returns the optional features of this Fs
@@ -268,33 +304,27 @@ func (f *Fs) caseInsensitive() bool {
 // and returns a new path, removing the suffix as needed,
 // It also returns whether this is a translated link at all
 //
-// for regular files, dstPath is returned unchanged
-func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
+// for regular files, localPath is returned unchanged
+func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
 	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
-	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
-	return newDstPath, isTranslatedLink
+	newLocalPath = strings.TrimSuffix(localPath, linkSuffix)
+	return newLocalPath, isTranslatedLink
 }
 
 // newObject makes a half completed Object
-//
-// if dstPath is empty then it is made from remote
-func (f *Fs) newObject(remote, dstPath string) *Object {
+func (f *Fs) newObject(remote string) *Object {
 	translatedLink := false
-
-	if dstPath == "" {
-		dstPath = f.cleanPath(filepath.Join(f.root, remote))
-	}
-	remote = f.cleanRemote(remote)
+	localPath := f.localPath(remote)
 
 	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for dstPath
-		dstPath, translatedLink = translateLink(remote, dstPath)
+		// Possibly receive a new name for localPath
+		localPath, translatedLink = translateLink(remote, localPath)
 	}
 
 	return &Object{
 		fs:             f,
 		remote:         remote,
-		path:           dstPath,
+		path:           localPath,
 		translatedLink: translatedLink,
 	}
 }
@@ -302,8 +332,8 @@ func (f *Fs) newObject(remote, dstPath string) *Object {
 // Return an Object from a path
 //
 // May return nil if an error occurred
-func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Object, error) {
-	o := f.newObject(remote, dstPath)
+func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) {
+	o := f.newObject(remote)
 	if info != nil {
 		o.setMetadata(info)
 	} else {
@@ -332,7 +362,7 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(remote, "", nil)
+	return f.newObjectWithInfo(remote, nil)
 }
 
 // List the objects and directories in dir into entries.  The
@@ -345,10 +375,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-
-	dir = f.dirNames.Load(dir)
-	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
-	remote := f.cleanRemote(dir)
+	fsDirPath := f.localPath(dir)
 	_, err = os.Stat(fsDirPath)
 	if err != nil {
 		return nil, fs.ErrorDirNotFound
@@ -360,7 +387,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		err = errors.Wrapf(err, "failed to open directory %q", dir)
 		fs.Errorf(dir, "%v", err)
 		if isPerm {
-			accounting.Stats(ctx).Error(fserrors.NoRetryError(err))
+			_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(err))
 			err = nil // ignore error but fail sync
 		}
 		return nil, err
@@ -396,7 +423,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					if fierr != nil {
 						err = errors.Wrapf(err, "failed to read directory %q", namepath)
 						fs.Errorf(dir, "%v", fierr)
-						accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
+						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue
 					}
 					fis = append(fis, fi)
@@ -410,16 +437,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		for _, fi := range fis {
 			name := fi.Name()
 			mode := fi.Mode()
-			newRemote := path.Join(remote, name)
-			newPath := filepath.Join(fsDirPath, name)
+			newRemote := f.cleanRemote(dir, name)
 			// Follow symlinks if required
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
-				fi, err = os.Stat(newPath)
+				localPath := filepath.Join(fsDirPath, name)
+				fi, err = os.Stat(localPath)
 				if os.IsNotExist(err) {
 					// Skip bad symlinks
 					err = fserrors.NoRetryError(errors.Wrap(err, "symlink"))
 					fs.Errorf(newRemote, "Listing error: %v", err)
-					accounting.Stats(ctx).Error(err)
+					err = accounting.Stats(ctx).Error(err)
 					continue
 				}
 				if err != nil {
@@ -431,7 +458,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
 				if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
-					d := fs.NewDir(f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), fi.ModTime())
+					d := fs.NewDir(newRemote, fi.ModTime())
 					entries = append(entries, d)
 				}
 			} else {
@@ -439,7 +466,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
 					newRemote += linkSuffix
 				}
-				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
+				fso, err := f.newObjectWithInfo(newRemote, fi)
 				if err != nil {
 					return nil, err
 				}
@@ -452,67 +479,28 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return entries, nil
 }
 
-// cleanRemote makes string a valid UTF-8 string for remote strings.
-//
-// Any invalid UTF-8 characters will be replaced with utf8.RuneError
-// It also normalises the UTF-8 and converts the slashes if necessary.
-func (f *Fs) cleanRemote(name string) string {
-	if !utf8.ValidString(name) {
-		f.wmu.Lock()
-		if _, ok := f.warned[name]; !ok {
-			fs.Logf(f, "Replacing invalid UTF-8 characters in %q", name)
-			f.warned[name] = struct{}{}
+func (f *Fs) cleanRemote(dir, filename string) (remote string) {
+	remote = path.Join(dir, f.opt.Enc.ToStandardName(filename))
+
+	if !utf8.ValidString(filename) {
+		f.warnedMu.Lock()
+		if _, ok := f.warned[remote]; !ok {
+			fs.Logf(f, "Replacing invalid UTF-8 characters in %q", remote)
+			f.warned[remote] = struct{}{}
 		}
-		f.wmu.Unlock()
-		name = string([]rune(name))
+		f.warnedMu.Unlock()
 	}
-	name = filepath.ToSlash(name)
-	return name
+	return
 }
 
-// mapper maps raw to cleaned directory names
-type mapper struct {
-	mu sync.RWMutex      // mutex to protect the below
-	m  map[string]string // map of un-normalised directory names
-}
-
-func newMapper() *mapper {
-	return &mapper{
-		m: make(map[string]string),
-	}
-}
-
-// Lookup a directory name to make a local name (reverses
-// cleanDirName)
-//
-// FIXME this is temporary before we make a proper Directory object
-func (m *mapper) Load(in string) string {
-	m.mu.RLock()
-	out, ok := m.m[in]
-	m.mu.RUnlock()
-	if ok {
-		return out
-	}
-	return in
-}
-
-// Cleans a directory name recording if it needed to be altered
-//
-// FIXME this is temporary before we make a proper Directory object
-func (m *mapper) Save(in, out string) string {
-	if in != out {
-		m.mu.Lock()
-		m.m[out] = in
-		m.mu.Unlock()
-	}
-	return out
+func (f *Fs) localPath(name string) string {
+	return filepath.Join(f.root, filepath.FromSlash(f.opt.Enc.FromStandardPath(name)))
 }
 
 // Put the Object to the local filesystem
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	remote := src.Remote()
 	// Temporary Object under construction - info filled in by Update()
-	o := f.newObject(remote, "")
+	o := f.newObject(src.Remote())
 	err := o.Update(ctx, in, src, options...)
 	if err != nil {
 		return nil, err
@@ -528,13 +516,13 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // Mkdir creates the directory if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	// FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go
-	root := f.cleanPath(filepath.Join(f.root, dir))
-	err := os.MkdirAll(root, 0777)
+	localPath := f.localPath(dir)
+	err := os.MkdirAll(localPath, 0777)
 	if err != nil {
 		return err
 	}
 	if dir == "" {
-		fi, err := f.lstat(root)
+		fi, err := f.lstat(localPath)
 		if err != nil {
 			return err
 		}
@@ -547,8 +535,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 //
 // If it isn't empty it will return an error
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	root := f.cleanPath(filepath.Join(f.root, dir))
-	return os.Remove(root)
+	return os.Remove(f.localPath(dir))
 }
 
 // Precision of the file system
@@ -644,7 +631,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Temporary Object under construction
-	dstObj := f.newObject(remote, "")
+	dstObj := f.newObject(remote)
 
 	// Check it is a file if it exists
 	err := dstObj.lstat()
@@ -701,8 +688,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := f.cleanPath(filepath.Join(srcFs.root, srcRemote))
-	dstPath := f.cleanPath(filepath.Join(f.root, dstRemote))
+	srcPath := srcFs.localPath(srcRemote)
+	dstPath := f.localPath(dstRemote)
 
 	// Check if destination exists
 	_, err := os.Lstat(dstPath)
@@ -736,7 +723,51 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Supported
+	return hash.Supported()
+}
+
+var commandHelp = []fs.CommandHelp{
+	{
+		Name:  "noop",
+		Short: "A null operation for testing backend commands",
+		Long: `This is a test command which has some options
+you can try to change the output.`,
+		Opts: map[string]string{
+			"echo":  "echo the input arguments",
+			"error": "return an error based on option value",
+		},
+	},
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+	switch name {
+	case "noop":
+		if txt, ok := opt["error"]; ok {
+			if txt == "" {
+				txt = "unspecified error"
+			}
+			return nil, errors.New(txt)
+		}
+		if _, ok := opt["echo"]; ok {
+			out := map[string]interface{}{}
+			out["name"] = name
+			out["arg"] = arg
+			out["opt"] = opt
+			return out, nil
+		}
+		return nil, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
 }
 
 // ------------------------------------------------------------
@@ -765,8 +796,17 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	oldtime := o.modTime
 	oldsize := o.size
 	err := o.lstat()
+	var changed bool
 	if err != nil {
-		return "", errors.Wrap(err, "hash: failed to stat")
+		if os.IsNotExist(errors.Cause(err)) {
+			// If file not found then we assume any accumulated
+			// hashes are OK - this will error on Open
+			changed = true
+		} else {
+			return "", errors.Wrap(err, "hash: failed to stat")
+		}
+	} else {
+		changed = !o.modTime.Equal(oldtime) || oldsize != o.size
 	}
 
 	o.fs.objectHashesMu.Lock()
@@ -774,7 +814,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	hashValue, hashFound := o.hashes[r]
 	o.fs.objectHashesMu.Unlock()
 
-	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
+	if changed || hashes == nil || !hashFound {
 		var in io.ReadCloser
 
 		if !o.translatedLink {
@@ -786,6 +826,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		} else {
 			in, err = o.openTranslatedLink(0, -1)
 		}
+		// If not checking for updates, only read size given
+		if o.fs.opt.NoCheckUpdated {
+			in = readers.NewLimitedReadCloser(in, o.size)
+		}
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -836,13 +880,6 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 
 // Storable returns a boolean showing if this object is storable
 func (o *Object) Storable() bool {
-	// Check for control characters in the remote name and show non storable
-	for _, c := range o.Remote() {
-		if c >= 0x00 && c < 0x20 || c == 0x7F {
-			fs.Logf(o.fs, "Can't store file with control characters: %q", o.Remote())
-			return false
-		}
-	}
 	mode := o.mode
 	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
@@ -877,10 +914,10 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 			return 0, errors.Wrap(err, "can't read status of source file while transferring")
 		}
 		if file.o.size != fi.Size() {
-			return 0, errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size())
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size()))
 		}
 		if !file.o.modTime.Equal(fi.ModTime()) {
-			return 0, errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime())
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime()))
 		}
 	}
 
@@ -939,6 +976,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 	}
 
+	// If not checking updated then limit to current size.  This means if
+	// file is being extended, readers will read a o.Size() bytes rather
+	// than the new size making for a consistent upload.
+	if limit < 0 && o.fs.opt.NoCheckUpdated {
+		limit = o.size
+	}
+
 	// Handle a translated link
 	if o.translatedLink {
 		return o.openTranslatedLink(offset, limit)
@@ -1013,10 +1057,20 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if !o.translatedLink {
 		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
 		if err != nil {
-			return err
+			if runtime.GOOS == "windows" && os.IsPermission(err) {
+				// If permission denied on Windows might be trying to update a
+				// hidden file, in which case try opening without CREATE
+				// See: https://stackoverflow.com/questions/13215716/ioerror-errno-13-permission-denied-when-trying-to-open-hidden-file-in-w-mod
+				f, err = file.OpenFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666)
+				if err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
 		}
 		// Pre-allocate the file for performance reasons
-		err = preAllocate(src.Size(), f)
+		err = file.PreAllocate(src.Size(), f)
 		if err != nil {
 			fs.Debugf(o, "Failed to pre-allocate: %v", err)
 		}
@@ -1080,6 +1134,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.lstat()
 }
 
+var sparseWarning sync.Once
+
 // OpenWriterAt opens with a handle for random access writes
 //
 // Pass in the remote desired and the size if known.
@@ -1087,7 +1143,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // It truncates any existing object
 func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
 	// Temporary Object under construction
-	o := f.newObject(remote, "")
+	o := f.newObject(remote)
 
 	err := o.mkdirAll()
 	if err != nil {
@@ -1103,15 +1159,30 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 		return nil, err
 	}
 	// Pre-allocate the file for performance reasons
-	err = preAllocate(size, out)
+	err = file.PreAllocate(size, out)
 	if err != nil {
 		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
+	if !f.opt.NoSparse && file.SetSparseImplemented {
+		sparseWarning.Do(func() {
+			fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
+		})
+		// Set the file to be a sparse file (important on Windows)
+		err = file.SetSparse(out)
+		if err != nil {
+			fs.Debugf(o, "Failed to set sparse: %v", err)
+		}
+	}
+
 	return out, nil
 }
 
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
+	// if not checking updated then don't update the stat
+	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
+		return
+	}
 	// Don't overwrite the info if we don't need to
 	// this avoids upsetting the race detector
 	if o.size != info.Size() {
@@ -1125,7 +1196,7 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	}
 }
 
-// Stat a Object into info
+// Stat an Object into info
 func (o *Object) lstat() error {
 	info, err := o.fs.lstat(o.path)
 	if err == nil {
@@ -1139,29 +1210,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	return remove(o.path)
 }
 
-// cleanPathFragment cleans an OS path fragment which is part of a
-// bigger path and not necessarily absolute
-func cleanPathFragment(s string) string {
-	if s == "" {
-		return s
-	}
-	s = filepath.Clean(s)
-	if runtime.GOOS == "windows" {
-		s = strings.Replace(s, `/`, `\`, -1)
-	}
-	return s
-}
-
-// cleanPath cleans and makes absolute the path passed in and returns
-// an OS path.
-//
-// The input might be in OS form or rclone form or a mixture, but the
-// output is in OS form.
-//
-// On windows it makes the path UNC also and replaces any characters
-// Windows can't deal with with their replacements.
-func (f *Fs) cleanPath(s string) string {
-	s = cleanPathFragment(s)
+func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 	if runtime.GOOS == "windows" {
 		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
 			s2, err := filepath.Abs(s)
@@ -1169,19 +1218,24 @@ func (f *Fs) cleanPath(s string) string {
 				s = s2
 			}
 		}
-		if !f.opt.NoUNC {
+		s = filepath.ToSlash(s)
+		vol := filepath.VolumeName(s)
+		s = vol + enc.FromStandardPath(s[len(vol):])
+		s = filepath.FromSlash(s)
+
+		if !noUNC {
 			// Convert to UNC
 			s = uncPath(s)
 		}
-		s = cleanWindowsName(f, s)
-	} else {
-		if !filepath.IsAbs(s) {
-			s2, err := filepath.Abs(s)
-			if err == nil {
-				s = s2
-			}
+		return s
+	}
+	if !filepath.IsAbs(s) {
+		s2, err := filepath.Abs(s)
+		if err == nil {
+			s = s2
 		}
 	}
+	s = enc.FromStandardPath(s)
 	return s
 }
 
@@ -1190,63 +1244,21 @@ var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
 
 // uncPath converts an absolute Windows path
 // to a UNC long path.
-func uncPath(s string) string {
-	// UNC can NOT use "/", so convert all to "\"
-	s = strings.Replace(s, `/`, `\`, -1)
-
+func uncPath(l string) string {
 	// If prefix is "\\", we already have a UNC path or server.
-	if strings.HasPrefix(s, `\\`) {
+	if strings.HasPrefix(l, `\\`) {
 		// If already long path, just keep it
-		if strings.HasPrefix(s, `\\?\`) {
-			return s
+		if strings.HasPrefix(l, `\\?\`) {
+			return l
 		}
 
 		// Trim "\\" from path and add UNC prefix.
-		return `\\?\UNC\` + strings.TrimPrefix(s, `\\`)
+		return `\\?\UNC\` + strings.TrimPrefix(l, `\\`)
 	}
-	if isAbsWinDrive.MatchString(s) {
-		return `\\?\` + s
+	if isAbsWinDrive.MatchString(l) {
+		return `\\?\` + l
 	}
-	return s
-}
-
-// cleanWindowsName will clean invalid Windows characters replacing them with _
-func cleanWindowsName(f *Fs, name string) string {
-	original := name
-	var name2 string
-	if strings.HasPrefix(name, `\\?\`) {
-		name2 = `\\?\`
-		name = strings.TrimPrefix(name, `\\?\`)
-	}
-	if strings.HasPrefix(name, `//?/`) {
-		name2 = `//?/`
-		name = strings.TrimPrefix(name, `//?/`)
-	}
-	// Colon is allowed as part of a drive name X:\
-	colonAt := strings.Index(name, ":")
-	if colonAt > 0 && colonAt < 3 && len(name) > colonAt+1 {
-		// Copy to name2, which is unfiltered
-		name2 += name[0 : colonAt+1]
-		name = name[colonAt+1:]
-	}
-
-	name2 += strings.Map(func(r rune) rune {
-		switch r {
-		case '<', '>', '"', '|', '?', '*', ':':
-			return '_'
-		}
-		return r
-	}, name)
-
-	if name2 != original && f != nil {
-		f.wmu.Lock()
-		if _, ok := f.warned[name]; !ok {
-			fs.Logf(f, "Replacing invalid characters in %q to %q", name, name2)
-			f.warned[name] = struct{}{}
-		}
-		f.wmu.Unlock()
-	}
-	return name2
+	return l
 }
 
 // Check the interfaces are satisfied
@@ -1256,6 +1268,7 @@ var (
 	_ fs.PutStreamer    = &Fs{}
 	_ fs.Mover          = &Fs{}
 	_ fs.DirMover       = &Fs{}
+	_ fs.Commander      = &Fs{}
 	_ fs.OpenWriterAter = &Fs{}
 	_ fs.Object         = &Object{}
 )

backend/local/local_internal_test.go

@@ -25,19 +25,6 @@ func TestMain(m *testing.M) {
 	fstest.TestMain(m)
 }
 
-func TestMapper(t *testing.T) {
-	m := newMapper()
-	assert.Equal(t, m.m, map[string]string{})
-	assert.Equal(t, "potato", m.Save("potato", "potato"))
-	assert.Equal(t, m.m, map[string]string{})
-	assert.Equal(t, "-r'áö", m.Save("-r?'a´o¨", "-r'áö"))
-	assert.Equal(t, m.m, map[string]string{
-		"-r'áö": "-r?'a´o¨",
-	})
-	assert.Equal(t, "potato", m.Load("potato"))
-	assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö"))
-}
-
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
@@ -57,7 +44,7 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 	o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
 	wrappedFd := readers.NewLimitedReadCloser(fd, -1)
-	hash, err := hash.NewMultiHasherTypes(hash.Supported)
+	hash, err := hash.NewMultiHasherTypes(hash.Supported())
 	require.NoError(t, err)
 	in := localOpenFile{
 		o:    o,

backend/local/preallocate_other.go

@@ -1,10 +0,0 @@
-//+build !windows,!linux
-
-package local
-
-import "os"
-
-// preAllocate the file for performance reasons
-func preAllocate(size int64, out *os.File) error {
-	return nil
-}

backend/local/tests_test.go

@@ -1,29 +1,26 @@
 package local
 
 import (
+	"runtime"
 	"testing"
 )
 
 var uncTestPaths = []string{
-	"C:\\Ba*d\\P|a?t<h>\\Windows\\Folder",
-	"C:/Ba*d/P|a?t<h>/Windows\\Folder",
-	"C:\\Windows\\Folder",
-	"\\\\?\\C:\\Windows\\Folder",
-	"//?/C:/Windows/Folder",
-	"\\\\?\\UNC\\server\\share\\Desktop",
-	"\\\\?\\unC\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"\\\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"C:\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"C:\\AbsoluteToRoot\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"\\\\server\\share\\Desktop",
-	"\\\\?\\UNC\\\\share\\folder\\Desktop",
-	"\\\\server\\share",
+	`C:\Ba*d\P|a?t<h>\Windows\Folder`,
+	`C:\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop`,
+	`\\?\UNC\\share\folder\Desktop`,
+	`\\server\share`,
 }
 
 var uncTestPathsResults = []string{
 	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
-	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
-	`\\?\C:\Windows\Folder`,
 	`\\?\C:\Windows\Folder`,
 	`\\?\C:\Windows\Folder`,
 	`\\?\UNC\server\share\Desktop`,
@@ -51,38 +48,23 @@ func TestUncPaths(t *testing.T) {
 	}
 }
 
-var utf8Tests = [][2]string{
-	{"ABC", "ABC"},
-	{string([]byte{0x80}), "�"},
-	{string([]byte{'a', 0x80, 'b'}), "a�b"},
-}
-
-func TestCleanRemote(t *testing.T) {
-	f := &Fs{}
-	f.warned = make(map[string]struct{})
-	for _, test := range utf8Tests {
-		got := f.cleanRemote(test[0])
-		expect := test[1]
-		if got != expect {
-			t.Fatalf("got %q, expected %q", got, expect)
-		}
-	}
-}
-
 // Test Windows character replacements
 var testsWindows = [][2]string{
 	{`c:\temp`, `c:\temp`},
 	{`\\?\UNC\theserver\dir\file.txt`, `\\?\UNC\theserver\dir\file.txt`},
-	{`//?/UNC/theserver/dir\file.txt`, `//?/UNC/theserver/dir\file.txt`},
-	{"c:/temp", "c:/temp"},
-	{"/temp/file.txt", "/temp/file.txt"},
-	{`!\"#¤%&/()=;:*^?+-`, "!\\_#¤%&/()=;__^_+-"},
-	{`<>"|?*:&\<>"|?*:&\<>"|?*:&`, "_______&\\_______&\\_______&"},
+	{`//?/UNC/theserver/dir\file.txt`, `\\?\UNC\theserver\dir\file.txt`},
+	{`c:/temp`, `c:\temp`},
+	{`C:/temp/file.txt`, `C:\temp\file.txt`},
+	{`c:\!\"#¤%&/()=;:*^?+-`, `c:\!\＂#¤%&\()=;：＊^？+-`},
+	{`c:\<>"|?*:&\<>"|?*:&\<>"|?*:&`, `c:\＜＞＂｜？＊：&\＜＞＂｜？＊：&\＜＞＂｜？＊：&`},
 }
 
 func TestCleanWindows(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skipf("windows only")
+	}
 	for _, test := range testsWindows {
-		got := cleanWindowsName(nil, test[0])
+		got := cleanRootPath(test[0], true, defaultEnc)
 		expect := test[1]
 		if got != expect {
 			t.Fatalf("got %q, expected %q", got, expect)

backend/mailru/api/bin.go

@@ -0,0 +1,107 @@
+package api
+
+// BIN protocol constants
+const (
+	BinContentType    = "application/x-www-form-urlencoded"
+	TreeIDLength      = 12
+	DunnoNodeIDLength = 16
+)
+
+// Operations in binary protocol
+const (
+	OperationAddFile           = 103 // 0x67
+	OperationRename            = 105 // 0x69
+	OperationCreateFolder      = 106 // 0x6A
+	OperationFolderList        = 117 // 0x75
+	OperationSharedFoldersList = 121 // 0x79
+	// TODO investigate opcodes below
+	Operation154MaybeItemInfo = 154 // 0x9A
+	Operation102MaybeAbout    = 102 // 0x66
+	Operation104MaybeDelete   = 104 // 0x68
+)
+
+// CreateDir protocol constants
+const (
+	MkdirResultOK                  = 0
+	MkdirResultSourceNotExists     = 1
+	MkdirResultAlreadyExists       = 4
+	MkdirResultExistsDifferentCase = 9
+	MkdirResultInvalidName         = 10
+	MkdirResultFailed254           = 254
+)
+
+// Move result codes
+const (
+	MoveResultOK              = 0
+	MoveResultSourceNotExists = 1
+	MoveResultFailed002       = 2
+	MoveResultAlreadyExists   = 4
+	MoveResultFailed005       = 5
+	MoveResultFailed254       = 254
+)
+
+// AddFile result codes
+const (
+	AddResultOK          = 0
+	AddResultError01     = 1
+	AddResultDunno04     = 4
+	AddResultWrongPath   = 5
+	AddResultNoFreeSpace = 7
+	AddResultDunno09     = 9
+	AddResultInvalidName = 10
+	AddResultNotModified = 12
+	AddResultFailedA     = 253
+	AddResultFailedB     = 254
+)
+
+// List request options
+const (
+	ListOptTotalSpace  = 1
+	ListOptDelete      = 2
+	ListOptFingerprint = 4
+	ListOptUnknown8    = 8
+	ListOptUnknown16   = 16
+	ListOptFolderSize  = 32
+	ListOptUsedSpace   = 64
+	ListOptUnknown128  = 128
+	ListOptUnknown256  = 256
+)
+
+// ListOptDefaults ...
+const ListOptDefaults = ListOptUnknown128 | ListOptUnknown256 | ListOptFolderSize | ListOptTotalSpace | ListOptUsedSpace
+
+// List parse flags
+const (
+	ListParseDone      = 0
+	ListParseReadItem  = 1
+	ListParsePin       = 2
+	ListParsePinUpper  = 3
+	ListParseUnknown15 = 15
+)
+
+// List operation results
+const (
+	ListResultOK              = 0
+	ListResultNotExists       = 1
+	ListResultDunno02         = 2
+	ListResultDunno03         = 3
+	ListResultAlreadyExists04 = 4
+	ListResultDunno05         = 5
+	ListResultDunno06         = 6
+	ListResultDunno07         = 7
+	ListResultDunno08         = 8
+	ListResultAlreadyExists09 = 9
+	ListResultDunno10         = 10
+	ListResultDunno11         = 11
+	ListResultDunno12         = 12
+	ListResultFailedB         = 253
+	ListResultFailedA         = 254
+)
+
+// Directory item types
+const (
+	ListItemMountPoint   = 0
+	ListItemFile         = 1
+	ListItemFolder       = 2
+	ListItemSharedFolder = 3
+)

backend/mailru/api/helpers.go

@@ -0,0 +1,225 @@
+package api
+
+// BIN protocol helpers
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/binary"
+	"io"
+	"log"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/lib/readers"
+)
+
+// protocol errors
+var (
+	ErrorPrematureEOF  = errors.New("Premature EOF")
+	ErrorInvalidLength = errors.New("Invalid length")
+	ErrorZeroTerminate = errors.New("String must end with zero")
+)
+
+// BinWriter is a binary protocol writer
+type BinWriter struct {
+	b *bytes.Buffer // growing byte buffer
+	a []byte        // temporary buffer for next varint
+}
+
+// NewBinWriter creates a binary protocol helper
+func NewBinWriter() *BinWriter {
+	return &BinWriter{
+		b: new(bytes.Buffer),
+		a: make([]byte, binary.MaxVarintLen64),
+	}
+}
+
+// Bytes returns binary data
+func (w *BinWriter) Bytes() []byte {
+	return w.b.Bytes()
+}
+
+// Reader returns io.Reader with binary data
+func (w *BinWriter) Reader() io.Reader {
+	return bytes.NewReader(w.b.Bytes())
+}
+
+// WritePu16 writes a short as unsigned varint
+func (w *BinWriter) WritePu16(val int) {
+	if val < 0 || val > 65535 {
+		log.Fatalf("Invalid UInt16 %v", val)
+	}
+	w.WritePu64(int64(val))
+}
+
+// WritePu32 writes a signed long as unsigned varint
+func (w *BinWriter) WritePu32(val int64) {
+	if val < 0 || val > 4294967295 {
+		log.Fatalf("Invalid UInt32 %v", val)
+	}
+	w.WritePu64(val)
+}
+
+// WritePu64 writes an unsigned (actually, signed) long as unsigned varint
+func (w *BinWriter) WritePu64(val int64) {
+	if val < 0 {
+		log.Fatalf("Invalid UInt64 %v", val)
+	}
+	w.b.Write(w.a[:binary.PutUvarint(w.a, uint64(val))])
+}
+
+// WriteString writes a zero-terminated string
+func (w *BinWriter) WriteString(str string) {
+	buf := []byte(str)
+	w.WritePu64(int64(len(buf) + 1))
+	w.b.Write(buf)
+	w.b.WriteByte(0)
+}
+
+// Write writes a byte buffer
+func (w *BinWriter) Write(buf []byte) {
+	w.b.Write(buf)
+}
+
+// WriteWithLength writes a byte buffer prepended with its length as varint
+func (w *BinWriter) WriteWithLength(buf []byte) {
+	w.WritePu64(int64(len(buf)))
+	w.b.Write(buf)
+}
+
+// BinReader is a binary protocol reader helper
+type BinReader struct {
+	b     *bufio.Reader
+	count *readers.CountingReader
+	err   error // keeps the first error encountered
+}
+
+// NewBinReader creates a binary protocol reader helper
+func NewBinReader(reader io.Reader) *BinReader {
+	r := &BinReader{}
+	r.count = readers.NewCountingReader(reader)
+	r.b = bufio.NewReader(r.count)
+	return r
+}
+
+// Count returns number of bytes read
+func (r *BinReader) Count() uint64 {
+	return r.count.BytesRead()
+}
+
+// Error returns first encountered error or nil
+func (r *BinReader) Error() error {
+	return r.err
+}
+
+// check() keeps the first error encountered in a stream
+func (r *BinReader) check(err error) bool {
+	if err == nil {
+		return true
+	}
+	if r.err == nil {
+		// keep the first error
+		r.err = err
+	}
+	if err != io.EOF {
+		log.Fatalf("Error parsing response: %v", err)
+	}
+	return false
+}
+
+// ReadByteAsInt reads a single byte as uint32, returns -1 for EOF or errors
+func (r *BinReader) ReadByteAsInt() int {
+	if octet, err := r.b.ReadByte(); r.check(err) {
+		return int(octet)
+	}
+	return -1
+}
+
+// ReadByteAsShort reads a single byte as uint16, returns -1 for EOF or errors
+func (r *BinReader) ReadByteAsShort() int16 {
+	if octet, err := r.b.ReadByte(); r.check(err) {
+		return int16(octet)
+	}
+	return -1
+}
+
+// ReadIntSpl reads two bytes as little-endian uint16, returns -1 for EOF or errors
+func (r *BinReader) ReadIntSpl() int {
+	var val uint16
+	if r.check(binary.Read(r.b, binary.LittleEndian, &val)) {
+		return int(val)
+	}
+	return -1
+}
+
+// ReadULong returns uint64 equivalent of -1 for EOF or errors
+func (r *BinReader) ReadULong() uint64 {
+	if val, err := binary.ReadUvarint(r.b); r.check(err) {
+		return val
+	}
+	return 0xffffffffffffffff
+}
+
+// ReadPu32 returns -1 for EOF or errors
+func (r *BinReader) ReadPu32() int64 {
+	if val, err := binary.ReadUvarint(r.b); r.check(err) {
+		return int64(val)
+	}
+	return -1
+}
+
+// ReadNBytes reads given number of bytes, returns invalid data for EOF or errors
+func (r *BinReader) ReadNBytes(len int) []byte {
+	buf := make([]byte, len)
+	n, err := r.b.Read(buf)
+	if r.check(err) {
+		return buf
+	}
+	if n != len {
+		r.check(ErrorPrematureEOF)
+	}
+	return buf
+}
+
+// ReadBytesByLength reads buffer length and its bytes
+func (r *BinReader) ReadBytesByLength() []byte {
+	len := r.ReadPu32()
+	if len < 0 {
+		r.check(ErrorInvalidLength)
+		return []byte{}
+	}
+	return r.ReadNBytes(int(len))
+}
+
+// ReadString reads a zero-terminated string with length
+func (r *BinReader) ReadString() string {
+	len := int(r.ReadPu32())
+	if len < 1 {
+		r.check(ErrorInvalidLength)
+		return ""
+	}
+	buf := make([]byte, len-1)
+	n, err := r.b.Read(buf)
+	if !r.check(err) {
+		return ""
+	}
+	if n != len-1 {
+		r.check(ErrorPrematureEOF)
+		return ""
+	}
+	zeroByte, err := r.b.ReadByte()
+	if !r.check(err) {
+		return ""
+	}
+	if zeroByte != 0 {
+		r.check(ErrorZeroTerminate)
+		return ""
+	}
+	return string(buf)
+}
+
+// ReadDate reads a Unix encoded time
+func (r *BinReader) ReadDate() time.Time {
+	return time.Unix(r.ReadPu32(), 0)
+}

backend/mailru/api/m1.go

@@ -0,0 +1,248 @@
+package api
+
+import (
+	"fmt"
+)
+
+// M1 protocol constants and structures
+const (
+	APIServerURL      = "https://cloud.mail.ru"
+	PublicLinkURL     = "https://cloud.mail.ru/public/"
+	DispatchServerURL = "https://dispatcher.cloud.mail.ru"
+	OAuthURL          = "https://o2.mail.ru/token"
+	OAuthClientID     = "cloud-win"
+)
+
+// ServerErrorResponse represents erroneous API response.
+type ServerErrorResponse struct {
+	Message string `json:"body"`
+	Time    int64  `json:"time"`
+	Status  int    `json:"status"`
+}
+
+func (e *ServerErrorResponse) Error() string {
+	return fmt.Sprintf("server error %d (%s)", e.Status, e.Message)
+}
+
+// FileErrorResponse represents erroneous API response for a file
+type FileErrorResponse struct {
+	Body struct {
+		Home struct {
+			Value string `json:"value"`
+			Error string `json:"error"`
+		} `json:"home"`
+	} `json:"body"`
+	Status  int    `json:"status"`
+	Account string `json:"email,omitempty"`
+	Time    int64  `json:"time,omitempty"`
+	Message string // non-json, calculated field
+}
+
+func (e *FileErrorResponse) Error() string {
+	return fmt.Sprintf("file error %d (%s)", e.Status, e.Body.Home.Error)
+}
+
+// UserInfoResponse contains account metadata
+type UserInfoResponse struct {
+	Body struct {
+		AccountType     string `json:"account_type"`
+		AccountVerified bool   `json:"account_verified"`
+		Cloud           struct {
+			Beta struct {
+				Allowed bool `json:"allowed"`
+				Asked   bool `json:"asked"`
+			} `json:"beta"`
+			Billing struct {
+				ActiveCostID string `json:"active_cost_id"`
+				ActiveRateID string `json:"active_rate_id"`
+				AutoProlong  bool   `json:"auto_prolong"`
+				Basequota    int64  `json:"basequota"`
+				Enabled      bool   `json:"enabled"`
+				Expires      int    `json:"expires"`
+				Prolong      bool   `json:"prolong"`
+				Promocodes   struct {
+				} `json:"promocodes"`
+				Subscription []interface{} `json:"subscription"`
+				Version      string        `json:"version"`
+			} `json:"billing"`
+			Bonuses struct {
+				CameraUpload bool `json:"camera_upload"`
+				Complete     bool `json:"complete"`
+				Desktop      bool `json:"desktop"`
+				Feedback     bool `json:"feedback"`
+				Links        bool `json:"links"`
+				Mobile       bool `json:"mobile"`
+				Registration bool `json:"registration"`
+			} `json:"bonuses"`
+			Enable struct {
+				Sharing bool `json:"sharing"`
+			} `json:"enable"`
+			FileSizeLimit int64 `json:"file_size_limit"`
+			Space         struct {
+				BytesTotal int64 `json:"bytes_total"`
+				BytesUsed  int   `json:"bytes_used"`
+				Overquota  bool  `json:"overquota"`
+			} `json:"space"`
+		} `json:"cloud"`
+		Cloudflags struct {
+			Exists bool `json:"exists"`
+		} `json:"cloudflags"`
+		Domain string `json:"domain"`
+		Login  string `json:"login"`
+		Newbie bool   `json:"newbie"`
+		UI     struct {
+			ExpandLoader bool   `json:"expand_loader"`
+			Kind         string `json:"kind"`
+			Sidebar      bool   `json:"sidebar"`
+			Sort         struct {
+				Order string `json:"order"`
+				Type  string `json:"type"`
+			} `json:"sort"`
+			Thumbs bool `json:"thumbs"`
+		} `json:"ui"`
+	} `json:"body"`
+	Email  string `json:"email"`
+	Status int    `json:"status"`
+	Time   int64  `json:"time"`
+}
+
+// ListItem ...
+type ListItem struct {
+	Count struct {
+		Folders int `json:"folders"`
+		Files   int `json:"files"`
+	} `json:"count,omitempty"`
+	Kind      string `json:"kind"`
+	Type      string `json:"type"`
+	Name      string `json:"name"`
+	Home      string `json:"home"`
+	Size      int64  `json:"size"`
+	Mtime     int64  `json:"mtime,omitempty"`
+	Hash      string `json:"hash,omitempty"`
+	VirusScan string `json:"virus_scan,omitempty"`
+	Tree      string `json:"tree,omitempty"`
+	Grev      int    `json:"grev,omitempty"`
+	Rev       int    `json:"rev,omitempty"`
+}
+
+// ItemInfoResponse ...
+type ItemInfoResponse struct {
+	Email  string   `json:"email"`
+	Body   ListItem `json:"body"`
+	Time   int64    `json:"time"`
+	Status int      `json:"status"`
+}
+
+// FolderInfoResponse ...
+type FolderInfoResponse struct {
+	Body struct {
+		Count struct {
+			Folders int `json:"folders"`
+			Files   int `json:"files"`
+		} `json:"count"`
+		Tree string `json:"tree"`
+		Name string `json:"name"`
+		Grev int    `json:"grev"`
+		Size int64  `json:"size"`
+		Sort struct {
+			Order string `json:"order"`
+			Type  string `json:"type"`
+		} `json:"sort"`
+		Kind string     `json:"kind"`
+		Rev  int        `json:"rev"`
+		Type string     `json:"type"`
+		Home string     `json:"home"`
+		List []ListItem `json:"list"`
+	} `json:"body,omitempty"`
+	Time   int64  `json:"time"`
+	Status int    `json:"status"`
+	Email  string `json:"email"`
+}
+
+// ShardInfoResponse ...
+type ShardInfoResponse struct {
+	Email string `json:"email"`
+	Body  struct {
+		Video []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"video"`
+		ViewDirect []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"view_direct"`
+		WeblinkView []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_view"`
+		WeblinkVideo []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_video"`
+		WeblinkGet []struct {
+			Count int    `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_get"`
+		Stock []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"stock"`
+		WeblinkThumbnails []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_thumbnails"`
+		PublicUpload []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"public_upload"`
+		Auth []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"auth"`
+		Web []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"web"`
+		View []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"view"`
+		Upload []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"upload"`
+		Get []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"get"`
+		Thumbnails []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"thumbnails"`
+	} `json:"body"`
+	Time   int64 `json:"time"`
+	Status int   `json:"status"`
+}
+
+// CleanupResponse ...
+type CleanupResponse struct {
+	Email     string `json:"email"`
+	Time      int64  `json:"time"`
+	StatusStr string `json:"status"`
+}
+
+// GenericResponse ...
+type GenericResponse struct {
+	Email  string `json:"email"`
+	Time   int64  `json:"time"`
+	Status int    `json:"status"`
+	// ignore other fields
+}
+
+// GenericBodyResponse ...
+type GenericBodyResponse struct {
+	Email  string `json:"email"`
+	Body   string `json:"body"`
+	Time   int64  `json:"time"`
+	Status int    `json:"status"`
+}

backend/mailru/mailru_test.go

@@ -0,0 +1,18 @@
+// Test Mailru filesystem interface
+package mailru_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/mailru"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName:               "TestMailru:",
+		NilObject:                (*mailru.Object)(nil),
+		SkipBadWindowsCharacters: true,
+	})
+}

backend/mailru/mrhash/mrhash.go

@@ -0,0 +1,134 @@
+// Package mrhash implements the mailru hash, which is a modified SHA1.
+// If file size is less than or equal to the SHA1 block size (20 bytes),
+// its hash is simply its data right-padded with zero bytes.
+// Hash sum of a larger file is computed as a SHA1 sum of the file data
+// bytes concatenated with a decimal representation of the data length.
+package mrhash
+
+import (
+	"crypto/sha1"
+	"encoding"
+	"encoding/hex"
+	"errors"
+	"hash"
+	"strconv"
+)
+
+const (
+	// BlockSize of the checksum in bytes.
+	BlockSize = sha1.BlockSize
+	// Size of the checksum in bytes.
+	Size        = sha1.Size
+	startString = "mrCloud"
+	hashError   = "hash function returned error"
+)
+
+// Global errors
+var (
+	ErrorInvalidHash = errors.New("invalid hash")
+)
+
+type digest struct {
+	total int       // bytes written into hash so far
+	sha   hash.Hash // underlying SHA1
+	small []byte    // small content
+}
+
+// New returns a new hash.Hash computing the Mailru checksum.
+func New() hash.Hash {
+	d := &digest{}
+	d.Reset()
+	return d
+}
+
+// Write writes len(p) bytes from p to the underlying data stream. It returns
+// the number of bytes written from p (0 <= n <= len(p)) and any error
+// encountered that caused the write to stop early. Write must return a non-nil
+// error if it returns n < len(p). Write must not modify the slice data, even
+// temporarily.
+//
+// Implementations must not retain p.
+func (d *digest) Write(p []byte) (n int, err error) {
+	n, err = d.sha.Write(p)
+	if err != nil {
+		panic(hashError)
+	}
+	d.total += n
+	if d.total <= Size {
+		d.small = append(d.small, p...)
+	}
+	return n, nil
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (d *digest) Sum(b []byte) []byte {
+	// If content is small, return it padded to Size
+	if d.total <= Size {
+		padded := make([]byte, Size)
+		copy(padded, d.small)
+		return append(b, padded...)
+	}
+	endString := strconv.Itoa(d.total)
+	copy, err := cloneSHA1(d.sha)
+	if err == nil {
+		_, err = copy.Write([]byte(endString))
+	}
+	if err != nil {
+		panic(hashError)
+	}
+	return copy.Sum(b)
+}
+
+// cloneSHA1 clones state of SHA1 hash
+func cloneSHA1(orig hash.Hash) (clone hash.Hash, err error) {
+	state, err := orig.(encoding.BinaryMarshaler).MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+	clone = sha1.New()
+	err = clone.(encoding.BinaryUnmarshaler).UnmarshalBinary(state)
+	return
+}
+
+// Reset resets the Hash to its initial state.
+func (d *digest) Reset() {
+	d.sha = sha1.New()
+	_, _ = d.sha.Write([]byte(startString))
+	d.total = 0
+}
+
+// Size returns the number of bytes Sum will return.
+func (d *digest) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (d *digest) BlockSize() int {
+	return BlockSize
+}
+
+// Sum returns the Mailru checksum of the data.
+func Sum(data []byte) []byte {
+	var d digest
+	d.Reset()
+	_, _ = d.Write(data)
+	return d.Sum(nil)
+}
+
+// DecodeString converts a string to the Mailru hash
+func DecodeString(s string) ([]byte, error) {
+	b, err := hex.DecodeString(s)
+	if err != nil || len(b) != Size {
+		return nil, ErrorInvalidHash
+	}
+	return b, nil
+}
+
+// must implement this interface
+var (
+	_ hash.Hash = (*digest)(nil)
+)

backend/mailru/mrhash/mrhash_test.go

@@ -0,0 +1,81 @@
+package mrhash_test
+
+import (
+	"encoding/hex"
+	"fmt"
+	"testing"
+
+	"github.com/rclone/rclone/backend/mailru/mrhash"
+	"github.com/stretchr/testify/assert"
+)
+
+func testChunk(t *testing.T, chunk int) {
+	data := make([]byte, chunk)
+	for i := 0; i < chunk; i++ {
+		data[i] = 'A'
+	}
+	for _, test := range []struct {
+		n    int
+		want string
+	}{
+		{0, "0000000000000000000000000000000000000000"},
+		{1, "4100000000000000000000000000000000000000"},
+		{2, "4141000000000000000000000000000000000000"},
+		{19, "4141414141414141414141414141414141414100"},
+		{20, "4141414141414141414141414141414141414141"},
+		{21, "eb1d05e78a18691a5aa196a6c2b60cd40b5faafb"},
+		{22, "037e6d960601118a0639afbeff30fe716c66ed2d"},
+		{4096, "45a16aa192502b010280fb5b44274c601a91fd9f"},
+		{4194303, "fa019d5bd26498cf6abe35e0d61801bf19bf704b"},
+		{4194304, "5ed0e07aa6ea5c1beb9402b4d807258f27d40773"},
+		{4194305, "67bd0b9247db92e0e7d7e29a0947a50fedcb5452"},
+		{8388607, "41a8e2eb044c2e242971b5445d7be2a13fc0dd84"},
+		{8388608, "267a970917c624c11fe624276ec60233a66dc2c0"},
+		{8388609, "37b60b308d553d2732aefb62b3ea88f74acfa13f"},
+	} {
+		d := mrhash.New()
+		var toWrite int
+		for toWrite = test.n; toWrite >= chunk; toWrite -= chunk {
+			n, err := d.Write(data)
+			assert.Nil(t, err)
+			assert.Equal(t, chunk, n)
+		}
+		n, err := d.Write(data[:toWrite])
+		assert.Nil(t, err)
+		assert.Equal(t, toWrite, n)
+		got1 := hex.EncodeToString(d.Sum(nil))
+		assert.Equal(t, test.want, got1, fmt.Sprintf("when testing length %d", n))
+		got2 := hex.EncodeToString(d.Sum(nil))
+		assert.Equal(t, test.want, got2, fmt.Sprintf("when testing length %d (2nd sum)", n))
+	}
+}
+
+func TestHashChunk16M(t *testing.T)  { testChunk(t, 16*1024*1024) }
+func TestHashChunk8M(t *testing.T)   { testChunk(t, 8*1024*1024) }
+func TestHashChunk4M(t *testing.T)   { testChunk(t, 4*1024*1024) }
+func TestHashChunk2M(t *testing.T)   { testChunk(t, 2*1024*1024) }
+func TestHashChunk1M(t *testing.T)   { testChunk(t, 1*1024*1024) }
+func TestHashChunk64k(t *testing.T)  { testChunk(t, 64*1024) }
+func TestHashChunk32k(t *testing.T)  { testChunk(t, 32*1024) }
+func TestHashChunk2048(t *testing.T) { testChunk(t, 2048) }
+func TestHashChunk2047(t *testing.T) { testChunk(t, 2047) }
+
+func TestSumCalledTwice(t *testing.T) {
+	d := mrhash.New()
+	assert.NotPanics(t, func() { d.Sum(nil) })
+	d.Reset()
+	assert.NotPanics(t, func() { d.Sum(nil) })
+	assert.NotPanics(t, func() { d.Sum(nil) })
+	_, _ = d.Write([]byte{1})
+	assert.NotPanics(t, func() { d.Sum(nil) })
+}
+
+func TestSize(t *testing.T) {
+	d := mrhash.New()
+	assert.Equal(t, 20, d.Size())
+}
+
+func TestBlockSize(t *testing.T) {
+	d := mrhash.New()
+	assert.Equal(t, 64, d.BlockSize())
+}

backend/mega/mega.go

@@ -26,11 +26,13 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	mega "github.com/t3rm1n4l/go-mega"
@@ -80,16 +82,24 @@ than permanently deleting them.  If you specify this then rclone will
 permanently delete objects instead.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	User       string `config:"user"`
-	Pass       string `config:"pass"`
-	Debug      bool   `config:"debug"`
-	HardDelete bool   `config:"hard_delete"`
+	User       string               `config:"user"`
+	Pass       string               `config:"pass"`
+	Debug      bool                 `config:"debug"`
+	HardDelete bool                 `config:"hard_delete"`
+	Enc        encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote mega
@@ -140,7 +150,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an mega 'url'
+// parsePath parses a mega 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -245,21 +255,22 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 }
 
 // splitNodePath splits nodePath into / separated parts, returning nil if it
-// should refer to the root
-func splitNodePath(nodePath string) (parts []string) {
+// should refer to the root.
+// It also encodes the parts into backend specific encoding
+func (f *Fs) splitNodePath(nodePath string) (parts []string) {
 	nodePath = path.Clean(nodePath)
-	parts = strings.Split(nodePath, "/")
-	if len(parts) == 1 && (parts[0] == "." || parts[0] == "/") {
+	if nodePath == "." || nodePath == "/" {
 		return nil
 	}
-	return parts
+	nodePath = f.opt.Enc.FromStandardPath(nodePath)
+	return strings.Split(nodePath, "/")
 }
 
 // findNode looks up the node for the path of the name given from the root given
 //
 // It returns mega.ENOENT if it wasn't found
 func (f *Fs) findNode(rootNode *mega.Node, nodePath string) (*mega.Node, error) {
-	parts := splitNodePath(nodePath)
+	parts := f.splitNodePath(nodePath)
 	if parts == nil {
 		return rootNode, nil
 	}
@@ -316,7 +327,7 @@ func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error)
 	f.mkdirMu.Lock()
 	defer f.mkdirMu.Unlock()
 
-	parts := splitNodePath(dir)
+	parts := f.splitNodePath(dir)
 	if parts == nil {
 		return rootNode, nil
 	}
@@ -418,7 +429,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 	errors := 0
 	// similar to f.deleteNode(trash) but with HardDelete as true
 	for _, item := range items {
-		fs.Debugf(f, "Deleting trash %q", item.GetName())
+		fs.Debugf(f, "Deleting trash %q", f.opt.Enc.ToStandardName(item.GetName()))
 		deleteErr := f.pacer.Call(func() (bool, error) {
 			err := f.srv.Delete(item, true)
 			return shouldRetry(err)
@@ -500,7 +511,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	var iErr error
 	_, err = f.list(ctx, dirNode, func(info *mega.Node) bool {
-		remote := path.Join(dir, info.GetName())
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(info.GetName()))
 		switch info.GetType() {
 		case mega.FOLDER, mega.ROOT, mega.INBOX, mega.TRASH:
 			d := fs.NewDir(remote, info.GetTimeStamp()).SetID(info.GetHash())
@@ -722,7 +733,7 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 	if srcLeaf != dstLeaf {
 		//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
 		err = f.pacer.Call(func() (bool, error) {
-			err = f.srv.Rename(info, dstLeaf)
+			err = f.srv.Rename(info, f.opt.Enc.FromStandardName(dstLeaf))
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -825,7 +836,7 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	root, err := f.findRoot(false)
 	if err != nil {
 		return "", errors.Wrap(err, "PublicLink failed to find root node")
@@ -860,7 +871,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 			return errors.Errorf("MergeDirs failed to find node for: %v", srcDir)
 		}
 
-		// list the the objects
+		// list the objects
 		infos := []*mega.Node{}
 		_, err := f.list(ctx, srcDirNode, func(info *mega.Node) bool {
 			infos = append(infos, info)
@@ -871,13 +882,13 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 		}
 		// move them into place
 		for _, info := range infos {
-			fs.Infof(srcDir, "merging %q", info.GetName())
+			fs.Infof(srcDir, "merging %q", f.opt.Enc.ToStandardName(info.GetName()))
 			err = f.pacer.Call(func() (bool, error) {
 				err = f.srv.Move(info, dstDirNode)
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", f.opt.Enc.ToStandardName(info.GetName()), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
@@ -1120,7 +1131,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	var u *mega.Upload
 	err = o.fs.pacer.Call(func() (bool, error) {
-		u, err = o.fs.srv.NewUpload(dirNode, leaf, size)
+		u, err = o.fs.srv.NewUpload(dirNode, o.fs.opt.Enc.FromStandardName(leaf), size)
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/memory/memory.go

@@ -0,0 +1,624 @@
+// Package memory provides an interface to an in memory object storage system
+package memory
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/bucket"
+)
+
+var (
+	hashType = hash.MD5
+	// the object storage is persistent
+	buckets = newBucketsInfo()
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "memory",
+		Description: "In memory object storage system.",
+		NewFs:       NewFs,
+		Options:     []fs.Option{},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+}
+
+// Fs represents a remote memory server
+type Fs struct {
+	name          string       // name of this remote
+	root          string       // the path we are working on if any
+	opt           Options      // parsed config options
+	rootBucket    string       // bucket part of root (if any)
+	rootDirectory string       // directory part of root (if any)
+	features      *fs.Features // optional features
+}
+
+// bucketsInfo holds info about all the buckets
+type bucketsInfo struct {
+	mu      sync.RWMutex
+	buckets map[string]*bucketInfo
+}
+
+func newBucketsInfo() *bucketsInfo {
+	return &bucketsInfo{
+		buckets: make(map[string]*bucketInfo, 16),
+	}
+}
+
+// getBucket gets a names bucket or nil
+func (bi *bucketsInfo) getBucket(name string) (b *bucketInfo) {
+	bi.mu.RLock()
+	b = bi.buckets[name]
+	bi.mu.RUnlock()
+	return b
+}
+
+// makeBucket returns the bucket or makes it
+func (bi *bucketsInfo) makeBucket(name string) (b *bucketInfo) {
+	bi.mu.Lock()
+	defer bi.mu.Unlock()
+	b = bi.buckets[name]
+	if b != nil {
+		return b
+	}
+	b = newBucketInfo()
+	bi.buckets[name] = b
+	return b
+}
+
+// deleteBucket deleted the bucket or returns an error
+func (bi *bucketsInfo) deleteBucket(name string) error {
+	bi.mu.Lock()
+	defer bi.mu.Unlock()
+	b := bi.buckets[name]
+	if b == nil {
+		return fs.ErrorDirNotFound
+	}
+	if !b.isEmpty() {
+		return fs.ErrorDirectoryNotEmpty
+	}
+	delete(bi.buckets, name)
+	return nil
+}
+
+// getObjectData gets an object from (bucketName, bucketPath) or nil
+func (bi *bucketsInfo) getObjectData(bucketName, bucketPath string) (od *objectData) {
+	b := bi.getBucket(bucketName)
+	if b == nil {
+		return nil
+	}
+	return b.getObjectData(bucketPath)
+}
+
+// updateObjectData updates an object from (bucketName, bucketPath)
+func (bi *bucketsInfo) updateObjectData(bucketName, bucketPath string, od *objectData) {
+	b := bi.makeBucket(bucketName)
+	b.mu.Lock()
+	b.objects[bucketPath] = od
+	b.mu.Unlock()
+}
+
+// removeObjectData removes an object from (bucketName, bucketPath) returning true if removed
+func (bi *bucketsInfo) removeObjectData(bucketName, bucketPath string) (removed bool) {
+	b := bi.getBucket(bucketName)
+	if b != nil {
+		b.mu.Lock()
+		od := b.objects[bucketPath]
+		if od != nil {
+			delete(b.objects, bucketPath)
+			removed = true
+		}
+		b.mu.Unlock()
+	}
+	return removed
+}
+
+// bucketInfo holds info about a single bucket
+type bucketInfo struct {
+	mu      sync.RWMutex
+	objects map[string]*objectData
+}
+
+func newBucketInfo() *bucketInfo {
+	return &bucketInfo{
+		objects: make(map[string]*objectData, 16),
+	}
+}
+
+// getBucket gets a names bucket or nil
+func (bi *bucketInfo) getObjectData(name string) (od *objectData) {
+	bi.mu.RLock()
+	od = bi.objects[name]
+	bi.mu.RUnlock()
+	return od
+}
+
+// getBucket gets a names bucket or nil
+func (bi *bucketInfo) isEmpty() (empty bool) {
+	bi.mu.RLock()
+	empty = len(bi.objects) == 0
+	bi.mu.RUnlock()
+	return empty
+}
+
+// the object data and metadata
+type objectData struct {
+	modTime  time.Time
+	hash     string
+	mimeType string
+	data     []byte
+}
+
+// Object describes a memory object
+type Object struct {
+	fs     *Fs         // what this object is part of
+	remote string      // The remote path
+	od     *objectData // the object data
+}
+
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("Memory root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// parsePath parses a remote 'url'
+func parsePath(path string) (root string) {
+	root = strings.Trim(path, "/")
+	return
+}
+
+// split returns bucket and bucketPath from the rootRelativePath
+// relative to f.root
+func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
+	return bucket.Split(path.Join(f.root, rootRelativePath))
+}
+
+// split returns bucket and bucketPath from the object
+func (o *Object) split() (bucket, bucketPath string) {
+	return o.fs.split(o.remote)
+}
+
+// setRoot changes the root of the Fs
+func (f *Fs) setRoot(root string) {
+	f.root = parsePath(root)
+	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
+}
+
+// NewFs contstructs an Fs from the path, bucket:path
+func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	root = strings.Trim(root, "/")
+	f := &Fs{
+		name: name,
+		root: root,
+		opt:  *opt,
+	}
+	f.setRoot(root)
+	f.features = (&fs.Features{
+		ReadMimeType:      true,
+		WriteMimeType:     true,
+		BucketBased:       true,
+		BucketBasedRootOK: true,
+	}).Fill(f)
+	if f.rootBucket != "" && f.rootDirectory != "" {
+		od := buckets.getObjectData(f.rootBucket, f.rootDirectory)
+		if od != nil {
+			newRoot := path.Dir(f.root)
+			if newRoot == "." {
+				newRoot = ""
+			}
+			f.setRoot(newRoot)
+			// return an error with an fs which points to the parent
+			err = fs.ErrorIsFile
+		}
+	}
+	return f, err
+}
+
+// newObject makes an object from a remote and an objectData
+func (f *Fs) newObject(remote string, od *objectData) *Object {
+	return &Object{fs: f, remote: remote, od: od}
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	bucket, bucketPath := f.split(remote)
+	od := buckets.getObjectData(bucket, bucketPath)
+	if od == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return f.newObject(remote, od), nil
+}
+
+// listFn is called from list to handle an object.
+type listFn func(remote string, entry fs.DirEntry, isDirectory bool) error
+
+// list the buckets to fn
+func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, fn listFn) (err error) {
+	if prefix != "" {
+		prefix += "/"
+	}
+	if directory != "" {
+		directory += "/"
+	}
+	b := buckets.getBucket(bucket)
+	if b == nil {
+		return fs.ErrorDirNotFound
+	}
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	dirs := make(map[string]struct{})
+	for absPath, od := range b.objects {
+		if strings.HasPrefix(absPath, directory) {
+			remote := absPath[len(prefix):]
+			if !recurse {
+				localPath := absPath[len(directory):]
+				slash := strings.IndexRune(localPath, '/')
+				if slash >= 0 {
+					// send a directory if have a slash
+					dir := directory + localPath[:slash]
+					if addBucket {
+						dir = path.Join(bucket, dir)
+					}
+					_, found := dirs[dir]
+					if !found {
+						err = fn(dir, fs.NewDir(dir, time.Time{}), true)
+						if err != nil {
+							return err
+						}
+						dirs[dir] = struct{}{}
+					}
+					continue // don't send this file if not recursing
+				}
+			}
+			// send an object
+			if addBucket {
+				remote = path.Join(bucket, remote)
+			}
+			err = fn(remote, f.newObject(remote, od), false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// listDir lists the bucket to the entries
+func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) {
+	// List the objects and directories
+	err = f.list(ctx, bucket, directory, prefix, addBucket, false, func(remote string, entry fs.DirEntry, isDirectory bool) error {
+		entries = append(entries, entry)
+		return nil
+	})
+	return entries, err
+}
+
+// listBuckets lists the buckets to entries
+func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
+	buckets.mu.RLock()
+	defer buckets.mu.RUnlock()
+	for name := range buckets.buckets {
+		entries = append(entries, fs.NewDir(name, time.Time{}))
+	}
+	return entries, nil
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	// defer fslog.Trace(dir, "")("entries = %q, err = %v", &entries, &err)
+	bucket, directory := f.split(dir)
+	if bucket == "" {
+		if directory != "" {
+			return nil, fs.ErrorListBucketRequired
+		}
+		return f.listBuckets(ctx)
+	}
+	return f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "")
+}
+
+// ListR lists the objects and directories of the Fs starting
+// from dir recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+//
+// Don't implement this unless you have a more efficient way
+// of listing recursively that doing a directory traversal.
+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+	bucket, directory := f.split(dir)
+	list := walk.NewListRHelper(callback)
+	listR := func(bucket, directory, prefix string, addBucket bool) error {
+		return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {
+			return list.Add(entry)
+		})
+	}
+	if bucket == "" {
+		entries, err := f.listBuckets(ctx)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			err = list.Add(entry)
+			if err != nil {
+				return err
+			}
+			bucket := entry.Remote()
+			err = listR(bucket, "", f.rootDirectory, true)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		err = listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
+		if err != nil {
+			return err
+		}
+	}
+	return list.Flush()
+}
+
+// Put the object into the bucket
+//
+// Copy the reader in to the new object which is returned
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	// Temporary Object under construction
+	fs := &Object{
+		fs:     f,
+		remote: src.Remote(),
+		od: &objectData{
+			modTime: src.ModTime(ctx),
+		},
+	}
+	return fs, fs.Update(ctx, in, src, options...)
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
+// Mkdir creates the bucket if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	bucket, _ := f.split(dir)
+	buckets.makeBucket(bucket)
+	return nil
+}
+
+// Rmdir deletes the bucket if the fs is at the root
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	bucket, directory := f.split(dir)
+	if bucket == "" || directory != "" {
+		return nil
+	}
+	return buckets.deleteBucket(bucket)
+}
+
+// Precision of the remote
+func (f *Fs) Precision() time.Duration {
+	return time.Nanosecond
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	dstBucket, dstPath := f.split(remote)
+	_ = buckets.makeBucket(dstBucket)
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+	srcBucket, srcPath := srcObj.split()
+	od := buckets.getObjectData(srcBucket, srcPath)
+	if od == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	buckets.updateObjectData(dstBucket, dstPath, od)
+	return f.NewObject(ctx, remote)
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hashType)
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Remote()
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns the hash of an object returning a lowercase hex string
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hashType {
+		return "", hash.ErrUnsupported
+	}
+	if o.od.hash == "" {
+		sum := md5.Sum(o.od.data)
+		o.od.hash = hex.EncodeToString(sum[:])
+	}
+	return o.od.hash, nil
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return int64(len(o.od.data))
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+//
+// SHA-1 will also be updated once the request has completed.
+func (o *Object) ModTime(ctx context.Context) (result time.Time) {
+	return o.od.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	o.od.modTime = modTime
+	return nil
+}
+
+// Storable returns if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	var offset, limit int64 = 0, -1
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			offset, limit = x.Decode(int64(len(o.od.data)))
+		case *fs.SeekOption:
+			offset = x.Offset
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	if offset > int64(len(o.od.data)) {
+		offset = int64(len(o.od.data))
+	}
+	data := o.od.data[offset:]
+	if limit >= 0 {
+		if limit > int64(len(data)) {
+			limit = int64(len(data))
+		}
+		data = data[:limit]
+	}
+	return ioutil.NopCloser(bytes.NewBuffer(data)), nil
+}
+
+// Update the object with the contents of the io.Reader, modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	bucket, bucketPath := o.split()
+	data, err := ioutil.ReadAll(in)
+	if err != nil {
+		return errors.Wrap(err, "failed to update memory object")
+	}
+	o.od = &objectData{
+		data:     data,
+		hash:     "",
+		modTime:  src.ModTime(ctx),
+		mimeType: fs.MimeType(ctx, o),
+	}
+	buckets.updateObjectData(bucket, bucketPath, o.od)
+	return nil
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	bucket, bucketPath := o.split()
+	removed := buckets.removeObjectData(bucket, bucketPath)
+	if !removed {
+		return fs.ErrorObjectNotFound
+	}
+	return nil
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.od.mimeType
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs          = &Fs{}
+	_ fs.Copier      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.Object      = &Object{}
+	_ fs.MimeTyper   = &Object{}
+)

backend/memory/memory_test.go

@@ -0,0 +1,16 @@
+// Test memory filesystem interface
+package memory
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: ":memory:",
+		NilObject:  (*Object)(nil),
+	})
+}

backend/onedrive/api/types.go

@@ -272,19 +272,19 @@ type CreateShareLinkResponse struct {
 	} `json:"link"`
 }
 
-// AsyncOperationStatus provides information on the status of a asynchronous job progress.
+// AsyncOperationStatus provides information on the status of an asynchronous job progress.
 //
 // The following API calls return AsyncOperationStatus resources:
 //
 // Copy Item
 // Upload From URL
 type AsyncOperationStatus struct {
-	PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
+	PercentageComplete float64 `json:"percentageComplete"` // A float value between 0 and 100 that indicates the percentage complete.
 	Status             string  `json:"status"`             // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
 }
 
 // GetID returns a normalized ID of the item
-// If DriveID is known it will be prefixed to the ID with # seperator
+// If DriveID is known it will be prefixed to the ID with # separator
 // Can be parsed using onedrive.parseNormalizedID(normalizedID)
 func (i *Item) GetID() string {
 	if i.IsRemote() && i.RemoteItem.ID != "" {

backend/onedrive/onedrive.go

@@ -12,13 +12,13 @@ import (
 	"log"
 	"net/http"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/rclone/rclone/lib/atexit"
-
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/onedrive/api"
+	"github.com/rclone/rclone/backend/onedrive/quickxorhash"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -26,7 +26,9 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -58,21 +60,26 @@ var (
 			AuthURL:  "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
 			TokenURL: "https://login.microsoftonline.com/common/oauth2/v2.0/token",
 		},
-		Scopes:       []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"},
+		Scopes:       []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access", "Sites.Read.All"},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
 	}
+
+	// QuickXorHashType is the hash.Type for OneDrive
+	QuickXorHashType hash.Type
 )
 
 // Register with Fs
 func init() {
+	QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "onedrive",
 		Description: "Microsoft OneDrive",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("onedrive", name, m, oauthConfig)
+			ctx := context.TODO()
+			err := oauthutil.Config("onedrive", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 				return
@@ -143,7 +150,7 @@ func init() {
 				}
 
 				sites := siteResponse{}
-				_, err := srv.CallJSON(&opts, nil, &sites)
+				_, err := srv.CallJSON(ctx, &opts, nil, &sites)
 				if err != nil {
 					log.Fatalf("Failed to query available sites: %v", err)
 				}
@@ -172,11 +179,33 @@ func init() {
 			// query Microsoft Graph
 			if finalDriveID == "" {
 				drives := drivesResponse{}
-				_, err := srv.CallJSON(&opts, nil, &drives)
+				_, err := srv.CallJSON(ctx, &opts, nil, &drives)
 				if err != nil {
 					log.Fatalf("Failed to query available drives: %v", err)
 				}
 
+				// Also call /me/drive as sometimes /me/drives doesn't return it #4068
+				if opts.Path == "/me/drives" {
+					opts.Path = "/me/drive"
+					meDrive := driveResource{}
+					_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
+					if err != nil {
+						log.Fatalf("Failed to query available drives: %v", err)
+					}
+					found := false
+					for _, drive := range drives.Drives {
+						if drive.DriveID == meDrive.DriveID {
+							found = true
+							break
+						}
+					}
+					// add the me drive if not found already
+					if !found {
+						fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
+						drives.Drives = append(drives.Drives, meDrive)
+					}
+				}
+
 				if len(drives.Drives) == 0 {
 					log.Fatalf("No drives found")
 				} else {
@@ -194,7 +223,7 @@ func init() {
 				RootURL: graphURL,
 				Path:    "/drives/" + finalDriveID + "/root"}
 			var rootItem api.Item
-			_, err = srv.CallJSON(&opts, nil, &rootItem)
+			_, err = srv.CallJSON(ctx, &opts, nil, &rootItem)
 			if err != nil {
 				log.Fatalf("Failed to query root for drive %s: %v", finalDriveID, err)
 			}
@@ -217,10 +246,11 @@ func init() {
 			Help: "Microsoft App Client Secret\nLeave blank normally.",
 		}, {
 			Name: "chunk_size",
-			Help: `Chunk size to upload files with - must be multiple of 320k.
+			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
 
-Above this size files will be chunked - must be multiple of 320k. Note
-that the chunks will be buffered into memory.`,
+Above this size files will be chunked - must be multiple of 320k (327,680 bytes) and
+should not exceed 250M (262,144,000 bytes) else you may encounter \"Microsoft.SharePoint.Client.InvalidClientQueryException: The request message is too big.\"
+Note that the chunks will be buffered into memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
@@ -244,16 +274,74 @@ delete OneNote files or otherwise want them to show up in directory
 listing, set this option.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different onedrive configs.
+
+This can be useful if you wish to do a server side copy between two
+different Onedrives.  Note that this isn't enabled by default
+because it isn't easy to tell if it will work between any two
+configurations.`,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// List of replaced characters:
+			//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+			//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+			//   : (colon)         -> '：' // FULLWIDTH COLON
+			//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+			//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+			//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+			//   # (number sign)  -> '＃'  // FULLWIDTH NUMBER SIGN
+			//   % (percent sign) -> '％'  // FULLWIDTH PERCENT SIGN
+			//
+			// Folder names cannot begin with a tilde ('~')
+			// List of replaced characters:
+			//   ~ (tilde)        -> '～'  // FULLWIDTH TILDE
+			//
+			// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).
+			// List of replaced characters:
+			//   . (period)        -> '．' // FULLWIDTH FULL STOP
+			//     (space)         -> '␠'  // SYMBOL FOR SPACE
+			//
+			// Also encode invalid UTF-8 bytes as json doesn't handle them.
+			//
+			// The OneDrive API documentation lists the set of reserved characters, but
+			// testing showed this list is incomplete. This are the differences:
+			//  - " (double quote) is rejected, but missing in the documentation
+			//  - space at the end of file and folder names is rejected, but missing in the documentation
+			//  - period at the end of file names is rejected, but missing in the documentation
+			//
+			// Adding these restrictions to the OneDrive API documentation yields exactly
+			// the same rules as the Windows naming conventions.
+			//
+			// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeHashPercent |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeLeftTilde |
+				encoder.EncodeRightPeriod |
+				encoder.EncodeRightSpace |
+				encoder.EncodeWin |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize          fs.SizeSuffix `config:"chunk_size"`
-	DriveID            string        `config:"drive_id"`
-	DriveType          string        `config:"drive_type"`
-	ExposeOneNoteFiles bool          `config:"expose_onenote_files"`
+	ChunkSize               fs.SizeSuffix        `config:"chunk_size"`
+	DriveID                 string               `config:"drive_id"`
+	DriveType               string               `config:"drive_type"`
+	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
+	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
+	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote one drive
@@ -308,7 +396,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an one drive 'url'
+// parsePath parses a one drive 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -327,13 +415,32 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
-
-	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
-		fs.Debugf(nil, "Should retry: %v", err)
+	retry := false
+	if resp != nil {
+		switch resp.StatusCode {
+		case 401:
+			if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
+				retry = true
+				fs.Debugf(nil, "Should retry: %v", err)
+			}
+		case 429: // Too Many Requests.
+			// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
+			if values := resp.Header["Retry-After"]; len(values) == 1 && values[0] != "" {
+				retryAfter, parseErr := strconv.Atoi(values[0])
+				if parseErr != nil {
+					fs.Debugf(nil, "Failed to parse Retry-After: %q: %v", values[0], parseErr)
+				} else {
+					duration := time.Second * time.Duration(retryAfter)
+					retry = true
+					err = pacer.RetryAfterError(err, duration)
+					fs.Debugf(nil, "Too many requests. Trying again in %d seconds.", retryAfter)
+				}
+			}
+		case 507: // Insufficient Storage
+			return false, fserrors.FatalError(err)
+		}
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return retry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
@@ -343,10 +450,15 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // instead of simply using `drives/driveID/root:/itemPath` because it works for
 // "shared with me" folders in OneDrive Personal (See #2536, #2778)
 // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
-func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
+//
+// If `relPath` == '', do not append the slash (See #3664)
+func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
+	if relPath != "" {
+		relPath = "/" + withTrailingColon(rest.URLPathEscape(f.opt.Enc.FromStandardPath(relPath)))
+	}
+	opts := newOptsCall(normalizedID, "GET", ":"+relPath)
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, nil, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
 		return shouldRetry(resp, err)
 	})
 
@@ -367,11 +479,11 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		} else {
 			opts = rest.Opts{
 				Method: "GET",
-				Path:   "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
+				Path:   "/root:/" + rest.URLPathEscape(f.opt.Enc.FromStandardPath(path)),
 			}
 		}
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(&opts, nil, &info)
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
 			return shouldRetry(resp, err)
 		})
 		return info, resp, err
@@ -386,13 +498,13 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var dirCacheRootIDExists bool
-		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
+		rootNormalizedID, err = f.dirCache.RootID(ctx, false)
+		dirCacheRootIDExists := err == nil
 		if f.root == "" {
 			// if f.root == "", it means f.root is the absolute root of the drive
 			// and its ID should have been found in NewFs
 			dirCacheFoundRoot = dirCacheRootIDExists
-		} else if _, err := f.dirCache.RootParentID(); err == nil {
+		} else if _, err := f.dirCache.RootParentID(ctx, false); err == nil {
 			// if root is in a folder, it must have a parent folder, and
 			// if dirCache has found root in NewFs, the parent folder's ID
 			// should be present.
@@ -426,7 +538,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		}
 	}
 
-	return f.readMetaDataForPathRelativeToID(baseNormalizedID, relPath)
+	return f.readMetaDataForPathRelativeToID(ctx, baseNormalizedID, relPath)
 }
 
 // errorHandler parses a non 2xx error response into an error
@@ -500,6 +612,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		CaseInsensitive:         true,
 		ReadMimeType:            true,
 		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f)
 	f.srv.SetErrorHandler(errorHandler)
 
@@ -592,7 +705,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	if !ok {
 		return "", false, errors.New("couldn't find parent ID")
 	}
-	info, resp, err := f.readMetaDataForPathRelativeToID(pathID, leaf)
+	info, resp, err := f.readMetaDataForPathRelativeToID(ctx, pathID, leaf)
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
 			return "", false, nil
@@ -615,11 +728,11 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
 	var info *api.Item
 	opts := newOptsCall(dirID, "POST", "/children")
 	mkdir := api.CreateItemRequest{
-		Name:             replaceReservedChars(leaf),
+		Name:             f.opt.Enc.FromStandardName(leaf),
 		ConflictBehavior: "fail",
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &mkdir, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &mkdir, &info)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -642,7 +755,7 @@ type listAllFn func(*api.Item) bool
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
-func (f *Fs) listAll(dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
+func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, filesOnly bool, fn listAllFn) (found bool, err error) {
 	// Top parameter asks for bigger pages of data
 	// https://dev.onedrive.com/odata/optional-query-parameters.htm
 	opts := newOptsCall(dirID, "GET", "/children?$top=1000")
@@ -651,7 +764,7 @@ OUTER:
 		var result api.ListChildrenResponse
 		var resp *http.Response
 		err = f.pacer.Call(func() (bool, error) {
-			resp, err = f.srv.CallJSON(&opts, nil, &result)
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
 			return shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -675,7 +788,7 @@ OUTER:
 			if item.Deleted != nil {
 				continue
 			}
-			item.Name = restoreReservedChars(item.GetName())
+			item.Name = f.opt.Enc.ToStandardName(item.GetName())
 			if fn(item) {
 				found = true
 				break OUTER
@@ -700,16 +813,12 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
 	}
 	var iErr error
-	_, err = f.listAll(directoryID, false, false, func(info *api.Item) bool {
+	_, err = f.listAll(ctx, directoryID, false, false, func(info *api.Item) bool {
 		if !f.opt.ExposeOneNoteFiles && info.GetPackageType() == api.PackageTypeOneNote {
 			fs.Debugf(info.Name, "OneNote file not shown in directory listing")
 			return false
@@ -751,7 +860,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, leaf, directoryID, err
 	}
@@ -782,23 +891,17 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
 // deleteObject removes an object by ID
-func (f *Fs) deleteObject(id string) error {
+func (f *Fs) deleteObject(ctx context.Context, id string) error {
 	opts := newOptsCall(id, "DELETE", "")
 	opts.NoResponse = true
 
 	return f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.Call(&opts)
+		resp, err := f.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 }
@@ -811,17 +914,13 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
 	}
 	if check {
 		// check to see if there are any items
-		found, err := f.listAll(rootID, false, false, func(item *api.Item) bool {
+		found, err := f.listAll(ctx, rootID, false, false, func(item *api.Item) bool {
 			return true
 		})
 		if err != nil {
@@ -831,7 +930,7 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 			return fs.ErrorDirectoryNotEmpty
 		}
 	}
-	err = f.deleteObject(rootID)
+	err = f.deleteObject(ctx, rootID)
 	if err != nil {
 		return err
 	}
@@ -912,10 +1011,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	srcPath := srcObj.fs.rootSlash() + srcObj.remote
-	dstPath := f.rootSlash() + remote
-	if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
-		return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
+	// Check we aren't overwriting a file on the same remote
+	if srcObj.fs == f {
+		srcPath := srcObj.rootPath()
+		dstPath := f.rootPath(remote)
+		if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
+			return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
+		}
 	}
 
 	// Create temporary object
@@ -931,7 +1033,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	id, dstDriveID, _ := parseNormalizedID(directoryID)
 
-	replacedLeaf := replaceReservedChars(leaf)
+	replacedLeaf := f.opt.Enc.FromStandardName(leaf)
 	copyReq := api.CopyItemRequest{
 		Name: &replacedLeaf,
 		ParentReference: api.ItemReference{
@@ -941,7 +1043,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &copyReq, nil)
+		resp, err = f.srv.CallJSON(ctx, &opts, &copyReq, nil)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1015,7 +1117,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	opts := newOptsCall(srcObj.id, "PATCH", "")
 
 	move := api.MoveItemRequest{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		ParentReference: &api.ItemReference{
 			DriveID: dstDriveID,
 			ID:      id,
@@ -1029,7 +1131,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	var resp *http.Response
 	var info api.Item
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &move, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1057,50 +1159,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, dstDirectoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
 	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
 	_, srcDriveID, _ := parseNormalizedID(srcID)
 
 	if dstDriveID != srcDriveID {
@@ -1109,20 +1174,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorCantDirMove
 	}
 
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
 	// Get timestamps of src so they can be preserved
-	srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(srcID, "")
+	srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(ctx, srcID, "")
 	if err != nil {
 		return err
 	}
@@ -1130,7 +1183,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	// Do the move
 	opts := newOptsCall(srcID, "PATCH", "")
 	move := api.MoveItemRequest{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(dstLeaf),
 		ParentReference: &api.ItemReference{
 			DriveID: dstDriveID,
 			ID:      parsedDstDirID,
@@ -1144,7 +1197,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	var resp *http.Response
 	var info api.Item
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &move, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, &move, &info)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1170,7 +1223,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	}
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, nil, &drive)
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &drive)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1191,12 +1244,12 @@ func (f *Fs) Hashes() hash.Set {
 	if f.driveType == driveTypePersonal {
 		return hash.Set(hash.SHA1)
 	}
-	return hash.Set(hash.QuickXorHash)
+	return hash.Set(QuickXorHashType)
 }
 
-// PublicLink returns a link for downloading without accout.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
-	info, _, err := f.readMetaDataForPath(ctx, f.srvPath(remote))
+// PublicLink returns a link for downloading without account.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
+	info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
 	if err != nil {
 		return "", err
 	}
@@ -1210,7 +1263,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 	var resp *http.Response
 	var result api.CreateShareLinkResponse
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, &share, &result)
+		resp, err = f.srv.CallJSON(ctx, &opts, &share, &result)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1240,9 +1293,19 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
+// rootPath returns a path for use in server given a remote
+func (f *Fs) rootPath(remote string) string {
+	return f.rootSlash() + remote
+}
+
+// rootPath returns a path for use in local functions
+func (o *Object) rootPath() string {
+	return o.fs.rootPath(o.remote)
+}
+
 // srvPath returns a path for use in server given a remote
 func (f *Fs) srvPath(remote string) string {
-	return replaceReservedChars(f.rootSlash() + remote)
+	return f.opt.Enc.FromStandardPath(f.rootSlash() + remote)
 }
 
 // srvPath returns a path for use in server
@@ -1257,7 +1320,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 			return o.sha1, nil
 		}
 	} else {
-		if t == hash.QuickXorHash {
+		if t == QuickXorHashType {
 			return o.quickxorhash, nil
 		}
 	}
@@ -1319,7 +1382,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.hasMetaData {
 		return nil
 	}
-	info, _, err := o.fs.readMetaDataForPath(ctx, o.srvPath())
+	info, _, err := o.fs.readMetaDataForPath(ctx, o.rootPath())
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
 			if apiErr.ErrorInfo.Code == "itemNotFound" {
@@ -1354,7 +1417,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 	} else {
 		opts = rest.Opts{
@@ -1370,7 +1433,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 	}
 	var info *api.Item
 	err := o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(&opts, &update, &info)
+		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
 		return shouldRetry(resp, err)
 	})
 	return info, err
@@ -1405,7 +1468,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	opts.Options = options
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1428,7 +1491,8 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 		opts = rest.Opts{
 			Method:  "POST",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(replaceReservedChars(leaf)) + ":/createUploadSession",
+			Path: fmt.Sprintf("/%s/items/%s:/%s:/createUploadSession",
+				drive, id, rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 	} else {
 		opts = rest.Opts{
@@ -1441,7 +1505,7 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 	createRequest.Item.FileSystemInfo.LastModifiedDateTime = api.Timestamp(modTime)
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(&opts, &createRequest, &response)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, &createRequest, &response)
 		if apiErr, ok := err.(*api.Error); ok {
 			if apiErr.ErrorInfo.Code == "nameAlreadyExists" {
 				// Make the error more user-friendly
@@ -1453,39 +1517,95 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 	return response, err
 }
 
-// uploadFragment uploads a part
-func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) {
+// getPosition gets the current position in a multipart upload
+func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err error) {
 	opts := rest.Opts{
-		Method:        "PUT",
-		RootURL:       url,
-		ContentLength: &chunkSize,
-		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", start, start+chunkSize-1, totalSize),
-		Body:          chunk,
+		Method:  "GET",
+		RootURL: url,
 	}
-	//	var response api.UploadFragmentResponse
+	var info api.UploadFragmentResponse
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, _ = chunk.Seek(0, io.SeekStart)
-		resp, err = o.fs.srv.Call(&opts)
-		if resp != nil {
-			defer fs.CheckClose(resp.Body, &err)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return 0, err
+	}
+	if len(info.NextExpectedRanges) != 1 {
+		return 0, errors.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
+	}
+	position := info.NextExpectedRanges[0]
+	i := strings.IndexByte(position, '-')
+	if i < 0 {
+		return 0, errors.Errorf("no '-' in next expected range: %q", position)
+	}
+	position = position[:i]
+	pos, err = strconv.ParseInt(position, 10, 64)
+	if err != nil {
+		return 0, errors.Wrapf(err, "bad expected range: %q", position)
+	}
+	return pos, nil
+}
+
+// uploadFragment uploads a part
+func (o *Object) uploadFragment(ctx context.Context, url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64, options ...fs.OpenOption) (info *api.Item, err error) {
+	//	var response api.UploadFragmentResponse
+	var resp *http.Response
+	var body []byte
+	var skip = int64(0)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		toSend := chunkSize - skip
+		opts := rest.Opts{
+			Method:        "PUT",
+			RootURL:       url,
+			ContentLength: &toSend,
+			ContentRange:  fmt.Sprintf("bytes %d-%d/%d", start+skip, start+chunkSize-1, totalSize),
+			Body:          chunk,
+			Options:       options,
 		}
-		retry, err := shouldRetry(resp, err)
-		if !retry && resp != nil {
-			if resp.StatusCode == 200 || resp.StatusCode == 201 {
-				// we are done :)
-				// read the item
-				info = &api.Item{}
-				return false, json.NewDecoder(resp.Body).Decode(info)
+		_, _ = chunk.Seek(skip, io.SeekStart)
+		resp, err = o.fs.srv.Call(ctx, &opts)
+		if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
+			fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
+			pos, posErr := o.getPosition(ctx, url)
+			if posErr != nil {
+				fs.Debugf(o, "Failed to read position: %v", posErr)
+				return false, posErr
 			}
+			skip = pos - start
+			fs.Debugf(o, "Read position %d, chunk is %d..%d, bytes to skip = %d", pos, start, start+chunkSize, skip)
+			switch {
+			case skip < 0:
+				return false, errors.Wrapf(err, "sent block already (skip %d < 0), can't rewind", skip)
+			case skip > chunkSize:
+				return false, errors.Wrapf(err, "position is in the future (skip %d > chunkSize %d), can't skip forward", skip, chunkSize)
+			case skip == chunkSize:
+				fs.Debugf(o, "Skipping chunk as already sent (skip %d == chunkSize %d)", skip, chunkSize)
+				return false, nil
+			}
+			return true, errors.Wrapf(err, "retry this chunk skipping %d bytes", skip)
 		}
-		return retry, err
+		if err != nil {
+			return shouldRetry(resp, err)
+		}
+		body, err = rest.ReadBody(resp)
+		if err != nil {
+			return shouldRetry(resp, err)
+		}
+		if resp.StatusCode == 200 || resp.StatusCode == 201 {
+			// we are done :)
+			// read the item
+			info = &api.Item{}
+			return false, json.Unmarshal(body, info)
+		}
+		return false, nil
 	})
 	return info, err
 }
 
 // cancelUploadSession cancels an upload session
-func (o *Object) cancelUploadSession(url string) (err error) {
+func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error) {
 	opts := rest.Opts{
 		Method:     "DELETE",
 		RootURL:    url,
@@ -1493,53 +1613,34 @@ func (o *Object) cancelUploadSession(url string) (err error) {
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	return
 }
 
 // uploadMultipart uploads a file using multipart upload
-func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
 	if size <= 0 {
 		return nil, errors.New("unknown-sized upload not supported")
 	}
 
-	uploadURLChan := make(chan string, 1)
-	gracefulCancel := func() {
-		uploadURL, ok := <-uploadURLChan
-		// Reading from uploadURLChan blocks the atexit process until
-		// we are able to use uploadURL to cancel the upload
-		if !ok { // createUploadSession failed - no need to cancel upload
-			return
-		}
-
-		fs.Debugf(o, "Cancelling multipart upload")
-		cancelErr := o.cancelUploadSession(uploadURL)
-		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
-		}
-	}
-	cancelFuncHandle := atexit.Register(gracefulCancel)
-
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(ctx, modTime)
 	if err != nil {
-		close(uploadURLChan)
-		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
-	uploadURLChan <- uploadURL
 
-	defer func() {
-		if err != nil {
-			fs.Debugf(o, "Error encountered during upload: %v", err)
-			gracefulCancel()
+	// Cancel the session if something went wrong
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(o, "Cancelling multipart upload: %v", err)
+		cancelErr := o.cancelUploadSession(ctx, uploadURL)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-		atexit.Unregister(cancelFuncHandle)
-	}()
+	})()
 
 	// Upload the chunks
 	remaining := size
@@ -1551,7 +1652,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 		}
 		seg := readers.NewRepeatableReader(io.LimitReader(in, n))
 		fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
-		info, err = o.uploadFragment(uploadURL, position, size, seg, n)
+		info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)
 		if err != nil {
 			return nil, err
 		}
@@ -1564,7 +1665,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 
 // Update the content of a remote file within 4MB size in one single request
 // This function will set modtime after uploading, which will create a new version for the remote file
-func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
+func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
 	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
 		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
 	}
@@ -1578,9 +1679,10 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 		opts = rest.Opts{
 			Method:        "PUT",
 			RootURL:       rootURL,
-			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf) + ":/content",
+			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf)) + ":/content",
 			ContentLength: &size,
 			Body:          in,
+			Options:       options,
 		}
 	} else {
 		opts = rest.Opts{
@@ -1588,11 +1690,12 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 			Path:          "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
 			ContentLength: &size,
 			Body:          in,
+			Options:       options,
 		}
 	}
 
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
 		if apiErr, ok := err.(*api.Error); ok {
 			if apiErr.ErrorInfo.Code == "nameAlreadyExists" {
 				// Make the error more user-friendly
@@ -1629,9 +1732,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	var info *api.Item
 	if size > 0 {
-		info, err = o.uploadMultipart(ctx, in, size, modTime)
+		info, err = o.uploadMultipart(ctx, in, size, modTime, options...)
 	} else if size == 0 {
-		info, err = o.uploadSinglepart(ctx, in, size, modTime)
+		info, err = o.uploadSinglepart(ctx, in, size, modTime, options...)
 	} else {
 		return errors.New("unknown-sized upload not supported")
 	}
@@ -1644,7 +1747,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	return o.fs.deleteObject(o.id)
+	return o.fs.deleteObject(ctx, o.id)
 }
 
 // MimeType of an Object if known, "" otherwise

backend/onedrive/replace.go

@@ -1,91 +0,0 @@
-/*
-Translate file names for one drive
-
-OneDrive reserved characters
-
-The following characters are OneDrive reserved characters, and can't
-be used in OneDrive folder and file names.
-
-  onedrive-reserved  = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|"
-  onedrive-business-reserved
-                     = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" / "#" / "%"
-
-Note: Folder names can't end with a period (.).
-
-Note: OneDrive for Business file or folder names cannot begin with a
-tilde ('~').
-
-*/
-
-package onedrive
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// Onedrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		'*':  '＊', // FULLWIDTH ASTERISK
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'?':  '？', // FULLWIDTH QUESTION MARK
-		':':  '：', // FULLWIDTH COLON
-		'|':  '｜', // FULLWIDTH VERTICAL LINE
-		'#':  '＃', // FULLWIDTH NUMBER SIGN
-		'%':  '％', // FULLWIDTH PERCENT SIGN
-		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-		'.':  '．', // FULLWIDTH FULL STOP
-		'~':  '～', // FULLWIDTH TILDE
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	invCharMap           map[rune]rune
-	fixEndingInPeriod    = regexp.MustCompile(`\.(/|$)`)
-	fixStartingWithTilde = regexp.MustCompile(`(/|^)~`)
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// Folder names can't end with a period '.'
-	in = fixEndingInPeriod.ReplaceAllString(in, string(charMap['.'])+"$1")
-	// OneDrive for Business file or folder names cannot begin with a tilde '~'
-	in = fixStartingWithTilde.ReplaceAllString(in, "$1"+string(charMap['~']))
-	// Apparently file names can't start with space either
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Replace reserved characters
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != '.' && c != '~' && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/onedrive/replace_test.go

@@ -1,30 +0,0 @@
-package onedrive
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{`\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~`},
-		{`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~/＼＊＜＞？：｜＃％＂.~`},
-		{" leading space", "␠leading space"},
-		{"~leading tilde", "～leading tilde"},
-		{"trailing dot.", "trailing dot．"},
-		{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
-		{"~leading tilde/~leading tilde/~leading tilde", "～leading tilde/～leading tilde/～leading tilde"},
-		{"trailing dot./trailing dot./trailing dot.", "trailing dot．/trailing dot．/trailing dot．"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/opendrive/opendrive.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -20,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
@@ -47,14 +49,57 @@ func init() {
 			Help:       "Password.",
 			IsPassword: true,
 			Required:   true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// List of replaced characters:
+			//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+			//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+			//   : (colon)         -> '：' // FULLWIDTH COLON
+			//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+			//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+			//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+			//
+			// Additionally names can't begin or end with an ASCII whitespace.
+			// List of replaced characters:
+			//     (space)           -> '␠'  // SYMBOL FOR SPACE
+			//     (horizontal tab)  -> '␉'  // SYMBOL FOR HORIZONTAL TABULATION
+			//     (line feed)       -> '␊'  // SYMBOL FOR LINE FEED
+			//     (vertical tab)    -> '␋'  // SYMBOL FOR VERTICAL TABULATION
+			//     (carriage return) -> '␍'  // SYMBOL FOR CARRIAGE RETURN
+			//
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			//
+			// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
+			Default: (encoder.Base |
+				encoder.EncodeWin |
+				encoder.EncodeLeftCrLfHtVt |
+				encoder.EncodeRightCrLfHtVt |
+				encoder.EncodeBackSlash |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
+		}, {
+			Name: "chunk_size",
+			Help: `Files will be uploaded in chunks this size.
+
+Note that these chunks are buffered in memory so increasing them will
+increase memory use.`,
+			Default:  10 * fs.MebiByte,
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	UserName string `config:"username"`
-	Password string `config:"password"`
+	UserName  string               `config:"username"`
+	Password  string               `config:"password"`
+	Enc       encoder.MultiEncoder `config:"encoding"`
+	ChunkSize fs.SizeSuffix        `config:"chunk_size"`
 }
 
 // Fs represents a remote server
@@ -161,7 +206,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			Method: "POST",
 			Path:   "/session/login.json",
 		}
-		resp, err = f.srv.CallJSON(&opts, &account, &f.session)
+		resp, err = f.srv.CallJSON(ctx, &opts, &account, &f.session)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -235,18 +280,12 @@ func errorHandler(resp *http.Response) error {
 // Mkdir creates the folder if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	// fs.Debugf(nil, "Mkdir(\"%s\")", dir)
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
 // deleteObject removes an object by ID
-func (f *Fs) deleteObject(id string) error {
+func (f *Fs) deleteObject(ctx context.Context, id string) error {
 	return f.pacer.Call(func() (bool, error) {
 		removeDirData := removeFolder{SessionID: f.session.SessionID, FolderID: id}
 		opts := rest.Opts{
@@ -254,7 +293,7 @@ func (f *Fs) deleteObject(id string) error {
 			NoResponse: true,
 			Path:       "/folder/remove.json",
 		}
-		resp, err := f.srv.CallJSON(&opts, &removeDirData, nil)
+		resp, err := f.srv.CallJSON(ctx, &opts, &removeDirData, nil)
 		return f.shouldRetry(resp, err)
 	})
 }
@@ -267,22 +306,18 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
 	}
-	item, err := f.readMetaDataForFolderID(rootID)
+	item, err := f.readMetaDataForFolderID(ctx, rootID)
 	if err != nil {
 		return err
 	}
 	if check && len(item.Files) != 0 {
 		return errors.New("folder not empty")
 	}
-	err = f.deleteObject(rootID)
+	err = f.deleteObject(ctx, rootID)
 	if err != nil {
 		return err
 	}
@@ -353,7 +388,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Method: "POST",
 			Path:   "/file/move_copy.json",
 		}
-		resp, err = f.srv.CallJSON(&opts, &copyFileData, &response)
+		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -410,7 +445,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Method: "POST",
 			Path:   "/file/move_copy.json",
 		}
-		resp, err = f.srv.CallJSON(&opts, &copyFileData, &response)
+		resp, err = f.srv.CallJSON(ctx, &opts, &copyFileData, &response)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -438,58 +473,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -501,15 +486,15 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		moveFolderData := moveCopyFolder{
 			SessionID:     f.session.SessionID,
 			FolderID:      srcID,
-			DstFolderID:   directoryID,
+			DstFolderID:   dstDirectoryID,
 			Move:          "true",
-			NewFolderName: leaf,
+			NewFolderName: dstLeaf,
 		}
 		opts := rest.Opts{
 			Method: "POST",
 			Path:   "/folder/move_copy.json",
 		}
-		resp, err = f.srv.CallJSON(&opts, &moveFolderData, &response)
+		resp, err = f.srv.CallJSON(ctx, &opts, &moveFolderData, &response)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -575,7 +560,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, leaf, directoryID, err
 	}
@@ -585,18 +570,18 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 		fs:     f,
 		remote: remote,
 	}
-	return o, leaf, directoryID, nil
+	return o, f.opt.Enc.FromStandardName(leaf), directoryID, nil
 }
 
 // readMetaDataForPath reads the metadata from the path
-func (f *Fs) readMetaDataForFolderID(id string) (info *FolderList, err error) {
+func (f *Fs) readMetaDataForFolderID(ctx context.Context, id string) (info *FolderList, err error) {
 	var resp *http.Response
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   "/folder/list.json/" + f.session.SessionID + "/" + id,
 	}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, nil, &info)
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -632,16 +617,21 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	}
 
 	if "" == o.id {
-		// We need to create a ID for this file
+		// We need to create an ID for this file
 		var resp *http.Response
 		response := createFileResponse{}
 		err := o.fs.pacer.Call(func() (bool, error) {
-			createFileData := createFile{SessionID: o.fs.session.SessionID, FolderID: directoryID, Name: replaceReservedChars(leaf)}
-			opts := rest.Opts{
-				Method: "POST",
-				Path:   "/upload/create_file.json",
+			createFileData := createFile{
+				SessionID: o.fs.session.SessionID,
+				FolderID:  directoryID,
+				Name:      leaf,
 			}
-			resp, err = o.fs.srv.CallJSON(&opts, &createFileData, &response)
+			opts := rest.Opts{
+				Method:  "POST",
+				Options: options,
+				Path:    "/upload/create_file.json",
+			}
+			resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
 			return o.fs.shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -683,7 +673,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		createDirData := createFolder{
 			SessionID:           f.session.SessionID,
-			FolderName:          replaceReservedChars(leaf),
+			FolderName:          f.opt.Enc.FromStandardName(leaf),
 			FolderSubParent:     pathID,
 			FolderIsPublic:      0,
 			FolderPublicUpl:     0,
@@ -694,7 +684,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			Method: "POST",
 			Path:   "/folder.json",
 		}
-		resp, err = f.srv.CallJSON(&opts, &createDirData, &response)
+		resp, err = f.srv.CallJSON(ctx, &opts, &createDirData, &response)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -722,15 +712,15 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 			Method: "GET",
 			Path:   "/folder/list.json/" + f.session.SessionID + "/" + pathID,
 		}
-		resp, err = f.srv.CallJSON(&opts, nil, &folderList)
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return "", false, errors.Wrap(err, "failed to get folder list")
 	}
 
+	leaf = f.opt.Enc.FromStandardName(leaf)
 	for _, folder := range folderList.Folders {
-		folder.Name = restoreReservedChars(folder.Name)
 		// fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID)
 
 		if leaf == folder.Name {
@@ -753,10 +743,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	// fs.Debugf(nil, "List(%v)", dir)
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -769,7 +755,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	folderList := FolderList{}
 	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(&opts, nil, &folderList)
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &folderList)
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -777,7 +763,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	for _, folder := range folderList.Folders {
-		folder.Name = restoreReservedChars(folder.Name)
+		folder.Name = f.opt.Enc.ToStandardName(folder.Name)
 		// fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID)
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
@@ -788,7 +774,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	for _, file := range folderList.Files {
-		file.Name = restoreReservedChars(file.Name)
+		file.Name = f.opt.Enc.ToStandardName(file.Name)
 		// fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID)
 		remote := path.Join(dir, file.Name)
 		o, err := f.newObjectWithInfo(ctx, remote, &file)
@@ -851,9 +837,13 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 		NoResponse: true,
 		Path:       "/file/filesettings.json",
 	}
-	update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)}
+	update := modTimeFile{
+		SessionID:            o.fs.session.SessionID,
+		FileID:               o.id,
+		FileModificationTime: strconv.FormatInt(modTime.Unix(), 10),
+	}
 	err := o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(&opts, &update, nil)
+		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
 		return o.fs.shouldRetry(resp, err)
 	})
 
@@ -873,7 +863,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err = o.fs.srv.Call(&opts)
+		resp, err = o.fs.srv.Call(ctx, &opts)
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -892,7 +882,7 @@ func (o *Object) Remove(ctx context.Context) error {
 			NoResponse: true,
 			Path:       "/file.json/" + o.fs.session.SessionID + "/" + o.id,
 		}
-		resp, err := o.fs.srv.Call(&opts)
+		resp, err := o.fs.srv.Call(ctx, &opts)
 		return o.fs.shouldRetry(resp, err)
 	})
 }
@@ -917,10 +907,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		openUploadData := openUpload{SessionID: o.fs.session.SessionID, FileID: o.id, Size: size}
 		// fs.Debugf(nil, "PreOpen: %#v", openUploadData)
 		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/upload/open_file_upload.json",
+			Method:  "POST",
+			Options: options,
+			Path:    "/upload/open_file_upload.json",
 		}
-		resp, err := o.fs.srv.CallJSON(&opts, &openUploadData, &openResponse)
+		resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -929,15 +920,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// resp.Body.Close()
 	// fs.Debugf(nil, "PostOpen: %#v", openResponse)
 
-	// 10 MB chunks size
-	chunkSize := int64(1024 * 1024 * 10)
-	buf := make([]byte, int(chunkSize))
+	buf := make([]byte, o.fs.opt.ChunkSize)
 	chunkOffset := int64(0)
 	remainingBytes := size
 	chunkCounter := 0
 
 	for remainingBytes > 0 {
-		currentChunkSize := chunkSize
+		currentChunkSize := int64(o.fs.opt.ChunkSize)
 		if currentChunkSize > remainingBytes {
 			currentChunkSize = remainingBytes
 		}
@@ -966,7 +955,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 				MultipartFileName:    o.remote,    // ..name of the file for the attached file
 
 			}
-			resp, err = o.fs.srv.CallJSON(&opts, nil, &reply)
+			resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &reply)
 			return o.fs.shouldRetry(resp, err)
 		})
 		if err != nil {
@@ -989,7 +978,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			Method: "POST",
 			Path:   "/upload/close_file_upload.json",
 		}
-		resp, err = o.fs.srv.CallJSON(&opts, &closeUploadData, &closeResponse)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, &closeUploadData, &closeResponse)
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1015,7 +1004,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			NoResponse: true,
 			Path:       "/file/access.json",
 		}
-		resp, err = o.fs.srv.CallJSON(&opts, &update, nil)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, &update, nil)
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1026,7 +1015,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 func (o *Object) readMetaData(ctx context.Context) (err error) {
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return fs.ErrorObjectNotFound
@@ -1038,9 +1027,10 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts := rest.Opts{
 			Method: "GET",
-			Path:   "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)),
+			Path: fmt.Sprintf("/folder/itembyname.json/%s/%s?name=%s",
+				o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
-		resp, err = o.fs.srv.CallJSON(&opts, nil, &folderList)
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {

backend/opendrive/replace.go

@@ -1,78 +0,0 @@
-/*
-Translate file names for OpenDrive
-
-OpenDrive reserved characters
-
-The following characters are OpenDrive reserved characters, and can't
-be used in OpenDrive folder and file names.
-
-\ / : * ? " < > |
-
-OpenDrive files and folders can't have leading or trailing spaces also.
-
-*/
-
-package opendrive
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// OpenDrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		':':  '：', // FULLWIDTH COLON
-		'*':  '＊', // FULLWIDTH ASTERISK
-		'?':  '？', // FULLWIDTH QUESTION MARK
-		'"':  '＂', // FULLWIDTH QUOTATION MARK
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'|':  '｜', // FULLWIDTH VERTICAL LINE
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-	fixEndingWithSpace   = regexp.MustCompile(` (/|$)`)
-	invCharMap           map[rune]rune
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// Filenames can't start with space
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Filenames can't end with space
-	in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}