vfs: factor the vfs cache into its own package

See: https://forum.rclone.org/t/directory-name-encryption-is-set-to-always-false-when-choosing-filename-encryption-off/14600

.appveyor.yml

@@ -1,50 +0,0 @@
-version: "{build}"
-
-os: Windows Server 2012 R2
-
-clone_folder: c:\gopath\src\github.com\rclone\rclone
-
-cache:
-  - '%LocalAppData%\go-build'
-
-environment:
-  GOPATH: C:\gopath
-  CPATH: C:\Program Files (x86)\WinFsp\inc\fuse
-  ORIGPATH: '%PATH%'
-  NOCCPATH: C:\MinGW\bin;%GOPATH%\bin;%PATH%
-  PATHCC64: C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%NOCCPATH%
-  PATHCC32: C:\mingw-w64\i686-6.3.0-posix-dwarf-rt_v5-rev1\mingw32\bin;%NOCCPATH%
-  PATH: '%PATHCC64%'
-  RCLONE_CONFIG_PASS:
-    secure: sq9CPBbwaeKJv+yd24U44neORYPQVy6jsjnQptC+5yk=
-
-install:
-- choco install winfsp -y
-- choco install zip -y
-- copy c:\MinGW\bin\mingw32-make.exe c:\MinGW\bin\make.exe
-
-build_script:
-- echo %PATH%
-- echo %GOPATH%
-- go version
-- go env
-- go install
-- go build
-- make log_since_last_release > %TEMP%\git-log.txt
-- make version > %TEMP%\version
-- set /p RCLONE_VERSION=<%TEMP%\version
-- set PATH=%PATHCC32%
-- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/386" -cgo -tags cmount %RCLONE_VERSION%
-- set PATH=%PATHCC64%
-- go run bin/cross-compile.go -release beta-latest -git-log %TEMP%\git-log.txt -include "^windows/amd64" -cgo -no-clean -tags cmount %RCLONE_VERSION%
-
-test_script:
-- make GOTAGS=cmount quicktest
-- make GOTAGS=cmount racequicktest
-
-artifacts:
-- path: rclone.exe
-- path: build/*-v*.zip
-
-deploy_script:
-- IF "%APPVEYOR_REPO_NAME%" == "rclone/rclone" IF "%APPVEYOR_PULL_REQUEST_NUMBER%" == "" make appveyor_upload

.circleci/config.yml

@@ -1,43 +0,0 @@
----
-version: 2
-
-jobs:
-
-  build:
-    machine: true
-
-    working_directory: ~/.go_workspace/src/github.com/rclone/rclone
-
-    steps:
-    - checkout
-
-    - run:
-        name: Cross-compile rclone
-        command: |
-            docker pull billziss/xgo-cgofuse
-            go get -v github.com/karalabe/xgo
-            xgo \
-                -image=billziss/xgo-cgofuse \
-                -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
-                -tags cmount \
-                -dest build \
-                .
-            xgo \
-                -image=billziss/xgo-cgofuse \
-                -targets=android/*,ios/* \
-                -dest build \
-                .
-
-    - run:
-        name: Build rclone
-        command: |
-            docker pull golang
-            docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=vendor -v
-
-    - run:
-        name: Upload artifacts
-        command: |
-          make circleci_upload
-
-    - store_artifacts:
-        path: build

.github/workflows/build.yml

@@ -19,13 +19,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'modules_race', 'go1.10', 'go1.11', 'go1.12']
+        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'modules_race', 'go1.11', 'go1.12', 'go1.13']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.13.x'
-            modules: off
+            go: '1.14.x'
+            modules: 'off'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
@@ -34,8 +34,8 @@ jobs:
 
           - job_name: mac
             os: macOS-latest
-            go: '1.13.x'
-            modules: off
+            go: '1.14.x'
+            modules: 'off'
             gotags: ''  # cmount doesn't work on osx travis for some reason
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
@@ -44,8 +44,8 @@ jobs:
 
           - job_name: windows_amd64
             os: windows-latest
-            go: '1.13.x'
-            modules: off
+            go: '1.14.x'
+            modules: 'off'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             quicktest: true
@@ -54,8 +54,8 @@ jobs:
 
           - job_name: windows_386
             os: windows-latest
-            go: '1.13.x'
-            modules: off
+            go: '1.14.x'
+            modules: 'off'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -65,35 +65,35 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.13.x'
-            modules: off
+            go: '1.14.x'
+            modules: 'off'
             build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
             compile_all: true
             deploy: true
 
           - job_name: modules_race
             os: ubuntu-latest
-            go: '1.13.x'
-            modules: on
+            go: '1.14.x'
+            modules: 'on'
             quicktest: true
             racequicktest: true
 
-          - job_name: go1.10
-            os: ubuntu-latest
-            go: '1.10.x'
-            modules: off
-            quicktest: true
-
           - job_name: go1.11
             os: ubuntu-latest
             go: '1.11.x'
-            modules: off
+            modules: 'off'
             quicktest: true
 
           - job_name: go1.12
             os: ubuntu-latest
             go: '1.12.x'
-            modules: off
+            modules: 'off'
+            quicktest: true
+
+          - job_name: go1.13
+            os: ubuntu-latest
+            go: '1.13.x'
+            modules: 'off'
             quicktest: true
 
     name: ${{ matrix.job_name }}
@@ -102,9 +102,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@master
+        uses: actions/checkout@v1
         with:
-          path: ./src/github.com/${{ github.repository }}
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
 
       - name: Install Go
         uses: actions/setup-go@v1
@@ -135,7 +136,6 @@ jobs:
         shell: bash
         run: |
           brew update
-          brew tap caskroom/cask
           brew cask install osxfuse
         if: matrix.os == 'macOS-latest'
 
@@ -197,10 +197,58 @@ jobs:
       - name: Deploy built binaries
         shell: bash
         run: |
-          make release_dep
+          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep ; fi
           make travis_beta
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
-          BETA_SUBDIR: 'github_actions'  # FIXME remove when removing travis/appveyor
         # working-directory: '$(modulePath)'
-        if: matrix.deploy && github.head_ref == ''
+        # Deploy binaries if enabled in config && not a PR && not a fork
+        if: matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
+
+  xgo:
+    timeout-minutes: 60
+    name: "xgo cross compile"
+    runs-on: ubuntu-latest
+
+    steps:
+
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          echo '::set-env name=GOPATH::${{ runner.workspace }}'
+          echo '::add-path::${{ runner.workspace }}/bin'
+
+      - name: Cross-compile rclone
+        run: |
+          docker pull billziss/xgo-cgofuse
+          go get -v github.com/karalabe/xgo
+          xgo \
+              -image=billziss/xgo-cgofuse \
+              -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+              -tags cmount \
+              -dest build \
+              .
+          xgo \
+              -image=billziss/xgo-cgofuse \
+              -targets=android/*,ios/* \
+              -dest build \
+              .
+
+      - name: Build rclone
+        run: |
+          docker pull golang
+          docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=vendor -v
+
+      - name: Upload artifacts
+        run: |
+          make circleci_upload
+        env:
+          RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
+        # Upload artifacts if not a PR && not a fork
+        if: github.head_ref == '' && github.repository == 'rclone/rclone'

.gitignore

@@ -7,4 +7,5 @@ rclone.iml
 .idea
 .history
 *.test
-*.log
+*.log
+*.iml

.pkgr.yml

@@ -1,2 +0,0 @@
-default_dependencies: false
-cli: rclone

.travis.yml

@@ -1,128 +0,0 @@
----
-language: go
-sudo: required
-dist: xenial
-os:
-  - linux
-go_import_path: github.com/rclone/rclone
-before_install:
-  - git fetch --unshallow --tags
-  - |
-    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
-      sudo modprobe fuse
-      sudo chmod 666 /dev/fuse
-      sudo chown root:$USER /etc/fuse.conf
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
-      brew update
-      brew tap caskroom/cask
-      brew cask install osxfuse
-    fi
-    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
-      choco install -y winfsp zip make
-      cd ../..                  # fix crlf in git checkout
-      mv $TRAVIS_REPO_SLUG _old
-      git config --global core.autocrlf false
-      git clone _old $TRAVIS_REPO_SLUG
-      cd $TRAVIS_REPO_SLUG
-    fi
-install:
-  - make vars
-env:
-  global:
-    - GOTAGS=cmount
-    - GOMAXPROCS=8              # workaround for cmd/mount tests locking up - see #3154
-    - GO111MODULE=off
-    - GITHUB_USER=ncw
-    - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
-    - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=
-addons:
-  apt:
-    packages:
-      - fuse
-      - libfuse-dev
-      - rpm
-      - pkg-config
-cache:
-  directories:
-    - $HOME/.cache/go-build
-matrix:
-  allow_failures:
-    - go: tip
-  include:
-    - go: 1.10.x
-      script:
-        - make quicktest
-    - go: 1.11.x
-      script:
-        - make quicktest
-    - go: 1.12.x
-      script:
-        - make quicktest
-    - go: 1.13.x
-      name: Linux
-      env:
-        - GOTAGS=cmount
-        - BUILD_FLAGS='-include "^linux/"'
-        - DEPLOY=true
-      script:
-        - make build_dep
-        - make check
-        - make quicktest
-    - go: 1.13.x
-      name: Go Modules / Race
-      env:
-        - GO111MODULE=on
-        - GOPROXY=https://proxy.golang.org
-      script:
-        - make quicktest
-        - make racequicktest
-    - go: 1.13.x
-      name: Other OS
-      env:
-        - DEPLOY=true
-        - BUILD_FLAGS='-exclude "^(windows|darwin|linux)/"'
-      script:
-        - make
-        - make compile_all
-    - go: 1.13.x
-      name: macOS
-      os: osx
-      env:
-        - GOTAGS=  # cmount doesn't work on osx travis for some reason
-        - BUILD_FLAGS='-include "^darwin/" -cgo'
-        - DEPLOY=true
-      cache:
-        directories:
-          - $HOME/Library/Caches/go-build
-      script:
-        - make
-        - make quicktest
-        - make racequicktest
-    # - os: windows
-    #   name: Windows
-    #   go: 1.13.x
-    #   env:
-    #     - GOTAGS=cmount
-    #     - CPATH='C:\Program Files (x86)\WinFsp\inc\fuse'
-    #     - BUILD_FLAGS='-include "^windows/amd64" -cgo' # 386 doesn't build yet
-    #   #filter_secrets: false     # works around a problem with secrets under windows
-    #   cache:
-    #     directories:
-    #       - ${LocalAppData}/go-build
-    #   script:
-    #     - make
-    #     - make quicktest
-    #     - make racequicktest
-    - go: tip
-      script:
-        - make quicktest
-
-deploy:
-  provider: script
-  script: make travis_beta
-  skip_cleanup: true
-  on:
-    repo: rclone/rclone
-    all_branches: true
-    condition: $TRAVIS_PULL_REQUEST == false && $DEPLOY == true

CONTRIBUTING.md

@@ -82,13 +82,9 @@ You patch will get reviewed and you might get asked to fix some stuff.
 If so, then make the changes in the same branch, squash the commits,
 rebase it to master then push it to GitHub with `--force`.
 
-## Enabling CI for your fork ##
+## CI for your fork ##
 
-The CI config files for rclone have taken care of forks of the project, so you can enable CI for your fork repo easily.
-
-rclone currently uses [Travis CI](https://travis-ci.org/), [AppVeyor](https://ci.appveyor.com/), and
-[Circle CI](https://circleci.com/) to build the project. To enable them for your fork, simply go into their
-websites, find your fork of rclone, and enable building there.
+rclone currently uses [GitHub Actions](https://github.com/rclone/rclone/actions) to build and test the project, which should be automatically available for your fork too from the `Actions` tab in your repository.
 
 ## Testing ##
 
@@ -152,6 +148,7 @@ with modules beneath.
     * ...commands
   * docs - the documentation and website
     * content - adjust these docs only - everything else is autogenerated
+      * command - these are auto generated - edit the corresponding .go file
   * fs - main rclone definitions - minimal amount of code
     * accounting - bandwidth limiting and statistics
     * asyncreader - an io.Reader which reads ahead
@@ -207,6 +204,9 @@ don't need to run these when adding a feature.
 Documentation for rclone sub commands is with their code, eg
 `cmd/ls/ls.go`.
 
+Note that you can use [GitHub's online editor](https://help.github.com/en/github/managing-files-in-a-repository/editing-files-in-another-users-repository)
+for small changes in the docs which makes it very easy.
+
 ## Making a release ##
 
 There are separate instructions for making a release in the RELEASE.md
@@ -341,6 +341,11 @@ Getting going
   * Add your remote to the imports in `backend/all/all.go`
   * HTTP based remotes are easiest to maintain if they use rclone's rest module, but if there is a really good go SDK then use that instead.
   * Try to implement as many optional methods as possible as it makes the remote more usable.
+  * Use lib/encoder to make sure we can encode any path name and `rclone info` to help determine the encodings needed
+    * `rclone purge -v TestRemote:rclone-info`
+    * `rclone info --remote-encoding None -vv --write-json remote.json TestRemote:rclone-info`
+    * `go run cmd/info/internal/build_csv/main.go -o remote.csv remote.json`
+    * open `remote.csv` in a spreadsheet and examine
 
 Unit tests
 
@@ -353,7 +358,7 @@ Integration tests
   * Add your backend to `fstest/test_all/config.yaml`
       * Once you've done that then you can use the integration test framework from the project root:
       * go install ./...
-      * test_all -backend remote
+      * test_all -backends remote
 
 Or if you want to run the integration tests manually:
 
@@ -367,12 +372,54 @@ Or if you want to run the integration tests manually:
 
 See the [testing](#testing) section for more information on integration tests.
 
-Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in alphabetical order but with the local file system last.
+Add your fs to the docs - you'll need to pick an icon for it from
+[fontawesome](http://fontawesome.io/icons/).  Keep lists of remotes in
+alphabetical order of full name of remote (eg `drive` is ordered as
+`Google Drive`) but with the local file system last.
 
   * `README.md` - main GitHub page
   * `docs/content/remote.md` - main docs page (note the backend options are automatically added to this file with `make backenddocs`)
+    * make sure this has the `autogenerated options` comments in (see your reference backend docs)
+    * update them with `make backenddocs` - revert any changes in other backends
   * `docs/content/overview.md` - overview docs
   * `docs/content/docs.md` - list of remotes in config section
   * `docs/content/about.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
+
+Once you've written the docs, run `make serve` and check they look OK
+in the web browser and the links (internal and external) all work.
+
+## Writing a plugin ##
+
+New features (backends, commands) can also be added "out-of-tree", through Go plugins.
+Changes will be kept in a dynamically loaded file instead of being compiled into the main binary.
+This is useful if you can't merge your changes upstream or don't want to maintain a fork of rclone.
+
+Usage
+
+ - Naming
+   - Plugins names must have the pattern `librcloneplugin_KIND_NAME.so`.
+   - `KIND` should be one of `backend`, `command` or `bundle`.
+   - Example: A plugin with backend support for PiFS would be called
+     `librcloneplugin_backend_pifs.so`.
+ - Loading
+   - Supported on macOS & Linux as of now. ([Go issue for Windows support](https://github.com/golang/go/issues/19282))
+   - Supported on rclone v1.50 or greater.
+   - All plugins in the folder specified by variable `$RCLONE_PLUGIN_PATH` are loaded.
+   - If this variable doesn't exist, plugin support is disabled.
+   - Plugins must be compiled against the exact version of rclone to work.
+     (The rclone used during building the plugin must be the same as the source of rclone)
+     
+Building
+
+To turn your existing additions into a Go plugin, move them to an external repository
+and change the top-level package name to `main`.
+
+Check `rclone --version` and make sure that the plugin's rclone dependency and host Go version match.
+
+Then, run `go build -buildmode=plugin -o PLUGIN_NAME.so .` to build the plugin.
+
+[Go reference](https://godoc.org/github.com/rclone/rclone/lib/plugin)
+
+[Minimal example](https://gist.github.com/terorie/21b517ee347828e899e1913efc1d684f)

MAINTAINERS.md

@@ -12,6 +12,7 @@ Current active maintainers of rclone are:
 | Alex Chen        | @Cnly       | onedrive backend             |
 | Sandeep Ummadi   | @sandeepkru | azureblob backend            |
 | Sebastian Bünger | @buengese   | jottacloud & yandex backends |
+| Ivan Andreev     | @ivandeex   | chunker & mailru backends    |
 
 **This is a work in progress Draft**
 

Makefile

@@ -1,18 +1,29 @@
 SHELL = bash
+# Branch we are working on
 BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
+# Tag of the current commit, if any.  If this is not "" then we are building a release
+RELEASE_TAG := $(shell git tag -l --points-at HEAD)
+# Version of last release (may not be on this branch)
+VERSION := $(shell cat VERSION)
+# Last tag on this branch
 LAST_TAG := $(shell git describe --tags --abbrev=0)
-ifeq ($(BRANCH),$(LAST_TAG))
+# If we are working on a release, override branch to master
+ifdef RELEASE_TAG
 	BRANCH := master
 endif
 TAG_BRANCH := -$(BRANCH)
 BRANCH_PATH := branch/
+# If building HEAD or master then unset TAG_BRANCH and BRANCH_PATH
 ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	TAG_BRANCH :=
 	BRANCH_PATH :=
 endif
-TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
-NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
-ifneq ($(TAG),$(LAST_TAG))
+# Make version suffix -DDD-gCCCCCCCC (D=commits since last relase, C=Commit) or blank
+VERSION_SUFFIX := $(shell git describe --abbrev=8 --tags | perl -lpe 's/^v\d+\.\d+\.\d+//; s/^-(\d+)/"-".sprintf("%03d",$$1)/e;')
+# TAG is current version + number of commits since last release + branch
+TAG := $(VERSION)$(VERSION_SUFFIX)$(TAG_BRANCH)
+NEXT_VERSION := $(shell echo $(VERSION) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+ifndef RELEASE_TAG
 	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
@@ -35,7 +46,8 @@ endif
 rclone:
 	go build -v --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS)
 	mkdir -p `go env GOPATH`/bin/
-	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/
+	cp -av rclone`go env GOEXE` `go env GOPATH`/bin/rclone`go env GOEXE`.new
+	mv -v `go env GOPATH`/bin/rclone`go env GOEXE`.new `go env GOPATH`/bin/rclone`go env GOEXE`
 
 test_all:
 	go install --ldflags "-s -X github.com/rclone/rclone/fs.Version=$(TAG)" $(BUILDTAGS) github.com/rclone/rclone/fstest/test_all
@@ -44,8 +56,8 @@ vars:
 	@echo SHELL="'$(SHELL)'"
 	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
-	@echo LAST_TAG="'$(LAST_TAG)'"
-	@echo NEW_TAG="'$(NEW_TAG)'"
+	@echo VERSION="'$(VERSION)'"
+	@echo NEXT_VERSION="'$(NEXT_VERSION)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
@@ -76,8 +88,8 @@ build_dep:
 
 # Get the release dependencies
 release_dep:
-	go get -u github.com/goreleaser/nfpm/...
-	go get -u github.com/aktau/github-release
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+	go run bin/get-github-release.go -extract github-release aktau/github-release 'linux-amd64-github-release.tar.bz2'
 
 # Update dependencies
 update:
@@ -85,6 +97,11 @@ update:
 	GO111MODULE=on go mod tidy
 	GO111MODULE=on go mod vendor
 
+# Tidy the module dependencies
+tidy:
+	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
+
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
@@ -166,6 +183,9 @@ endif
 	@echo Beta release ready at $(BETA_URL)
 
 circleci_upload:
+	sudo chown -R $$USER build
+	find build -type l -delete
+	gzip -r9v build
 	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
 ifndef BRANCH_PATH
 	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
@@ -192,24 +212,25 @@ serve:	website
 	cd docs && hugo server -v -w
 
 tag:	doc
-	@echo "Old tag is $(LAST_TAG)"
-	@echo "New tag is $(NEW_TAG)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEW_TAG)\"\n" | gofmt > fs/version.go
-	echo -n "$(NEW_TAG)" > docs/layouts/partials/version.html
-	git tag -s -m "Version $(NEW_TAG)" $(NEW_TAG)
-	bin/make_changelog.py $(LAST_TAG) $(NEW_TAG) > docs/content/changelog.md.new
+	@echo "Old tag is $(VERSION)"
+	@echo "New tag is $(NEXT_VERSION)"
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)\"\n" | gofmt > fs/version.go
+	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
+	echo "$(NEXT_VERSION)" > VERSION
+	git tag -s -m "Version $(NEXT_VERSION)" $(NEXT_VERSION)
+	bin/make_changelog.py $(LAST_TAG) $(NEXT_VERSION) > docs/content/changelog.md.new
 	mv docs/content/changelog.md.new docs/content/changelog.md
 	@echo "Edit the new changelog in docs/content/changelog.md"
 	@echo "Then commit all the changes"
-	@echo git commit -m \"Version $(NEW_TAG)\" -a -v
+	@echo git commit -m \"Version $(NEXT_VERSION)\" -a -v
 	@echo "And finally run make retag before make cross etc"
 
 retag:
-	git tag -f -s -m "Version $(LAST_TAG)" $(LAST_TAG)
+	git tag -f -s -m "Version $(VERSION)" $(VERSION)
 
 startdev:
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(LAST_TAG)-DEV\"\n" | gofmt > fs/version.go
-	git commit -m "Start $(LAST_TAG)-DEV development" fs/version.go
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(VERSION)-DEV\"\n" | gofmt > fs/version.go
+	git commit -m "Start $(VERSION)-DEV development" fs/version.go
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -8,10 +8,7 @@
 [Installation](https://rclone.org/install/) |
 [Forum](https://forum.rclone.org/)
 
-[![Build Status](https://travis-ci.org/rclone/rclone.svg?branch=master)](https://travis-ci.org/rclone/rclone)
-[![Windows Build Status](https://ci.appveyor.com/api/projects/status/github/rclone/rclone?branch=master&passingText=windows%20-%20ok&svg=true)](https://ci.appveyor.com/project/rclone/rclone)
-[![Build Status](https://dev.azure.com/rclone/rclone/_apis/build/status/rclone.rclone?branchName=master)](https://dev.azure.com/rclone/rclone/_build/latest?definitionId=2&branchName=master)
-[![CircleCI](https://circleci.com/gh/rclone/rclone/tree/master.svg?style=svg)](https://circleci.com/gh/rclone/rclone/tree/master)
+[![Build Status](https://github.com/rclone/rclone/workflows/build/badge.svg)](https://github.com/rclone/rclone/actions?query=workflow%3Abuild)
 [![Go Report Card](https://goreportcard.com/badge/github.com/rclone/rclone)](https://goreportcard.com/report/github.com/rclone/rclone)
 [![GoDoc](https://godoc.org/github.com/rclone/rclone?status.svg)](https://godoc.org/github.com/rclone/rclone) 
 [![Docker Pulls](https://img.shields.io/docker/pulls/rclone/rclone)](https://hub.docker.com/r/rclone/rclone)
@@ -22,17 +19,19 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
 
 ## Storage providers
 
-  * 1Fichier [:page_facing_up:](https://rclone.org/ficher/)
+  * 1Fichier [:page_facing_up:](https://rclone.org/fichier/)
   * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
   * Box [:page_facing_up:](https://rclone.org/box/)
   * Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
+  * Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
   * DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
   * Dreamhost [:page_facing_up:](https://rclone.org/s3/#dreamhost)
   * Dropbox [:page_facing_up:](https://rclone.org/dropbox/)
   * FTP [:page_facing_up:](https://rclone.org/ftp/)
+  * GetSky [:page_facing_up:](https://rclone.org/jottacloud/)
   * Google Cloud Storage [:page_facing_up:](https://rclone.org/googlecloudstorage/)
   * Google Drive [:page_facing_up:](https://rclone.org/drive/)
   * Google Photos [:page_facing_up:](https://rclone.org/googlephotos/)
@@ -44,6 +43,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Mail.ru Cloud [:page_facing_up:](https://rclone.org/mailru/)
   * Memset Memstore [:page_facing_up:](https://rclone.org/swift/)
   * Mega [:page_facing_up:](https://rclone.org/mega/)
+  * Memory [:page_facing_up:](https://rclone.org/memory/)
   * Microsoft Azure Blob Storage [:page_facing_up:](https://rclone.org/azureblob/)
   * Microsoft OneDrive [:page_facing_up:](https://rclone.org/onedrive/)
   * Minio [:page_facing_up:](https://rclone.org/s3/#minio)
@@ -60,6 +60,8 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
+  * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
+  * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)
@@ -76,6 +78,7 @@ Please see [the full list of all storage providers and their features](https://r
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
   * Can sync to and from network, e.g. two different cloud accounts
+  * Optional large file chunking ([Chunker](https://rclone.org/chunker/))
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))

RELEASE.md

@@ -9,18 +9,20 @@ This file describes how to make the various kinds of releases
 
 ## Making a release
 
+  * git checkout master
+  * git pull
   * git status - make sure everything is checked in
-  * Check travis & appveyor builds are green
-  * make check
+  * Check GitHub actions build for master is Green
   * make test # see integration test server or run locally
   * make tag
-  * edit docs/content/changelog.md
+  * edit docs/content/changelog.md # make sure to remove duplicate logs from point releases
+  * make tidy
   * make doc
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
   * git push --tags origin master
-  * # Wait for the appveyor and travis builds to complete then...
+  * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
   * make tarball
   * make sign_upload
@@ -56,8 +58,7 @@ Can be fixed with
 
 ## Making a point release
 
-If rclone needs a point release due to some horrendous bug then a
-point release is necessary.
+If rclone needs a point release due to some horrendous bug:
 
 First make the release branch.  If this is a second point release then
 this will be done already.
@@ -72,7 +73,7 @@ Now
   * git co ${BASE_TAG}-fixes
   * git cherry-pick any fixes
   * Test (see above)
-  * make NEW_TAG=${NEW_TAG} tag
+  * make NEXT_VERSION=${NEW_TAG} tag
   * edit docs/content/changelog.md
   * make TAG=${NEW_TAG} doc
   * git commit -a -v -m "Version ${NEW_TAG}"
@@ -89,9 +90,9 @@ Now
   * make TAG=${NEW_TAG} upload_github
   * NB this overwrites the current beta so we need to do this
   * git co master
-  * make LAST_TAG=${NEW_TAG} startdev
-  * # cherry pick the changes to the changelog
-  * git checkout ${BASE_TAG}-fixes docs/content/changelog.md
+  * make VERSION=${NEW_TAG} startdev
+  * # cherry pick the changes to the changelog and VERSION
+  * git checkout ${BASE_TAG}-fixes VERSION docs/content/changelog.md
   * git commit --amend
   * git push
   * Announce!

VERSION

@@ -0,0 +1 @@
+v1.51.0

backend/all/all.go

@@ -8,6 +8,7 @@ import (
 	_ "github.com/rclone/rclone/backend/b2"
 	_ "github.com/rclone/rclone/backend/box"
 	_ "github.com/rclone/rclone/backend/cache"
+	_ "github.com/rclone/rclone/backend/chunker"
 	_ "github.com/rclone/rclone/backend/crypt"
 	_ "github.com/rclone/rclone/backend/drive"
 	_ "github.com/rclone/rclone/backend/dropbox"
@@ -22,6 +23,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	_ "github.com/rclone/rclone/backend/mailru"
 	_ "github.com/rclone/rclone/backend/mega"
+	_ "github.com/rclone/rclone/backend/memory"
 	_ "github.com/rclone/rclone/backend/onedrive"
 	_ "github.com/rclone/rclone/backend/opendrive"
 	_ "github.com/rclone/rclone/backend/pcloud"
@@ -30,6 +32,8 @@ import (
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"
 	_ "github.com/rclone/rclone/backend/sftp"
+	_ "github.com/rclone/rclone/backend/sharefile"
+	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/webdav"

backend/amazonclouddrive/amazonclouddrive.go

@@ -32,6 +32,7 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"golang.org/x/oauth2"
@@ -135,15 +136,23 @@ which downloads the file through a temporary URL directly from the
 underlying S3 storage.`,
 			Default:  defaultTempLinkThreshold,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Checkpoint        string        `config:"checkpoint"`
-	UploadWaitPerGB   fs.Duration   `config:"upload_wait_per_gb"`
-	TempLinkThreshold fs.SizeSuffix `config:"templink_threshold"`
+	Checkpoint        string               `config:"checkpoint"`
+	UploadWaitPerGB   fs.Duration          `config:"upload_wait_per_gb"`
+	TempLinkThreshold fs.SizeSuffix        `config:"templink_threshold"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote acd server
@@ -384,7 +393,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 	var resp *http.Response
 	var subFolder *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
-		subFolder, resp, err = folder.GetFolder(leaf)
+		subFolder, resp, err = folder.GetFolder(f.opt.Enc.FromStandardName(leaf))
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -411,7 +420,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	var resp *http.Response
 	var info *acd.Folder
 	err = f.pacer.Call(func() (bool, error) {
-		info, resp, err = folder.CreateFolder(leaf)
+		info, resp, err = folder.CreateFolder(f.opt.Enc.FromStandardName(leaf))
 		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -479,6 +488,7 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 				if !hasValidParent {
 					continue
 				}
+				*node.Name = f.opt.Enc.ToStandardName(*node.Name)
 				// Store the nodes up in case we have to retry the listing
 				out = append(out, node)
 			}
@@ -668,7 +678,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	err = f.pacer.CallNoRetry(func() (bool, error) {
 		start := time.Now()
 		f.tokenRenewer.Start()
-		info, resp, err = folder.Put(in, leaf)
+		info, resp, err = folder.Put(in, f.opt.Enc.FromStandardName(leaf))
 		f.tokenRenewer.Stop()
 		var ok bool
 		ok, info, err = f.checkUpload(ctx, resp, in, src, info, err, time.Since(start))
@@ -1038,7 +1048,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	var resp *http.Response
 	var info *acd.File
 	err = o.fs.pacer.Call(func() (bool, error) {
-		info, resp, err = folder.GetFile(leaf)
+		info, resp, err = folder.GetFile(o.fs.opt.Enc.FromStandardName(leaf))
 		return o.fs.shouldRetry(resp, err)
 	})
 	if err != nil {
@@ -1158,7 +1168,7 @@ func (f *Fs) restoreNode(info *acd.Node) (newInfo *acd.Node, err error) {
 func (f *Fs) renameNode(info *acd.Node, newName string) (newInfo *acd.Node, err error) {
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
-		newInfo, resp, err = info.Rename(newName)
+		newInfo, resp, err = info.Rename(f.opt.Enc.FromStandardName(newName))
 		return f.shouldRetry(resp, err)
 	})
 	return newInfo, err
@@ -1354,10 +1364,11 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoin
 					if len(node.Parents) > 0 {
 						if path, ok := f.dirCache.GetInv(node.Parents[0]); ok {
 							// and append the drive file name to compute the full file name
+							name := f.opt.Enc.ToStandardName(*node.Name)
 							if len(path) > 0 {
-								path = path + "/" + *node.Name
+								path = path + "/" + name
 							} else {
-								path = *node.Name
+								path = name
 							}
 							// this will now clear the actual file too
 							pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject})

backend/azureblob/azureblob.go

@@ -16,7 +16,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -26,6 +25,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 )
 
@@ -124,21 +125,32 @@ If blobs are in "archive tier" at remote, trying to perform data transfer
 operations from remote will not be allowed. User should first restore by
 tiering blob to "Hot" or "Cool".`,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.EncodeInvalidUtf8 |
+				encoder.EncodeSlash |
+				encoder.EncodeCtl |
+				encoder.EncodeDel |
+				encoder.EncodeBackSlash |
+				encoder.EncodeRightPeriod),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account       string        `config:"account"`
-	Key           string        `config:"key"`
-	Endpoint      string        `config:"endpoint"`
-	SASURL        string        `config:"sas_url"`
-	UploadCutoff  fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
-	ListChunkSize uint          `config:"list_chunk"`
-	AccessTier    string        `config:"access_tier"`
-	UseEmulator   bool          `config:"use_emulator"`
+	Account       string               `config:"account"`
+	Key           string               `config:"key"`
+	Endpoint      string               `config:"endpoint"`
+	SASURL        string               `config:"sas_url"`
+	UploadCutoff  fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
+	ListChunkSize uint                 `config:"list_chunk"`
+	AccessTier    string               `config:"access_tier"`
+	UseEmulator   bool                 `config:"use_emulator"`
+	Enc           encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote azure server
@@ -186,7 +198,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootContainer == "" {
-		return fmt.Sprintf("Azure root")
+		return "Azure root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("Azure container %s", f.rootContainer)
@@ -208,7 +220,8 @@ func parsePath(path string) (root string) {
 // split returns container and containerPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (containerName, containerPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	containerName, containerPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(containerName), f.opt.Enc.FromStandardPath(containerPath)
 }
 
 // split returns container and containerPath from the object
@@ -308,6 +321,9 @@ func httpClientFactory(client *http.Client) pipeline.Factory {
 //
 // this code was copied from azblob.NewPipeline
 func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline {
+	// Don't log stuff to syslog/Windows Event log
+	pipeline.SetForceLogEnabled(false)
+
 	// Closest to API goes first; closest to the wire goes last
 	factories := []pipeline.Factory{
 		azblob.NewTelemetryPolicyFactory(o.Telemetry),
@@ -575,18 +591,18 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		}
 		// Advance marker to next
 		marker = response.NextMarker
-
 		for i := range response.Segment.BlobItems {
 			file := &response.Segment.BlobItems[i]
 			// Finish if file name no longer has prefix
 			// if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
 			// 	return nil
 			// }
-			if !strings.HasPrefix(file.Name, prefix) {
-				fs.Debugf(f, "Odd name received %q", file.Name)
+			remote := f.opt.Enc.ToStandardPath(file.Name)
+			if !strings.HasPrefix(remote, prefix) {
+				fs.Debugf(f, "Odd name received %q", remote)
 				continue
 			}
-			remote := file.Name[len(prefix):]
+			remote = remote[len(prefix):]
 			if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
 				continue // skip directory marker
 			}
@@ -602,6 +618,7 @@ func (f *Fs) list(ctx context.Context, container, directory, prefix string, addC
 		// Send the subdirectories
 		for _, remote := range response.Segment.BlobPrefixes {
 			remote := strings.TrimRight(remote.Name, "/")
+			remote = f.opt.Enc.ToStandardPath(remote)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Debugf(f, "Odd directory name received %q", remote)
 				continue
@@ -665,7 +682,7 @@ func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err err
 		return entries, nil
 	}
 	err = f.listContainersToFn(func(container *azblob.ContainerItem) error {
-		d := fs.NewDir(container.Name, container.Properties.LastModified)
+		d := fs.NewDir(f.opt.Enc.ToStandardName(container.Name), container.Properties.LastModified)
 		f.cache.MarkOK(container.Name)
 		entries = append(entries, d)
 		return nil
@@ -1103,22 +1120,6 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
-// parseTimeString converts a decimal string number of milliseconds
-// elapsed since January 1, 1970 UTC into a time.Time and stores it in
-// the modTime variable.
-func (o *Object) parseTimeString(timeString string) (err error) {
-	if timeString == "" {
-		return nil
-	}
-	unixMilliseconds, err := strconv.ParseInt(timeString, 10, 64)
-	if err != nil {
-		fs.Debugf(o, "Failed to parse mod time string %q: %v", timeString, err)
-		return err
-	}
-	o.modTime = time.Unix(unixMilliseconds/1e3, (unixMilliseconds%1e3)*1e6).UTC()
-	return nil
-}
-
 // ModTime returns the modification time of the object
 //
 // It attempts to read the objects mtime and if that isn't present the

backend/b2/b2.go

@@ -23,6 +23,7 @@ import (
 	"github.com/rclone/rclone/backend/b2/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -30,6 +31,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 )
@@ -143,23 +145,34 @@ The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// See: https://www.backblaze.com/b2/docs/files.html
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			// FIXME: allow /, but not leading, trailing or double
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account                       string        `config:"account"`
-	Key                           string        `config:"key"`
-	Endpoint                      string        `config:"endpoint"`
-	TestMode                      string        `config:"test_mode"`
-	Versions                      bool          `config:"versions"`
-	HardDelete                    bool          `config:"hard_delete"`
-	UploadCutoff                  fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize                     fs.SizeSuffix `config:"chunk_size"`
-	DisableCheckSum               bool          `config:"disable_checksum"`
-	DownloadURL                   string        `config:"download_url"`
-	DownloadAuthorizationDuration fs.Duration   `config:"download_auth_duration"`
+	Account                       string               `config:"account"`
+	Key                           string               `config:"key"`
+	Endpoint                      string               `config:"endpoint"`
+	TestMode                      string               `config:"test_mode"`
+	Versions                      bool                 `config:"versions"`
+	HardDelete                    bool                 `config:"hard_delete"`
+	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize                     fs.SizeSuffix        `config:"chunk_size"`
+	DisableCheckSum               bool                 `config:"disable_checksum"`
+	DownloadURL                   string               `config:"download_url"`
+	DownloadAuthorizationDuration fs.Duration          `config:"download_auth_duration"`
+	Enc                           encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote b2 server
@@ -399,7 +412,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 	// If this is a key limited to a single bucket, it must exist already
 	if f.rootBucket != "" && f.info.Allowed.BucketID != "" {
-		allowedBucket := f.info.Allowed.BucketName
+		allowedBucket := f.opt.Enc.ToStandardName(f.info.Allowed.BucketName)
 		if allowedBucket == "" {
 			return nil, errors.New("bucket that application key is restricted to no longer exists")
 		}
@@ -620,11 +633,11 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	var request = api.ListFileNamesRequest{
 		BucketID:     bucketID,
 		MaxFileCount: chunkSize,
-		Prefix:       directory,
+		Prefix:       f.opt.Enc.FromStandardPath(directory),
 		Delimiter:    delimiter,
 	}
 	if directory != "" {
-		request.StartFileName = directory
+		request.StartFileName = f.opt.Enc.FromStandardPath(directory)
 	}
 	opts := rest.Opts{
 		Method: "POST",
@@ -644,6 +657,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		}
 		for i := range response.Files {
 			file := &response.Files[i]
+			file.Name = f.opt.Enc.ToStandardPath(file.Name)
 			// Finish if file name no longer has prefix
 			if prefix != "" && !strings.HasPrefix(file.Name, prefix) {
 				return nil
@@ -844,6 +858,7 @@ func (f *Fs) listBucketsToFn(ctx context.Context, fn listBucketFn) error {
 	f._bucketType = make(map[string]string, 1)
 	for i := range response.Buckets {
 		bucket := &response.Buckets[i]
+		bucket.Name = f.opt.Enc.ToStandardName(bucket.Name)
 		f.cache.MarkOK(bucket.Name)
 		f._bucketID[bucket.Name] = bucket.ID
 		f._bucketType[bucket.Name] = bucket.Type
@@ -965,7 +980,7 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 		}
 		var request = api.CreateBucketRequest{
 			AccountID: f.info.AccountID,
-			Name:      bucket,
+			Name:      f.opt.Enc.FromStandardName(bucket),
 			Type:      "allPrivate",
 		}
 		var response api.Bucket
@@ -1049,7 +1064,7 @@ func (f *Fs) hide(ctx context.Context, bucket, bucketPath string) error {
 	}
 	var request = api.HideFileRequest{
 		BucketID: bucketID,
-		Name:     bucketPath,
+		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
 	var response api.File
 	err = f.pacer.Call(func() (bool, error) {
@@ -1077,7 +1092,7 @@ func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
 	}
 	var request = api.DeleteFileRequest{
 		ID:   ID,
-		Name: Name,
+		Name: f.opt.Enc.FromStandardPath(Name),
 	}
 	var response api.File
 	err := f.pacer.Call(func() (bool, error) {
@@ -1215,7 +1230,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	var request = api.CopyFileRequest{
 		SourceID:          srcObj.id,
-		Name:              dstPath,
+		Name:              f.opt.Enc.FromStandardPath(dstPath),
 		MetadataDirective: "COPY",
 		DestBucketID:      destBucketID,
 	}
@@ -1263,7 +1278,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
 	}
 	var request = api.GetDownloadAuthorizationRequest{
 		BucketID:               bucketID,
-		FileNamePrefix:         path.Join(f.root, remote),
+		FileNamePrefix:         f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
 		ValidDurationInSeconds: validDurationInSeconds,
 	}
 	var response api.GetDownloadAuthorizationResponse
@@ -1370,6 +1385,12 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if o.sha1 == "" || o.sha1 == "none" {
 		o.sha1 = Info[sha1Key]
 	}
+	// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
+	// Some tools (eg Cyberduck) use this
+	const unverified = "unverified:"
+	if strings.HasPrefix(o.sha1, unverified) {
+		o.sha1 = o.sha1[len(unverified):]
+	}
 	o.size = Size
 	// Use the UploadTimestamp if can't get file info
 	o.modTime = time.Time(UploadTimestamp)
@@ -1498,7 +1519,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	}
 	var request = api.CopyFileRequest{
 		SourceID:          o.id,
-		Name:              bucketPath, // copy to same name
+		Name:              o.fs.opt.Enc.FromStandardPath(bucketPath), // copy to same name
 		MetadataDirective: "REPLACE",
 		ContentType:       info.ContentType,
 		Info:              info.Info,
@@ -1600,7 +1621,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
 	} else {
 		bucket, bucketPath := o.split()
-		opts.Path += "/file/" + urlEncode(bucket) + "/" + urlEncode(bucketPath)
+		opts.Path += "/file/" + urlEncode(o.fs.opt.Enc.FromStandardName(bucket)) + "/" + urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath))
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
@@ -1797,7 +1818,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Body:    in,
 		ExtraHeaders: map[string]string{
 			"Authorization":  upload.AuthorizationToken,
-			"X-Bz-File-Name": urlEncode(bucketPath),
+			"X-Bz-File-Name": urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath)),
 			"Content-Type":   fs.MimeType(ctx, src),
 			sha1Header:       calculatedSha1,
 			timeHeader:       timeString(modTime),

backend/b2/upload.go

@@ -111,7 +111,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	}
 	var request = api.StartLargeFileRequest{
 		BucketID:    bucketID,
-		Name:        bucketPath,
+		Name:        f.opt.Enc.FromStandardPath(bucketPath),
 		ContentType: fs.MimeType(ctx, src),
 		Info: map[string]string{
 			timeKey: timeString(modTime),

backend/box/box.go

@@ -25,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/jwtutil"
 
 	"github.com/youmark/pkcs8"
@@ -143,6 +144,21 @@ func init() {
 			Help:     "Max number of times to try committing a multipart file.",
 			Default:  100,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// From https://developer.box.com/docs/error-codes#section-400-bad-request :
+			// > Box only supports file or folder names that are 255 characters or less.
+			// > File names containing non-printable ascii, "/" or "\", names with leading
+			// > or trailing spaces, and the special names “.” and “..” are also unsupported.
+			//
+			// Testing revealed names with leading spaces work fine.
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
@@ -217,8 +233,9 @@ func getDecryptedPrivateKey(boxConfig *api.ConfigJSON) (key *rsa.PrivateKey, err
 
 // Options defines the configuration for this backend
 type Options struct {
-	UploadCutoff  fs.SizeSuffix `config:"upload_cutoff"`
-	CommitRetries int           `config:"commit_retries"`
+	UploadCutoff  fs.SizeSuffix        `config:"upload_cutoff"`
+	CommitRetries int                  `config:"commit_retries"`
+	Enc           encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote box
@@ -298,18 +315,6 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// substitute reserved characters for box
-func replaceReservedChars(x string) string {
-	// Backslash for FULLWIDTH REVERSE SOLIDUS
-	return strings.Replace(x, "\\", "＼", -1)
-}
-
-// restore reserved characters for box
-func restoreReservedChars(x string) string {
-	// FULLWIDTH REVERSE SOLIDUS for Backslash
-	return strings.Replace(x, "＼", "\\", -1)
-}
-
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
@@ -497,7 +502,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Parameters: fieldsValue(),
 	}
 	mkdir := api.CreateFolder{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
 			ID: pathID,
 		},
@@ -563,7 +568,7 @@ OUTER:
 			if item.ItemStatus != api.ItemStatusActive {
 				continue
 			}
-			item.Name = restoreReservedChars(item.Name)
+			item.Name = f.opt.Enc.ToStandardName(item.Name)
 			if fn(item) {
 				found = true
 				break OUTER
@@ -799,9 +804,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Path:       "/files/" + srcObj.id + "/copy",
 		Parameters: fieldsValue(),
 	}
-	replacedLeaf := replaceReservedChars(leaf)
 	copyFile := api.CopyFile{
-		Name: replacedLeaf,
+		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
 			ID: directoryID,
 		},
@@ -840,7 +844,7 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
 		Parameters: fieldsValue(),
 	}
 	move := api.UpdateFileMove{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		Parent: api.Parent{
 			ID: directoryID,
 		},
@@ -1041,11 +1045,6 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// srvPath returns a path for use in server
-func (o *Object) srvPath() string {
-	return replaceReservedChars(o.fs.rootSlash() + o.remote)
-}
-
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	if t != hash.SHA1 {
@@ -1170,7 +1169,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // This is recommended for less than 50 MB of content
 func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time) (err error) {
 	upload := api.UploadFile{
-		Name:              replaceReservedChars(leaf),
+		Name:              o.fs.opt.Enc.FromStandardName(leaf),
 		ContentModifiedAt: api.Time(modTime),
 		ContentCreatedAt:  api.Time(modTime),
 		Parent: api.Parent{

backend/box/upload.go

@@ -38,7 +38,7 @@ func (o *Object) createUploadSession(ctx context.Context, leaf, directoryID stri
 	} else {
 		opts.Path = "/files/upload_sessions"
 		request.FolderID = directoryID
-		request.FileName = replaceReservedChars(leaf)
+		request.FileName = o.fs.opt.Enc.FromStandardName(leaf)
 	}
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {

backend/cache/cache_internal_test.go

@@ -1,4 +1,5 @@
 // +build !plan9
+// +build !race
 
 package cache_test
 
@@ -17,7 +18,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
-	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -1139,23 +1139,6 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	return f
 }
 
-func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string {
-	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
-	// create some rand test data
-	testData := randStringBytes(int(size))
-
-	r.writeRemoteBytes(t, f, remote, testData)
-	return remote
-}
-
-func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object {
-	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
-	// create some rand test data
-	testData := randStringBytes(int(size))
-
-	return r.writeObjectBytes(t, f, remote, testData)
-}
-
 func (r *run) writeRemoteString(t *testing.T, f fs.Fs, remote, content string) {
 	r.writeRemoteBytes(t, f, remote, []byte(content))
 }
@@ -1344,26 +1327,6 @@ func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error)
 	return l, err
 }
 
-func (r *run) listPath(t *testing.T, f fs.Fs, remote string) []string {
-	var err error
-	var l []string
-	if r.useMount {
-		var list []os.FileInfo
-		list, err = ioutil.ReadDir(path.Join(r.mntDir, remote))
-		for _, ll := range list {
-			l = append(l, ll.Name())
-		}
-	} else {
-		var list fs.DirEntries
-		list, err = f.List(context.Background(), remote)
-		for _, ll := range list {
-			l = append(l, ll.Remote())
-		}
-	}
-	require.NoError(t, err)
-	return l
-}
-
 func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
 	in, err := os.Open(src)
 	if err != nil {

backend/cache/cache_mount_other_test.go

@@ -0,0 +1,21 @@
+// +build !linux !go1.13
+// +build !darwin !go1.13
+// +build !freebsd !go1.13
+// +build !windows
+// +build !race
+
+package cache_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	panic("mountFs not defined for this platform")
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	panic("unmountFs not defined for this platform")
+}

backend/cache/cache_mount_unix_test.go

@@ -1,4 +1,5 @@
-// +build !plan9,!windows
+// +build linux,go1.13 darwin,go1.13 freebsd,go1.13
+// +build !race
 
 package cache_test
 

backend/cache/cache_mount_windows_test.go

@@ -1,4 +1,5 @@
 // +build windows
+// +build !race
 
 package cache_test
 

backend/cache/cache_test.go

@@ -1,6 +1,7 @@
 // Test Cache filesystem interface
 
 // +build !plan9
+// +build !race
 
 package cache_test
 
@@ -19,5 +20,6 @@ func TestIntegration(t *testing.T) {
 		NilObject:                    (*cache.Object)(nil),
 		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
+		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})
 }

backend/cache/cache_upload_test.go

@@ -1,4 +1,5 @@
 // +build !plan9
+// +build !race
 
 package cache_test
 

backend/cache/directory.go

@@ -101,15 +101,6 @@ func (d *Directory) abs() string {
 	return cleanPath(path.Join(d.Dir, d.Name))
 }
 
-// parentRemote returns the absolute path parent remote
-func (d *Directory) parentRemote() string {
-	absPath := d.abs()
-	if absPath == "" {
-		return ""
-	}
-	return cleanPath(path.Dir(absPath))
-}
-
 // ModTime returns the cached ModTime
 func (d *Directory) ModTime(ctx context.Context) time.Time {
 	return time.Unix(0, d.CacheModTime)

backend/cache/object.go

@@ -24,15 +24,16 @@ const (
 type Object struct {
 	fs.Object `json:"-"`
 
-	ParentFs      fs.Fs                `json:"-"`        // parent fs
-	CacheFs       *Fs                  `json:"-"`        // cache fs
-	Name          string               `json:"name"`     // name of the directory
-	Dir           string               `json:"dir"`      // abs path of the object
-	CacheModTime  int64                `json:"modTime"`  // modification or creation time - IsZero for unknown
-	CacheSize     int64                `json:"size"`     // size of directory and contents or -1 if unknown
-	CacheStorable bool                 `json:"storable"` // says whether this object can be stored
-	CacheType     string               `json:"cacheType"`
-	CacheTs       time.Time            `json:"cacheTs"`
+	ParentFs      fs.Fs     `json:"-"`        // parent fs
+	CacheFs       *Fs       `json:"-"`        // cache fs
+	Name          string    `json:"name"`     // name of the directory
+	Dir           string    `json:"dir"`      // abs path of the object
+	CacheModTime  int64     `json:"modTime"`  // modification or creation time - IsZero for unknown
+	CacheSize     int64     `json:"size"`     // size of directory and contents or -1 if unknown
+	CacheStorable bool      `json:"storable"` // says whether this object can be stored
+	CacheType     string    `json:"cacheType"`
+	CacheTs       time.Time `json:"cacheTs"`
+	cacheHashesMu sync.Mutex
 	CacheHashes   map[hash.Type]string // all supported hashes cached
 
 	refreshMutex sync.Mutex
@@ -103,7 +104,9 @@ func (o *Object) updateData(ctx context.Context, source fs.Object) {
 	o.CacheSize = source.Size()
 	o.CacheStorable = source.Storable()
 	o.CacheTs = time.Now()
+	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
+	o.cacheHashesMu.Unlock()
 }
 
 // Fs returns its FS info
@@ -268,7 +271,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	o.CacheModTime = src.ModTime(ctx).UnixNano()
 	o.CacheSize = src.Size()
+	o.cacheHashesMu.Lock()
 	o.CacheHashes = make(map[hash.Type]string)
+	o.cacheHashesMu.Unlock()
 	o.CacheTs = time.Now()
 	o.persist()
 
@@ -309,11 +314,12 @@ func (o *Object) Remove(ctx context.Context) error {
 // since it might or might not be called, this is lazy loaded
 func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	_ = o.refresh(ctx)
+	o.cacheHashesMu.Lock()
 	if o.CacheHashes == nil {
 		o.CacheHashes = make(map[hash.Type]string)
 	}
-
 	cachedHash, found := o.CacheHashes[ht]
+	o.cacheHashesMu.Unlock()
 	if found {
 		return cachedHash, nil
 	}
@@ -324,7 +330,9 @@ func (o *Object) Hash(ctx context.Context, ht hash.Type) (string, error) {
 	if err != nil {
 		return "", err
 	}
+	o.cacheHashesMu.Lock()
 	o.CacheHashes[ht] = liveHash
+	o.cacheHashesMu.Unlock()
 
 	o.persist()
 	fs.Debugf(o, "object hash cached: %v", liveHash)

backend/cache/storage_persistent.go

@@ -16,7 +16,7 @@ import (
 	"sync"
 	"time"
 
-	bolt "github.com/coreos/bbolt"
+	bolt "github.com/etcd-io/bbolt"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
@@ -767,31 +767,6 @@ func (b *Persistent) iterateBuckets(buk *bolt.Bucket, bucketFn func(name string)
 	return err
 }
 
-func (b *Persistent) dumpRoot() string {
-	var itBuckets func(buk *bolt.Bucket) map[string]interface{}
-
-	itBuckets = func(buk *bolt.Bucket) map[string]interface{} {
-		m := make(map[string]interface{})
-		c := buk.Cursor()
-		for k, v := c.First(); k != nil; k, v = c.Next() {
-			if v == nil {
-				buk2 := buk.Bucket(k)
-				m[string(k)] = itBuckets(buk2)
-			} else {
-				m[string(k)] = "-"
-			}
-		}
-		return m
-	}
-	var mm map[string]interface{}
-	_ = b.db.View(func(tx *bolt.Tx) error {
-		mm = itBuckets(tx.Bucket([]byte(RootBucket)))
-		return nil
-	})
-	raw, _ := json.MarshalIndent(mm, "", "  ")
-	return string(raw)
-}
-
 // addPendingUpload adds a new file to the pending queue of uploads
 func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 	return b.db.Update(func(tx *bolt.Tx) error {

backend/chunker/chunker_internal_test.go

@@ -0,0 +1,691 @@
+package chunker
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Command line flags
+var (
+	UploadKilobytes = flag.Int("upload-kilobytes", 0, "Upload size in Kilobytes, set this to test large uploads")
+)
+
+// test that chunking does not break large uploads
+func testPutLarge(t *testing.T, f *Fs, kilobytes int) {
+	t.Run(fmt.Sprintf("PutLarge%dk", kilobytes), func(t *testing.T) {
+		fstests.TestPutLarge(context.Background(), t, f, &fstest.Item{
+			ModTime: fstest.Time("2001-02-03T04:05:06.499999999Z"),
+			Path:    fmt.Sprintf("chunker-upload-%dk", kilobytes),
+			Size:    int64(kilobytes) * int64(fs.KibiByte),
+		})
+	})
+}
+
+// test chunk name parser
+func testChunkNameFormat(t *testing.T, f *Fs) {
+	saveOpt := f.opt
+	defer func() {
+		// restore original settings (f is pointer, f.opt is struct)
+		f.opt = saveOpt
+		_ = f.setChunkNameFormat(f.opt.NameFormat)
+	}()
+
+	assertFormat := func(pattern, wantDataFormat, wantCtrlFormat, wantNameRegexp string) {
+		err := f.setChunkNameFormat(pattern)
+		assert.NoError(t, err)
+		assert.Equal(t, wantDataFormat, f.dataNameFmt)
+		assert.Equal(t, wantCtrlFormat, f.ctrlNameFmt)
+		assert.Equal(t, wantNameRegexp, f.nameRegexp.String())
+	}
+
+	assertFormatValid := func(pattern string) {
+		err := f.setChunkNameFormat(pattern)
+		assert.NoError(t, err)
+	}
+
+	assertFormatInvalid := func(pattern string) {
+		err := f.setChunkNameFormat(pattern)
+		assert.Error(t, err)
+	}
+
+	assertMakeName := func(wantChunkName, mainName string, chunkNo int, ctrlType, xactID string) {
+		gotChunkName := ""
+		assert.NotPanics(t, func() {
+			gotChunkName = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
+		}, "makeChunkName(%q,%d,%q,%q) must not panic", mainName, chunkNo, ctrlType, xactID)
+		if gotChunkName != "" {
+			assert.Equal(t, wantChunkName, gotChunkName)
+		}
+	}
+
+	assertMakeNamePanics := func(mainName string, chunkNo int, ctrlType, xactID string) {
+		assert.Panics(t, func() {
+			_ = f.makeChunkName(mainName, chunkNo, ctrlType, xactID)
+		}, "makeChunkName(%q,%d,%q,%q) should panic", mainName, chunkNo, ctrlType, xactID)
+	}
+
+	assertParseName := func(fileName, wantMainName string, wantChunkNo int, wantCtrlType, wantXactID string) {
+		gotMainName, gotChunkNo, gotCtrlType, gotXactID := f.parseChunkName(fileName)
+		assert.Equal(t, wantMainName, gotMainName)
+		assert.Equal(t, wantChunkNo, gotChunkNo)
+		assert.Equal(t, wantCtrlType, gotCtrlType)
+		assert.Equal(t, wantXactID, gotXactID)
+	}
+
+	const newFormatSupported = false // support for patterns not starting with base name (*)
+
+	// valid formats
+	assertFormat(`*.rclone_chunk.###`, `%s.rclone_chunk.%03d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*.rclone_chunk.#`, `%s.rclone_chunk.%d`, `%s.rclone_chunk._%s`, `^(.+?)\.rclone_chunk\.(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*_chunk_#####`, `%s_chunk_%05d`, `%s_chunk__%s`, `^(.+?)_chunk_(?:([0-9]{5,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*-chunk-#`, `%s-chunk-%d`, `%s-chunk-_%s`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	assertFormat(`*-chunk-#-%^$()[]{}.+-!?:\`, `%s-chunk-%d-%%^$()[]{}.+-!?:\`, `%s-chunk-_%s-%%^$()[]{}.+-!?:\`, `^(.+?)-chunk-(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))-%\^\$\(\)\[\]\{\}\.\+-!\?:\\(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	if newFormatSupported {
+		assertFormat(`_*-chunk-##,`, `_%s-chunk-%02d,`, `_%s-chunk-_%s,`, `^_(.+?)-chunk-(?:([0-9]{2,})|_([a-z][a-z0-9]{2,6})),(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	}
+
+	// invalid formats
+	assertFormatInvalid(`chunk-#`)
+	assertFormatInvalid(`*-chunk`)
+	assertFormatInvalid(`*-*-chunk-#`)
+	assertFormatInvalid(`*-chunk-#-#`)
+	assertFormatInvalid(`#-chunk-*`)
+	assertFormatInvalid(`*/#`)
+
+	assertFormatValid(`*#`)
+	assertFormatInvalid(`**#`)
+	assertFormatInvalid(`#*`)
+	assertFormatInvalid(``)
+	assertFormatInvalid(`-`)
+
+	// quick tests
+	if newFormatSupported {
+		assertFormat(`part_*_#`, `part_%s_%d`, `part_%s__%s`, `^part_(.+?)_(?:([0-9]+)|_([a-z][a-z0-9]{2,6}))(?:_([0-9][0-9a-z]{3,8})\.\.tmp_([0-9]{10,13}))?$`)
+		f.opt.StartFrom = 1
+
+		assertMakeName(`part_fish_1`, "fish", 0, "", "")
+		assertParseName(`part_fish_43`, "fish", 42, "", "")
+		assertMakeName(`part_fish__locks`, "fish", -2, "locks", "")
+		assertParseName(`part_fish__locks`, "fish", -1, "locks", "")
+		assertMakeName(`part_fish__x2y`, "fish", -2, "x2y", "")
+		assertParseName(`part_fish__x2y`, "fish", -1, "x2y", "")
+		assertMakeName(`part_fish_3_0004`, "fish", 2, "", "4")
+		assertParseName(`part_fish_4_0005`, "fish", 3, "", "0005")
+		assertMakeName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -3, "blkinfo", "jj5fvo3wr")
+		assertParseName(`part_fish__blkinfo_zz9fvo3wr`, "fish", -1, "blkinfo", "zz9fvo3wr")
+
+		// old-style temporary suffix (parse only)
+		assertParseName(`part_fish_4..tmp_0000000011`, "fish", 3, "", "000b")
+		assertParseName(`part_fish__blkinfo_jj5fvo3wr`, "fish", -1, "blkinfo", "jj5fvo3wr")
+	}
+
+	// prepare format for long tests
+	assertFormat(`*.chunk.###`, `%s.chunk.%03d`, `%s.chunk._%s`, `^(.+?)\.chunk\.(?:([0-9]{3,})|_([a-z][a-z0-9]{2,6}))(?:_([0-9a-z]{4,9})|\.\.tmp_([0-9]{10,13}))?$`)
+	f.opt.StartFrom = 2
+
+	// valid data chunks
+	assertMakeName(`fish.chunk.003`, "fish", 1, "", "")
+	assertParseName(`fish.chunk.003`, "fish", 1, "", "")
+	assertMakeName(`fish.chunk.021`, "fish", 19, "", "")
+	assertParseName(`fish.chunk.021`, "fish", 19, "", "")
+
+	// valid temporary data chunks
+	assertMakeName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
+	assertParseName(`fish.chunk.011_4321`, "fish", 9, "", "4321")
+	assertMakeName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
+	assertParseName(`fish.chunk.011_00bc`, "fish", 9, "", "00bc")
+	assertMakeName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
+	assertParseName(`fish.chunk.1916_5jjfvo3wr`, "fish", 1914, "", "5jjfvo3wr")
+	assertMakeName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
+	assertParseName(`fish.chunk.1917_zz9fvo3wr`, "fish", 1915, "", "zz9fvo3wr")
+
+	// valid temporary data chunks (old temporary suffix, only parse)
+	assertParseName(`fish.chunk.004..tmp_0000000047`, "fish", 2, "", "001b")
+	assertParseName(`fish.chunk.323..tmp_9994567890123`, "fish", 321, "", "3jjfvo3wr")
+
+	// parsing invalid data chunk names
+	assertParseName(`fish.chunk.3`, "", -1, "", "")
+	assertParseName(`fish.chunk.001`, "", -1, "", "")
+	assertParseName(`fish.chunk.21`, "", -1, "", "")
+	assertParseName(`fish.chunk.-21`, "", -1, "", "")
+
+	assertParseName(`fish.chunk.004abcd`, "", -1, "", "")        // missing underscore delimiter
+	assertParseName(`fish.chunk.004__1234`, "", -1, "", "")      // extra underscore delimiter
+	assertParseName(`fish.chunk.004_123`, "", -1, "", "")        // too short temporary suffix
+	assertParseName(`fish.chunk.004_1234567890`, "", -1, "", "") // too long temporary suffix
+	assertParseName(`fish.chunk.004_-1234`, "", -1, "", "")      // temporary suffix must be positive
+	assertParseName(`fish.chunk.004_123E`, "", -1, "", "")       // uppercase not allowed
+	assertParseName(`fish.chunk.004_12.3`, "", -1, "", "")       // punctuation not allowed
+
+	// parsing invalid data chunk names (old temporary suffix)
+	assertParseName(`fish.chunk.004.tmp_0000000021`, "", -1, "", "")
+	assertParseName(`fish.chunk.003..tmp_123456789`, "", -1, "", "")
+	assertParseName(`fish.chunk.003..tmp_012345678901234567890123456789`, "", -1, "", "")
+	assertParseName(`fish.chunk.323..tmp_12345678901234`, "", -1, "", "")
+	assertParseName(`fish.chunk.003..tmp_-1`, "", -1, "", "")
+
+	// valid control chunks
+	assertMakeName(`fish.chunk._info`, "fish", -1, "info", "")
+	assertMakeName(`fish.chunk._locks`, "fish", -2, "locks", "")
+	assertMakeName(`fish.chunk._blkinfo`, "fish", -3, "blkinfo", "")
+	assertMakeName(`fish.chunk._x2y`, "fish", -4, "x2y", "")
+
+	assertParseName(`fish.chunk._info`, "fish", -1, "info", "")
+	assertParseName(`fish.chunk._locks`, "fish", -1, "locks", "")
+	assertParseName(`fish.chunk._blkinfo`, "fish", -1, "blkinfo", "")
+	assertParseName(`fish.chunk._x2y`, "fish", -1, "x2y", "")
+
+	// valid temporary control chunks
+	assertMakeName(`fish.chunk._info_0001`, "fish", -1, "info", "1")
+	assertMakeName(`fish.chunk._locks_4321`, "fish", -2, "locks", "4321")
+	assertMakeName(`fish.chunk._uploads_abcd`, "fish", -3, "uploads", "abcd")
+	assertMakeName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -4, "blkinfo", "xyzabcdef")
+	assertMakeName(`fish.chunk._x2y_1aaa`, "fish", -5, "x2y", "1aaa")
+
+	assertParseName(`fish.chunk._info_0001`, "fish", -1, "info", "0001")
+	assertParseName(`fish.chunk._locks_4321`, "fish", -1, "locks", "4321")
+	assertParseName(`fish.chunk._uploads_9abc`, "fish", -1, "uploads", "9abc")
+	assertParseName(`fish.chunk._blkinfo_xyzabcdef`, "fish", -1, "blkinfo", "xyzabcdef")
+	assertParseName(`fish.chunk._x2y_1aaa`, "fish", -1, "x2y", "1aaa")
+
+	// valid temporary control chunks (old temporary suffix, parse only)
+	assertParseName(`fish.chunk._info..tmp_0000000047`, "fish", -1, "info", "001b")
+	assertParseName(`fish.chunk._locks..tmp_0000054321`, "fish", -1, "locks", "15wx")
+	assertParseName(`fish.chunk._uploads..tmp_0000000000`, "fish", -1, "uploads", "0000")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123`, "fish", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._x2y..tmp_0000000000`, "fish", -1, "x2y", "0000")
+
+	// parsing invalid control chunk names
+	assertParseName(`fish.chunk.metadata`, "", -1, "", "") // must be prepended by underscore
+	assertParseName(`fish.chunk.info`, "", -1, "", "")
+	assertParseName(`fish.chunk.locks`, "", -1, "", "")
+	assertParseName(`fish.chunk.uploads`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._os`, "", -1, "", "")        // too short
+	assertParseName(`fish.chunk._metadata`, "", -1, "", "")  // too long
+	assertParseName(`fish.chunk._blockinfo`, "", -1, "", "") // way too long
+	assertParseName(`fish.chunk._4me`, "", -1, "", "")       // cannot start with digit
+	assertParseName(`fish.chunk._567`, "", -1, "", "")       // cannot be all digits
+	assertParseName(`fish.chunk._me_ta`, "", -1, "", "")     // punctuation not allowed
+	assertParseName(`fish.chunk._in-fo`, "", -1, "", "")
+	assertParseName(`fish.chunk._.bin`, "", -1, "", "")
+	assertParseName(`fish.chunk._.2xy`, "", -1, "", "")
+
+	// parsing invalid temporary control chunks
+	assertParseName(`fish.chunk._blkinfo1234`, "", -1, "", "")     // missing underscore delimiter
+	assertParseName(`fish.chunk._info__1234`, "", -1, "", "")      // extra underscore delimiter
+	assertParseName(`fish.chunk._info_123`, "", -1, "", "")        // too short temporary suffix
+	assertParseName(`fish.chunk._info_1234567890`, "", -1, "", "") // too long temporary suffix
+	assertParseName(`fish.chunk._info_-1234`, "", -1, "", "")      // temporary suffix must be positive
+	assertParseName(`fish.chunk._info_123E`, "", -1, "", "")       // uppercase not allowed
+	assertParseName(`fish.chunk._info_12.3`, "", -1, "", "")       // punctuation not allowed
+
+	assertParseName(`fish.chunk._locks..tmp_123456789`, "", -1, "", "")
+	assertParseName(`fish.chunk._meta..tmp_-1`, "", -1, "", "")
+	assertParseName(`fish.chunk._blockinfo..tmp_012345678901234567890123456789`, "", -1, "", "")
+
+	// short control chunk names: 3 letters ok, 1-2 letters not allowed
+	assertMakeName(`fish.chunk._ext`, "fish", -1, "ext", "")
+	assertParseName(`fish.chunk._int`, "fish", -1, "int", "")
+
+	assertMakeNamePanics("fish", -1, "in", "")
+	assertMakeNamePanics("fish", -1, "up", "4")
+	assertMakeNamePanics("fish", -1, "x", "")
+	assertMakeNamePanics("fish", -1, "c", "1z")
+
+	assertMakeName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0")
+	assertMakeName(`fish.chunk._ext_0026`, "fish", -1, "ext", "26")
+	assertMakeName(`fish.chunk._int_0abc`, "fish", -1, "int", "abc")
+	assertMakeName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
+	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+	assertMakeName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+
+	assertParseName(`fish.chunk._ext_0000`, "fish", -1, "ext", "0000")
+	assertParseName(`fish.chunk._ext_0026`, "fish", -1, "ext", "0026")
+	assertParseName(`fish.chunk._int_0abc`, "fish", -1, "int", "0abc")
+	assertParseName(`fish.chunk._int_9xyz`, "fish", -1, "int", "9xyz")
+	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+	assertParseName(`fish.chunk._out_jj5fvo3wr`, "fish", -1, "out", "jj5fvo3wr")
+
+	// base file name can sometimes look like a valid chunk name
+	assertParseName(`fish.chunk.003.chunk.004`, "fish.chunk.003", 2, "", "")
+	assertParseName(`fish.chunk.003.chunk._info`, "fish.chunk.003", -1, "info", "")
+	assertParseName(`fish.chunk.003.chunk._Meta`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._info.chunk.004`, "fish.chunk._info", 2, "", "")
+	assertParseName(`fish.chunk._info.chunk._info`, "fish.chunk._info", -1, "info", "")
+	assertParseName(`fish.chunk._info.chunk._info.chunk._Meta`, "", -1, "", "")
+
+	// base file name looking like a valid chunk name (old temporary suffix)
+	assertParseName(`fish.chunk.003.chunk.005..tmp_0000000022`, "fish.chunk.003", 3, "", "000m")
+	assertParseName(`fish.chunk.003.chunk._x..tmp_0000054321`, "", -1, "", "")
+	assertParseName(`fish.chunk._info.chunk.005..tmp_0000000023`, "fish.chunk._info", 3, "", "000n")
+	assertParseName(`fish.chunk._info.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	assertParseName(`fish.chunk.003.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.003", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._info.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._info", -1, "blkinfo", "3jjfvo3wr")
+
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.004`, "fish.chunk.004..tmp_0000000021", 2, "", "")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk.005..tmp_0000000025`, "fish.chunk.004..tmp_0000000021", 3, "", "000p")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._info`, "fish.chunk.004..tmp_0000000021", -1, "info", "")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._blkinfo..tmp_9994567890123`, "fish.chunk.004..tmp_0000000021", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._Meta`, "", -1, "", "")
+	assertParseName(`fish.chunk.004..tmp_0000000021.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.004`, "fish.chunk._blkinfo..tmp_9994567890123", 2, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk.005..tmp_0000000026`, "fish.chunk._blkinfo..tmp_9994567890123", 3, "", "000q")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "info", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_9994567890123", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._Meta`, "", -1, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_9994567890123.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.004`, "fish.chunk._blkinfo..tmp_1234567890123456789", 2, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk.005..tmp_0000000022`, "fish.chunk._blkinfo..tmp_1234567890123456789", 3, "", "000m")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "info", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._blkinfo..tmp_9994567890123`, "fish.chunk._blkinfo..tmp_1234567890123456789", -1, "blkinfo", "3jjfvo3wr")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._Meta`, "", -1, "", "")
+	assertParseName(`fish.chunk._blkinfo..tmp_1234567890123456789.chunk._info.chunk._x..tmp_0000054321`, "", -1, "", "")
+
+	// attempts to make invalid chunk names
+	assertMakeNamePanics("fish", -1, "", "")          // neither data nor control
+	assertMakeNamePanics("fish", 0, "info", "")       // both data and control
+	assertMakeNamePanics("fish", -1, "metadata", "")  // control type too long
+	assertMakeNamePanics("fish", -1, "blockinfo", "") // control type way too long
+	assertMakeNamePanics("fish", -1, "2xy", "")       // first digit not allowed
+	assertMakeNamePanics("fish", -1, "123", "")       // all digits not allowed
+	assertMakeNamePanics("fish", -1, "Meta", "")      // only lower case letters allowed
+	assertMakeNamePanics("fish", -1, "in-fo", "")     // punctuation not allowed
+	assertMakeNamePanics("fish", -1, "_info", "")
+	assertMakeNamePanics("fish", -1, "info_", "")
+	assertMakeNamePanics("fish", -2, ".bind", "")
+	assertMakeNamePanics("fish", -2, "bind.", "")
+
+	assertMakeNamePanics("fish", -1, "", "1")          // neither data nor control
+	assertMakeNamePanics("fish", 0, "info", "23")      // both data and control
+	assertMakeNamePanics("fish", -1, "metadata", "45") // control type too long
+	assertMakeNamePanics("fish", -1, "blockinfo", "7") // control type way too long
+	assertMakeNamePanics("fish", -1, "2xy", "abc")     // first digit not allowed
+	assertMakeNamePanics("fish", -1, "123", "def")     // all digits not allowed
+	assertMakeNamePanics("fish", -1, "Meta", "mnk")    // only lower case letters allowed
+	assertMakeNamePanics("fish", -1, "in-fo", "xyz")   // punctuation not allowed
+	assertMakeNamePanics("fish", -1, "_info", "5678")
+	assertMakeNamePanics("fish", -1, "info_", "999")
+	assertMakeNamePanics("fish", -2, ".bind", "0")
+	assertMakeNamePanics("fish", -2, "bind.", "0")
+
+	assertMakeNamePanics("fish", 0, "", "1234567890") // temporary suffix too long
+	assertMakeNamePanics("fish", 0, "", "123F4")      // uppercase not allowed
+	assertMakeNamePanics("fish", 0, "", "123.")       // punctuation not allowed
+	assertMakeNamePanics("fish", 0, "", "_123")
+}
+
+func testSmallFileInternals(t *testing.T, f *Fs) {
+	const dir = "small"
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+	f.opt.FailHard = false
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+
+	checkSmallFileInternals := func(obj fs.Object) {
+		assert.NotNil(t, obj)
+		o, ok := obj.(*Object)
+		assert.True(t, ok)
+		assert.NotNil(t, o)
+		if o == nil {
+			return
+		}
+		switch {
+		case !f.useMeta:
+			// If meta format is "none", non-chunked file (even empty)
+			// internally is a single chunk without meta object.
+			assert.Nil(t, o.main)
+			assert.True(t, o.isComposite()) // sorry, sometimes a name is misleading
+			assert.Equal(t, 1, len(o.chunks))
+		case f.hashAll:
+			// Consistent hashing forces meta object on small files too
+			assert.NotNil(t, o.main)
+			assert.True(t, o.isComposite())
+			assert.Equal(t, 1, len(o.chunks))
+		default:
+			// normally non-chunked file is kept in the Object's main field
+			assert.NotNil(t, o.main)
+			assert.False(t, o.isComposite())
+			assert.Equal(t, 0, len(o.chunks))
+		}
+	}
+
+	checkContents := func(obj fs.Object, contents string) {
+		assert.NotNil(t, obj)
+		assert.Equal(t, int64(len(contents)), obj.Size())
+
+		r, err := obj.Open(ctx)
+		assert.NoError(t, err)
+		assert.NotNil(t, r)
+		if r == nil {
+			return
+		}
+		data, err := ioutil.ReadAll(r)
+		assert.NoError(t, err)
+		assert.Equal(t, contents, string(data))
+		_ = r.Close()
+	}
+
+	checkHashsum := func(obj fs.Object) {
+		var ht hash.Type
+		switch {
+		case !f.hashAll:
+			return
+		case f.useMD5:
+			ht = hash.MD5
+		case f.useSHA1:
+			ht = hash.SHA1
+		default:
+			return
+		}
+		// even empty files must have hashsum in consistent mode
+		sum, err := obj.Hash(ctx, ht)
+		assert.NoError(t, err)
+		assert.NotEqual(t, sum, "")
+	}
+
+	checkSmallFile := func(name, contents string) {
+		filename := path.Join(dir, name)
+		item := fstest.Item{Path: filename, ModTime: modTime}
+		_, put := fstests.PutTestContents(ctx, t, f, &item, contents, false)
+		assert.NotNil(t, put)
+		checkSmallFileInternals(put)
+		checkContents(put, contents)
+		checkHashsum(put)
+
+		// objects returned by Put and NewObject must have similar structure
+		obj, err := f.NewObject(ctx, filename)
+		assert.NoError(t, err)
+		assert.NotNil(t, obj)
+		checkSmallFileInternals(obj)
+		checkContents(obj, contents)
+		checkHashsum(obj)
+
+		_ = obj.Remove(ctx)
+		_ = put.Remove(ctx) // for good
+	}
+
+	checkSmallFile("emptyfile", "")
+	checkSmallFile("smallfile", "Ok")
+}
+
+func testPreventCorruption(t *testing.T, f *Fs) {
+	if f.opt.ChunkSize > 50 {
+		t.Skip("this test requires small chunks")
+	}
+	const dir = "corrupted"
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+	f.opt.FailHard = true
+
+	contents := random.String(250)
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	const overlapMessage = "chunk overlap"
+
+	assertOverlapError := func(err error) {
+		assert.Error(t, err)
+		if err != nil {
+			assert.Contains(t, err.Error(), overlapMessage)
+		}
+	}
+
+	newFile := func(name string) fs.Object {
+		item := fstest.Item{Path: path.Join(dir, name), ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		require.NotNil(t, obj)
+		return obj
+	}
+	billyObj := newFile("billy")
+
+	billyChunkName := func(chunkNo int) string {
+		return f.makeChunkName(billyObj.Remote(), chunkNo, "", "")
+	}
+
+	err := f.Mkdir(ctx, billyChunkName(1))
+	assertOverlapError(err)
+
+	_, err = f.Move(ctx, newFile("silly1"), billyChunkName(2))
+	assert.Error(t, err)
+	assert.True(t, err == fs.ErrorCantMove || (err != nil && strings.Contains(err.Error(), overlapMessage)))
+
+	_, err = f.Copy(ctx, newFile("silly2"), billyChunkName(3))
+	assert.Error(t, err)
+	assert.True(t, err == fs.ErrorCantCopy || (err != nil && strings.Contains(err.Error(), overlapMessage)))
+
+	// accessing chunks in strict mode is prohibited
+	f.opt.FailHard = true
+	billyChunk4Name := billyChunkName(4)
+	billyChunk4, err := f.NewObject(ctx, billyChunk4Name)
+	assertOverlapError(err)
+
+	f.opt.FailHard = false
+	billyChunk4, err = f.NewObject(ctx, billyChunk4Name)
+	assert.NoError(t, err)
+	require.NotNil(t, billyChunk4)
+
+	f.opt.FailHard = true
+	_, err = f.Put(ctx, bytes.NewBufferString(contents), billyChunk4)
+	assertOverlapError(err)
+
+	// you can freely read chunks (if you have an object)
+	r, err := billyChunk4.Open(ctx)
+	assert.NoError(t, err)
+	var chunkContents []byte
+	assert.NotPanics(t, func() {
+		chunkContents, err = ioutil.ReadAll(r)
+		_ = r.Close()
+	})
+	assert.NoError(t, err)
+	assert.NotEqual(t, contents, string(chunkContents))
+
+	// but you can't change them
+	err = billyChunk4.Update(ctx, bytes.NewBufferString(contents), newFile("silly3"))
+	assertOverlapError(err)
+
+	// Remove isn't special, you can't corrupt files even if you have an object
+	err = billyChunk4.Remove(ctx)
+	assertOverlapError(err)
+
+	// recreate billy in case it was anyhow corrupted
+	willyObj := newFile("willy")
+	willyChunkName := f.makeChunkName(willyObj.Remote(), 1, "", "")
+	f.opt.FailHard = false
+	willyChunk, err := f.NewObject(ctx, willyChunkName)
+	f.opt.FailHard = true
+	assert.NoError(t, err)
+	require.NotNil(t, willyChunk)
+
+	_, err = operations.Copy(ctx, f, willyChunk, willyChunkName, newFile("silly4"))
+	assertOverlapError(err)
+
+	// operations.Move will return error when chunker's Move refused
+	// to corrupt target file, but reverts to copy/delete method
+	// still trying to delete target chunk. Chunker must come to rescue.
+	_, err = operations.Move(ctx, f, willyChunk, willyChunkName, newFile("silly5"))
+	assertOverlapError(err)
+	r, err = willyChunk.Open(ctx)
+	assert.NoError(t, err)
+	assert.NotPanics(t, func() {
+		_, err = ioutil.ReadAll(r)
+		_ = r.Close()
+	})
+	assert.NoError(t, err)
+}
+
+func testChunkNumberOverflow(t *testing.T, f *Fs) {
+	if f.opt.ChunkSize > 50 {
+		t.Skip("this test requires small chunks")
+	}
+	const dir = "wreaked"
+	const wreakNumber = 10200300
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+	contents := random.String(100)
+
+	newFile := func(f fs.Fs, name string) (fs.Object, string) {
+		filename := path.Join(dir, name)
+		item := fstest.Item{Path: filename, ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, true)
+		require.NotNil(t, obj)
+		return obj, filename
+	}
+
+	f.opt.FailHard = false
+	file, fileName := newFile(f, "wreaker")
+	wreak, _ := newFile(f.base, f.makeChunkName("wreaker", wreakNumber, "", ""))
+
+	f.opt.FailHard = false
+	fstest.CheckListingWithRoot(t, f, dir, nil, nil, f.Precision())
+	_, err := f.NewObject(ctx, fileName)
+	assert.Error(t, err)
+
+	f.opt.FailHard = true
+	_, err = f.List(ctx, dir)
+	assert.Error(t, err)
+	_, err = f.NewObject(ctx, fileName)
+	assert.Error(t, err)
+
+	f.opt.FailHard = false
+	_ = wreak.Remove(ctx)
+	_ = file.Remove(ctx)
+}
+
+func testMetadataInput(t *testing.T, f *Fs) {
+	const minChunkForTest = 50
+	if f.opt.ChunkSize < minChunkForTest {
+		t.Skip("this test requires chunks that fit metadata")
+	}
+
+	const dir = "usermeta"
+	ctx := context.Background()
+	saveOpt := f.opt
+	defer func() {
+		f.opt.FailHard = false
+		_ = operations.Purge(ctx, f.base, dir)
+		f.opt = saveOpt
+	}()
+	f.opt.FailHard = false
+
+	modTime := fstest.Time("2001-02-03T04:05:06.499999999Z")
+
+	putFile := func(f fs.Fs, name, contents, message string, check bool) fs.Object {
+		item := fstest.Item{Path: name, ModTime: modTime}
+		_, obj := fstests.PutTestContents(ctx, t, f, &item, contents, check)
+		assert.NotNil(t, obj, message)
+		return obj
+	}
+
+	runSubtest := func(contents, name string) {
+		description := fmt.Sprintf("file with %s metadata", name)
+		filename := path.Join(dir, name)
+		require.True(t, len(contents) > 2 && len(contents) < minChunkForTest, description+" test data is correct")
+
+		part := putFile(f.base, f.makeChunkName(filename, 0, "", ""), "oops", "", true)
+		_ = putFile(f, filename, contents, "upload "+description, false)
+
+		obj, err := f.NewObject(ctx, filename)
+		assert.NoError(t, err, "access "+description)
+		assert.NotNil(t, obj)
+		assert.Equal(t, int64(len(contents)), obj.Size(), "size "+description)
+
+		o, ok := obj.(*Object)
+		assert.NotNil(t, ok)
+		if o != nil {
+			assert.True(t, o.isComposite() && len(o.chunks) == 1, description+" is forced composite")
+			o = nil
+		}
+
+		defer func() {
+			_ = obj.Remove(ctx)
+			_ = part.Remove(ctx)
+		}()
+
+		r, err := obj.Open(ctx)
+		assert.NoError(t, err, "open "+description)
+		assert.NotNil(t, r, "open stream of "+description)
+		if err == nil && r != nil {
+			data, err := ioutil.ReadAll(r)
+			assert.NoError(t, err, "read all of "+description)
+			assert.Equal(t, contents, string(data), description+" contents is ok")
+			_ = r.Close()
+		}
+	}
+
+	metaData, err := marshalSimpleJSON(ctx, 3, 1, "", "")
+	require.NoError(t, err)
+	todaysMeta := string(metaData)
+	runSubtest(todaysMeta, "today")
+
+	pastMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":1`)
+	pastMeta = regexp.MustCompile(`"size":[0-9]+`).ReplaceAllLiteralString(pastMeta, `"size":0`)
+	runSubtest(pastMeta, "past")
+
+	futureMeta := regexp.MustCompile(`"ver":[0-9]+`).ReplaceAllLiteralString(todaysMeta, `"ver":999`)
+	futureMeta = regexp.MustCompile(`"nchunks":[0-9]+`).ReplaceAllLiteralString(futureMeta, `"nchunks":0,"x":"y"`)
+	runSubtest(futureMeta, "future")
+}
+
+// InternalTest dispatches all internal tests
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("PutLarge", func(t *testing.T) {
+		if *UploadKilobytes <= 0 {
+			t.Skip("-upload-kilobytes is not set")
+		}
+		testPutLarge(t, f, *UploadKilobytes)
+	})
+	t.Run("ChunkNameFormat", func(t *testing.T) {
+		testChunkNameFormat(t, f)
+	})
+	t.Run("SmallFileInternals", func(t *testing.T) {
+		testSmallFileInternals(t, f)
+	})
+	t.Run("PreventCorruption", func(t *testing.T) {
+		testPreventCorruption(t, f)
+	})
+	t.Run("ChunkNumberOverflow", func(t *testing.T) {
+		testChunkNumberOverflow(t, f)
+	})
+	t.Run("MetadataInput", func(t *testing.T) {
+		testMetadataInput(t, f)
+	})
+}
+
+var _ fstests.InternalTester = (*Fs)(nil)

backend/chunker/chunker_test.go

@@ -0,0 +1,58 @@
+// Test the Chunker filesystem interface
+package chunker_test
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+	"testing"
+
+	_ "github.com/rclone/rclone/backend/all" // for integration tests
+	"github.com/rclone/rclone/backend/chunker"
+	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// Command line flags
+var (
+	// Invalid characters are not supported by some remotes, eg. Mailru.
+	// We enable testing with invalid characters when -remote is not set, so
+	// chunker overlays a local directory, but invalid characters are disabled
+	// by default when -remote is set, eg. when test_all runs backend tests.
+	// You can still test with invalid characters using the below flag.
+	UseBadChars = flag.Bool("bad-chars", false, "Set to test bad characters in file names when -remote is set")
+)
+
+// TestIntegration runs integration tests against a concrete remote
+// set by the -remote flag. If the flag is not set, it creates a
+// dynamic chunker overlay wrapping a local temporary directory.
+func TestIntegration(t *testing.T) {
+	opt := fstests.Opt{
+		RemoteName:               *fstest.RemoteName,
+		NilObject:                (*chunker.Object)(nil),
+		SkipBadWindowsCharacters: !*UseBadChars,
+		UnimplementableObjectMethods: []string{
+			"MimeType",
+			"GetTier",
+			"SetTier",
+		},
+		UnimplementableFsMethods: []string{
+			"PublicLink",
+			"OpenWriterAt",
+			"MergeDirs",
+			"DirCacheFlush",
+			"UserInfo",
+			"Disconnect",
+		},
+	}
+	if *fstest.RemoteName == "" {
+		name := "TestChunker"
+		opt.RemoteName = name + ":"
+		tempDir := filepath.Join(os.TempDir(), "rclone-chunker-test-standard")
+		opt.ExtraConfig = []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "chunker"},
+			{Name: name, Key: "remote", Value: tempDir},
+		}
+	}
+	fstests.Run(t, &opt)
+}

backend/crypt/cipher.go

@@ -208,21 +208,6 @@ func (c *cipher) putBlock(buf []byte) {
 	c.buffers.Put(buf)
 }
 
-// check to see if the byte string is valid with no control characters
-// from 0x00 to 0x1F and is a valid UTF-8 string
-func checkValidString(buf []byte) error {
-	for i := range buf {
-		c := buf[i]
-		if c >= 0x00 && c < 0x20 || c == 0x7F {
-			return ErrorBadDecryptControlChar
-		}
-	}
-	if !utf8.Valid(buf) {
-		return ErrorBadDecryptUTF8
-	}
-	return nil
-}
-
 // encodeFileName encodes a filename using a modified version of
 // standard base32 as described in RFC4648
 //
@@ -294,10 +279,6 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	err = checkValidString(plaintext)
-	if err != nil {
-		return "", err
-	}
 	return string(plaintext), err
 }
 

backend/crypt/cipher_test.go

@@ -44,69 +44,6 @@ func TestNewNameEncryptionModeString(t *testing.T) {
 	assert.Equal(t, NameEncryptionMode(3).String(), "Unknown mode #3")
 }
 
-func TestValidString(t *testing.T) {
-	for _, test := range []struct {
-		in       string
-		expected error
-	}{
-		{"", nil},
-		{"\x01", ErrorBadDecryptControlChar},
-		{"a\x02", ErrorBadDecryptControlChar},
-		{"abc\x03", ErrorBadDecryptControlChar},
-		{"abc\x04def", ErrorBadDecryptControlChar},
-		{"\x05d", ErrorBadDecryptControlChar},
-		{"\x06def", ErrorBadDecryptControlChar},
-		{"\x07", ErrorBadDecryptControlChar},
-		{"\x08", ErrorBadDecryptControlChar},
-		{"\x09", ErrorBadDecryptControlChar},
-		{"\x0A", ErrorBadDecryptControlChar},
-		{"\x0B", ErrorBadDecryptControlChar},
-		{"\x0C", ErrorBadDecryptControlChar},
-		{"\x0D", ErrorBadDecryptControlChar},
-		{"\x0E", ErrorBadDecryptControlChar},
-		{"\x0F", ErrorBadDecryptControlChar},
-		{"\x10", ErrorBadDecryptControlChar},
-		{"\x11", ErrorBadDecryptControlChar},
-		{"\x12", ErrorBadDecryptControlChar},
-		{"\x13", ErrorBadDecryptControlChar},
-		{"\x14", ErrorBadDecryptControlChar},
-		{"\x15", ErrorBadDecryptControlChar},
-		{"\x16", ErrorBadDecryptControlChar},
-		{"\x17", ErrorBadDecryptControlChar},
-		{"\x18", ErrorBadDecryptControlChar},
-		{"\x19", ErrorBadDecryptControlChar},
-		{"\x1A", ErrorBadDecryptControlChar},
-		{"\x1B", ErrorBadDecryptControlChar},
-		{"\x1C", ErrorBadDecryptControlChar},
-		{"\x1D", ErrorBadDecryptControlChar},
-		{"\x1E", ErrorBadDecryptControlChar},
-		{"\x1F", ErrorBadDecryptControlChar},
-		{"\x20", nil},
-		{"\x7E", nil},
-		{"\x7F", ErrorBadDecryptControlChar},
-		{"£100", nil},
-		{`hello? sausage/êé/Hello, 世界/ " ' @ < > & ?/z.txt`, nil},
-		{"£100", nil},
-		// Following tests from https://secure.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
-		{"a", nil},                                        // Valid ASCII
-		{"\xc3\xb1", nil},                                 // Valid 2 Octet Sequence
-		{"\xc3\x28", ErrorBadDecryptUTF8},                 // Invalid 2 Octet Sequence
-		{"\xa0\xa1", ErrorBadDecryptUTF8},                 // Invalid Sequence Identifier
-		{"\xe2\x82\xa1", nil},                             // Valid 3 Octet Sequence
-		{"\xe2\x28\xa1", ErrorBadDecryptUTF8},             // Invalid 3 Octet Sequence (in 2nd Octet)
-		{"\xe2\x82\x28", ErrorBadDecryptUTF8},             // Invalid 3 Octet Sequence (in 3rd Octet)
-		{"\xf0\x90\x8c\xbc", nil},                         // Valid 4 Octet Sequence
-		{"\xf0\x28\x8c\xbc", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 2nd Octet)
-		{"\xf0\x90\x28\xbc", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 3rd Octet)
-		{"\xf0\x28\x8c\x28", ErrorBadDecryptUTF8},         // Invalid 4 Octet Sequence (in 4th Octet)
-		{"\xf8\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8},     // Valid 5 Octet Sequence (but not Unicode!)
-		{"\xfc\xa1\xa1\xa1\xa1\xa1", ErrorBadDecryptUTF8}, // Valid 6 Octet Sequence (but not Unicode!)
-	} {
-		actual := checkValidString([]byte(test.in))
-		assert.Equal(t, actual, test.expected, fmt.Sprintf("in=%q", test.in))
-	}
-}
-
 func TestEncodeFileName(t *testing.T) {
 	for _, test := range []struct {
 		in       string
@@ -210,8 +147,6 @@ func TestDecryptSegment(t *testing.T) {
 		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},
-		{c.encryptSegment("\x01"), ErrorBadDecryptControlChar},
-		{c.encryptSegment("\xc3\x28"), ErrorBadDecryptUTF8},
 	} {
 		actual, actualErr := c.decryptSegment(test.in)
 		assert.Equal(t, test.expectedErr, actualErr, fmt.Sprintf("in=%q got actual=%q, err = %v %T", test.in, actual, actualErr, actualErr))

backend/crypt/crypt.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"path"
 	"strings"
 	"time"
 
@@ -35,19 +36,21 @@ func init() {
 			Default: "standard",
 			Examples: []fs.OptionExample{
 				{
-					Value: "off",
-					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
-				}, {
 					Value: "standard",
 					Help:  "Encrypt the filenames see the docs for the details.",
 				}, {
 					Value: "obfuscate",
 					Help:  "Very simple filename obfuscation.",
+				}, {
+					Value: "off",
+					Help:  "Don't encrypt the file names.  Adds a \".bin\" extension only.",
 				},
 			},
 		}, {
-			Name:    "directory_name_encryption",
-			Help:    "Option to either encrypt directory names or leave them intact.",
+			Name: "directory_name_encryption",
+			Help: `Option to either encrypt directory names or leave them intact.
+
+NB If filename_encryption is "off" then this option will do nothing.`,
 			Default: true,
 			Examples: []fs.OptionExample{
 				{
@@ -63,6 +66,7 @@ func init() {
 			Name:       "password",
 			Help:       "Password or pass phrase for encryption.",
 			IsPassword: true,
+			Required:   true,
 		}, {
 			Name:       "password2",
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
@@ -143,6 +147,10 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
 	}
+	// Make sure to remove trailing . reffering to the current dir
+	if path.Base(rpath) == "." {
+		rpath = strings.TrimSuffix(rpath, ".")
+	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath))
 	wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig)

backend/drive/drive.go

@@ -10,6 +10,7 @@ package drive
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -37,6 +38,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -210,7 +212,15 @@ func init() {
 			}},
 		}, {
 			Name: "root_folder_id",
-			Help: "ID of the root folder\nLeave blank normally.\nFill in to access \"Computers\" folders. (see docs).",
+			Help: `ID of the root folder
+Leave blank normally.
+
+Fill in to access "Computers" folders (see docs), or for rclone to use
+a non root folder as its starting point.
+
+Note that if this is blank, the first time rclone runs it will fill it
+in with the ID of the root folder.
+`,
 		}, {
 			Name: "service_account_file",
 			Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
@@ -314,6 +324,19 @@ Photos folder" option in your google drive settings. You can then copy
 or move the photos locally and use the date the image was taken
 (created) set as the modification date.`,
 			Advanced: true,
+			Hide:     fs.OptionHideConfigurator,
+		}, {
+			Name:    "use_shared_date",
+			Default: false,
+			Help: `Use date file was shared instead of modified date.
+
+Note that, as with "--drive-use-created-date", this flag may have
+unexpected consequences when uploading/downloading files.
+
+If both this flag and "--drive-use-created-date" are set, the created
+date is used.`,
+			Advanced: true,
+			Hide:     fs.OptionHideConfigurator,
 		}, {
 			Name:     "list_chunk",
 			Default:  1000,
@@ -372,11 +395,22 @@ will download it anyway.`,
 		}, {
 			Name:    "size_as_quota",
 			Default: false,
-			Help: `Show storage quota usage for file size.
+			Help: `Show sizes as storage quota usage, not actual size.
 
-The storage used by a file is the size of the current version plus any
-older versions that have been set to keep forever.`,
+Show the size of a file as the the storage quota used. This is the
+current version plus any older versions that have been set to keep
+forever.
+
+**WARNING**: This flag may have some unexpected consequences.
+
+It is not recommended to set this flag in your config - the
+recommended usage is using the flag form --drive-size-as-quota when
+doing rclone ls/lsl/lsf/lsjson/etc only.
+
+If you do use this flag for syncing (not recommended) then you will
+need to use --ignore size also.`,
 			Advanced: true,
+			Hide:     fs.OptionHideConfigurator,
 		}, {
 			Name:     "v2_download_min_size",
 			Default:  fs.SizeSuffix(-1),
@@ -399,9 +433,47 @@ older versions that have been set to keep forever.`,
 
 This can be useful if you wish to do a server side copy between two
 different Google drives.  Note that this isn't enabled by default
-because it isn't easy to tell if it will work beween any two
+because it isn't easy to tell if it will work between any two
 configurations.`,
 			Advanced: true,
+		}, {
+			Name:    "disable_http2",
+			Default: true,
+			Help: `Disable drive using http2
+
+There is currently an unsolved issue with the google drive backend and
+HTTP/2.  HTTP/2 is therefore disabled by default for the drive backend
+but can be re-enabled here.  When the issue is solved this flag will
+be removed.
+
+See: https://github.com/rclone/rclone/issues/3631
+
+`,
+			Advanced: true,
+		}, {
+			Name:    "stop_on_upload_limit",
+			Default: false,
+			Help: `Make upload limit errors be fatal
+
+At the time of writing it is only possible to upload 750GB of data to
+Google Drive a day (this is an undocumented limit). When this limit is
+reached Google Drive produces a slightly different error message. When
+this flag is set it causes these errors to be fatal.  These will stop
+the in-progress sync.
+
+Note that this detection is relying on error message strings which
+Google don't document so it may break in the future.
+
+See: https://github.com/rclone/rclone/issues/3857
+`,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			// Don't encode / as it's a valid name character in drive.
+			Default: encoder.EncodeInvalidUtf8,
 		}},
 	})
 
@@ -421,34 +493,38 @@ configurations.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Scope                     string        `config:"scope"`
-	RootFolderID              string        `config:"root_folder_id"`
-	ServiceAccountFile        string        `config:"service_account_file"`
-	ServiceAccountCredentials string        `config:"service_account_credentials"`
-	TeamDriveID               string        `config:"team_drive"`
-	AuthOwnerOnly             bool          `config:"auth_owner_only"`
-	UseTrash                  bool          `config:"use_trash"`
-	SkipGdocs                 bool          `config:"skip_gdocs"`
-	SkipChecksumGphotos       bool          `config:"skip_checksum_gphotos"`
-	SharedWithMe              bool          `config:"shared_with_me"`
-	TrashedOnly               bool          `config:"trashed_only"`
-	Extensions                string        `config:"formats"`
-	ExportExtensions          string        `config:"export_formats"`
-	ImportExtensions          string        `config:"import_formats"`
-	AllowImportNameChange     bool          `config:"allow_import_name_change"`
-	UseCreatedDate            bool          `config:"use_created_date"`
-	ListChunk                 int64         `config:"list_chunk"`
-	Impersonate               string        `config:"impersonate"`
-	AlternateExport           bool          `config:"alternate_export"`
-	UploadCutoff              fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize                 fs.SizeSuffix `config:"chunk_size"`
-	AcknowledgeAbuse          bool          `config:"acknowledge_abuse"`
-	KeepRevisionForever       bool          `config:"keep_revision_forever"`
-	SizeAsQuota               bool          `config:"size_as_quota"`
-	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
-	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
-	PacerBurst                int           `config:"pacer_burst"`
-	ServerSideAcrossConfigs   bool          `config:"server_side_across_configs"`
+	Scope                     string               `config:"scope"`
+	RootFolderID              string               `config:"root_folder_id"`
+	ServiceAccountFile        string               `config:"service_account_file"`
+	ServiceAccountCredentials string               `config:"service_account_credentials"`
+	TeamDriveID               string               `config:"team_drive"`
+	AuthOwnerOnly             bool                 `config:"auth_owner_only"`
+	UseTrash                  bool                 `config:"use_trash"`
+	SkipGdocs                 bool                 `config:"skip_gdocs"`
+	SkipChecksumGphotos       bool                 `config:"skip_checksum_gphotos"`
+	SharedWithMe              bool                 `config:"shared_with_me"`
+	TrashedOnly               bool                 `config:"trashed_only"`
+	Extensions                string               `config:"formats"`
+	ExportExtensions          string               `config:"export_formats"`
+	ImportExtensions          string               `config:"import_formats"`
+	AllowImportNameChange     bool                 `config:"allow_import_name_change"`
+	UseCreatedDate            bool                 `config:"use_created_date"`
+	UseSharedDate             bool                 `config:"use_shared_date"`
+	ListChunk                 int64                `config:"list_chunk"`
+	Impersonate               string               `config:"impersonate"`
+	AlternateExport           bool                 `config:"alternate_export"`
+	UploadCutoff              fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize                 fs.SizeSuffix        `config:"chunk_size"`
+	AcknowledgeAbuse          bool                 `config:"acknowledge_abuse"`
+	KeepRevisionForever       bool                 `config:"keep_revision_forever"`
+	SizeAsQuota               bool                 `config:"size_as_quota"`
+	V2DownloadMinSize         fs.SizeSuffix        `config:"v2_download_min_size"`
+	PacerMinSleep             fs.Duration          `config:"pacer_min_sleep"`
+	PacerBurst                int                  `config:"pacer_burst"`
+	ServerSideAcrossConfigs   bool                 `config:"server_side_across_configs"`
+	DisableHTTP2              bool                 `config:"disable_http2"`
+	StopOnUploadLimit         bool                 `config:"stop_on_upload_limit"`
+	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote drive server
@@ -519,7 +595,7 @@ func (f *Fs) Features() *fs.Features {
 }
 
 // shouldRetry determines whether a given err rates being retried
-func shouldRetry(err error) (bool, error) {
+func (f *Fs) shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
 	}
@@ -535,6 +611,10 @@ func shouldRetry(err error) (bool, error) {
 		if len(gerr.Errors) > 0 {
 			reason := gerr.Errors[0].Reason
 			if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
+				if f.opt.StopOnUploadLimit && gerr.Errors[0].Message == "User rate limit exceeded." {
+					fs.Errorf(f, "Received upload limit error: %v", err)
+					return false, fserrors.FatalError(err)
+				}
 				return true, err
 			}
 		}
@@ -562,6 +642,23 @@ func containsString(slice []string, s string) bool {
 	return false
 }
 
+// getRootID returns the canonical ID for the "root" ID
+func (f *Fs) getRootID() (string, error) {
+	var info *drive.File
+	var err error
+	err = f.pacer.CallNoRetry(func() (bool, error) {
+		info, err = f.svc.Files.Get("root").
+			Fields("id").
+			SupportsAllDrives(true).
+			Do()
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return "", errors.Wrap(err, "couldn't find root directory ID")
+	}
+	return info.Id, nil
+}
+
 // Lists the directory required calling the user function on each item found
 //
 // If the user fn ever returns true then it early exits with found = true
@@ -599,11 +696,10 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	}
 	var stems []string
 	if title != "" {
+		searchTitle := f.opt.Enc.FromStandardName(title)
 		// Escaping the backslash isn't documented but seems to work
-		searchTitle := strings.Replace(title, `\`, `\\`, -1)
+		searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
 		searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
-		// Convert ／ to / for search
-		searchTitle = strings.Replace(searchTitle, "／", "/", -1)
 
 		var titleQuery bytes.Buffer
 		_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
@@ -651,6 +747,9 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 	if f.opt.AuthOwnerOnly {
 		fields += ",owners"
 	}
+	if f.opt.UseSharedDate {
+		fields += ",sharedWithMeTime"
+	}
 	if f.opt.SkipChecksumGphotos {
 		fields += ",spaces"
 	}
@@ -658,24 +757,25 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		fields += ",quotaBytesUsed"
 	}
 
-	fields = fmt.Sprintf("files(%s),nextPageToken", fields)
+	fields = fmt.Sprintf("files(%s),nextPageToken,incompleteSearch", fields)
 
 OUTER:
 	for {
 		var files *drive.FileList
 		err = f.pacer.Call(func() (bool, error) {
 			files, err = list.Fields(googleapi.Field(fields)).Context(ctx).Do()
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return false, errors.Wrap(err, "couldn't list directory")
 		}
+		if files.IncompleteSearch {
+			fs.Errorf(f, "search result INCOMPLETE")
+		}
 		for _, item := range files.Files {
-			// Convert / to ／ for listing purposes
-			item.Name = strings.Replace(item.Name, "/", "／", -1)
+			item.Name = f.opt.Enc.ToStandardName(item.Name)
 			// Check the case of items is correct since
 			// the `=` operator is case insensitive.
-
 			if title != "" && title != item.Name {
 				found := false
 				for _, stem := range stems {
@@ -789,7 +889,7 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
 	} else {
 		fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
 	}
-	if !config.Confirm() {
+	if !config.Confirm(false) {
 		return nil
 	}
 	client, err := createOAuthClient(opt, name, m)
@@ -804,11 +904,12 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
 	var driveIDs, driveNames []string
 	listTeamDrives := svc.Teamdrives.List().PageSize(100)
 	listFailed := false
+	var defaultFs Fs // default Fs with default Options
 	for {
 		var teamDrives *drive.TeamDriveList
 		err = newPacer(opt).Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Context(ctx).Do()
-			return shouldRetry(err)
+			return defaultFs.shouldRetry(err)
 		})
 		if err != nil {
 			fmt.Printf("Listing team drives failed: %v\n", err)
@@ -840,6 +941,18 @@ func newPacer(opt *Options) *fs.Pacer {
 	return fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst)))
 }
 
+// getClient makes an http client according to the options
+func getClient(opt *Options) *http.Client {
+	t := fshttp.NewTransportCustom(fs.Config, func(t *http.Transport) {
+		if opt.DisableHTTP2 {
+			t.TLSNextProto = map[string]func(string, *tls.Conn) http.RoundTripper{}
+		}
+	})
+	return &http.Client{
+		Transport: t,
+	}
+}
+
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
 	scopes := driveScopes(opt.Scope)
 	conf, err := google.JWTConfigFromJSON(credentialsData, scopes...)
@@ -849,7 +962,7 @@ func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client
 	if opt.Impersonate != "" {
 		conf.Subject = opt.Impersonate
 	}
-	ctxWithSpecialClient := oauthutil.Context(fshttp.NewClient(fs.Config))
+	ctxWithSpecialClient := oauthutil.Context(getClient(opt))
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 
@@ -871,7 +984,7 @@ func createOAuthClient(opt *Options, name string, m configmap.Mapper) (*http.Cli
 			return nil, errors.Wrap(err, "failed to create oauth client from service account")
 		}
 	} else {
-		oAuthClient, _, err = oauthutil.NewClient(name, m, driveConfig)
+		oAuthClient, _, err = oauthutil.NewClientWithBaseClient(name, m, driveConfig, getClient(opt))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create oauth client")
 		}
@@ -968,15 +1081,25 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// set root folder for a team drive or query the user root folder
-	if f.isTeamDrive {
+	if opt.RootFolderID != "" {
+		// override root folder if set or cached in the config
+		f.rootFolderID = opt.RootFolderID
+	} else if f.isTeamDrive {
 		f.rootFolderID = f.opt.TeamDriveID
 	} else {
-		f.rootFolderID = "root"
-	}
-
-	// override root folder if set in the config
-	if opt.RootFolderID != "" {
-		f.rootFolderID = opt.RootFolderID
+		// Look up the root ID and cache it in the config
+		rootID, err := f.getRootID()
+		if err != nil {
+			if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
+				// 404 means that this scope does not have permission to get the
+				// root so just use "root"
+				rootID = "root"
+			} else {
+				return nil, err
+			}
+		}
+		f.rootFolderID = rootID
+		m.Set("root_folder_id", rootID)
 	}
 
 	f.dirCache = dircache.New(root, f.rootFolderID, f)
@@ -1032,6 +1155,8 @@ func (f *Fs) newBaseObject(remote string, info *drive.File) baseObject {
 	modifiedDate := info.ModifiedTime
 	if f.opt.UseCreatedDate {
 		modifiedDate = info.CreatedTime
+	} else if f.opt.UseSharedDate && info.SharedWithMeTime != "" {
+		modifiedDate = info.SharedWithMeTime
 	}
 	size := info.Size
 	if f.opt.SizeAsQuota {
@@ -1210,6 +1335,7 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 
 // CreateDir makes a directory with pathID as parent and name leaf
 func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string, err error) {
+	leaf = f.opt.Enc.FromStandardName(leaf)
 	// fmt.Println("Making", path)
 	// Define the metadata for the directory we are going to create.
 	createInfo := &drive.File{
@@ -1224,7 +1350,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 			Fields("id").
 			SupportsAllDrives(true).
 			Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return "", err
@@ -1267,7 +1393,7 @@ func (f *Fs) fetchFormats() {
 			about, err = f.svc.About.Get().
 				Fields("exportFormats,importFormats").
 				Do()
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			fs.Errorf(f, "Failed to get Drive exportFormats and importFormats: %v", err)
@@ -1399,6 +1525,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	if iErr != nil {
 		return nil, iErr
 	}
+	// If listing the root of a teamdrive and got no entries,
+	// double check we have access
+	if f.isTeamDrive && len(entries) == 0 && f.root == "" && dir == "" {
+		err = f.teamDriveOK(ctx)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return entries, nil
 }
 
@@ -1458,10 +1592,22 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
 		var iErr error
 		_, err := f.list(ctx, dirs, "", false, false, false, func(item *drive.File) bool {
 			for _, parent := range item.Parents {
-				// only handle parents that are in the requested dirs list
-				i := sort.SearchStrings(dirs, parent)
-				if i == len(dirs) || dirs[i] != parent {
-					continue
+				var i int
+				// If only one item in paths then no need to search for the ID
+				// assuming google drive is doing its job properly.
+				//
+				// Note that we at the root when len(paths) == 1 && paths[0] == ""
+				if len(paths) == 1 {
+					// don't check parents at root because
+					// - shared with me items have no parents at the root
+					// - if using a root alias, eg "root" or "appDataFolder" the ID won't match
+					i = 0
+				} else {
+					// only handle parents that are in the requested dirs list if not at root
+					i = sort.SearchStrings(dirs, parent)
+					if i == len(dirs) || dirs[i] != parent {
+						continue
+					}
 				}
 				remote := path.Join(paths[i], item.Name)
 				entry, err := f.itemToDirEntry(remote, item)
@@ -1525,20 +1671,6 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	if err != nil {
 		return err
 	}
-	if directoryID == "root" {
-		var info *drive.File
-		err = f.pacer.CallNoRetry(func() (bool, error) {
-			info, err = f.svc.Files.Get("root").
-				Fields("id").
-				SupportsAllDrives(true).
-				Do()
-			return shouldRetry(err)
-		})
-		if err != nil {
-			return err
-		}
-		directoryID = info.Id
-	}
 
 	mu := sync.Mutex{} // protects in and overflow
 	wg := sync.WaitGroup{}
@@ -1546,6 +1678,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	out := make(chan error, fs.Config.Checkers)
 	list := walk.NewListRHelper(callback)
 	overflow := []listREntry{}
+	listed := 0
 
 	cb := func(entry fs.DirEntry) error {
 		mu.Lock()
@@ -1558,6 +1691,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 				overflow = append(overflow, listREntry{d.ID(), d.Remote()})
 			}
 		}
+		listed++
 		return list.Add(entry)
 	}
 
@@ -1614,7 +1748,21 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		return err
 	}
 
-	return list.Flush()
+	err = list.Flush()
+	if err != nil {
+		return err
+	}
+
+	// If listing the root of a teamdrive and got no entries,
+	// double check we have access
+	if f.isTeamDrive && listed == 0 && f.root == "" && dir == "" {
+		err = f.teamDriveOK(ctx)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
 }
 
 // itemToDirEntry converts a drive.File to a fs.DirEntry.
@@ -1645,6 +1793,7 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 		return nil, err
 	}
 
+	leaf = f.opt.Enc.FromStandardName(leaf)
 	// Define the metadata for the file we are going to create.
 	createInfo := &drive.File{
 		Name:         leaf,
@@ -1718,7 +1867,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	}
 
 	var info *drive.File
-	if size == 0 || size < int64(f.opt.UploadCutoff) {
+	if size >= 0 && size < int64(f.opt.UploadCutoff) {
 		// Make the API request to upload metadata and file data.
 		// Don't retry, return a retry error instead
 		err = f.pacer.CallNoRetry(func() (bool, error) {
@@ -1728,7 +1877,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 				SupportsAllDrives(true).
 				KeepRevisionForever(f.opt.KeepRevisionForever).
 				Do()
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return nil, err
@@ -1771,7 +1920,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 					Fields("").
 					SupportsAllDrives(true).
 					Do()
-				return shouldRetry(err)
+				return f.shouldRetry(err)
 			})
 			if err != nil {
 				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
@@ -1817,7 +1966,7 @@ func (f *Fs) rmdir(ctx context.Context, directoryID string, useTrash bool) error
 				SupportsAllDrives(true).
 				Do()
 		}
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 }
 
@@ -1915,7 +2064,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			SupportsAllDrives(true).
 			KeepRevisionForever(f.opt.KeepRevisionForever).
 			Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1964,7 +2113,7 @@ func (f *Fs) Purge(ctx context.Context) error {
 				SupportsAllDrives(true).
 				Do()
 		}
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	f.dirCache.ResetRoot()
 	if err != nil {
@@ -1977,7 +2126,7 @@ func (f *Fs) Purge(ctx context.Context) error {
 func (f *Fs) CleanUp(ctx context.Context) error {
 	err := f.pacer.Call(func() (bool, error) {
 		err := f.svc.Files.EmptyTrash().Context(ctx).Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 
 	if err != nil {
@@ -1986,9 +2135,30 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return nil
 }
 
+// teamDriveOK checks to see if we can access the team drive
+func (f *Fs) teamDriveOK(ctx context.Context) (err error) {
+	if !f.isTeamDrive {
+		return nil
+	}
+	var td *drive.Drive
+	err = f.pacer.Call(func() (bool, error) {
+		td, err = f.svc.Drives.Get(f.opt.TeamDriveID).Fields("name,id,capabilities,createdTime,restrictions").Context(ctx).Do()
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "failed to get Team/Shared Drive info")
+	}
+	fs.Debugf(f, "read info from team drive %q", td.Name)
+	return err
+}
+
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	if f.isTeamDrive {
+		err := f.teamDriveOK(ctx)
+		if err != nil {
+			return nil, err
+		}
 		// Teamdrives don't appear to have a usage API so just return empty
 		return &fs.Usage{}, nil
 	}
@@ -1996,7 +2166,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		about, err = f.svc.About.Get().Fields("storageQuota").Context(ctx).Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to get Drive storageQuota")
@@ -2068,7 +2238,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 			Fields(partialFields).
 			SupportsAllDrives(true).
 			Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -2104,7 +2274,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 			Fields("").
 			SupportsAllDrives(true).
 			Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return "", err
@@ -2204,7 +2374,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 			Fields("").
 			SupportsAllDrives(true).
 			Do()
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -2265,10 +2435,12 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 func (f *Fs) changeNotifyStartPageToken() (pageToken string, err error) {
 	var startPageToken *drive.StartPageToken
 	err = f.pacer.Call(func() (bool, error) {
-		startPageToken, err = f.svc.Changes.GetStartPageToken().
-			SupportsAllDrives(true).
-			Do()
-		return shouldRetry(err)
+		changes := f.svc.Changes.GetStartPageToken().SupportsAllDrives(true)
+		if f.isTeamDrive {
+			changes.DriveId(f.opt.TeamDriveID)
+		}
+		startPageToken, err = changes.Do()
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return
@@ -2290,10 +2462,14 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 			changesCall.SupportsAllDrives(true)
 			changesCall.IncludeItemsFromAllDrives(true)
 			if f.isTeamDrive {
-				changesCall.TeamDriveId(f.opt.TeamDriveID)
+				changesCall.DriveId(f.opt.TeamDriveID)
+			}
+			// If using appDataFolder then need to add Spaces
+			if f.rootFolderID == "appDataFolder" {
+				changesCall.Spaces("appDataFolder")
 			}
 			changeList, err = changesCall.Context(ctx).Do()
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			return
@@ -2316,6 +2492,7 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
 
 			// find the new path
 			if change.File != nil {
+				change.File.Name = f.opt.Enc.ToStandardName(change.File.Name)
 				changeType := fs.EntryDirectory
 				if change.File.MimeType != driveFolderType {
 					changeType = fs.EntryObject
@@ -2483,7 +2660,7 @@ func (o *baseObject) SetModTime(ctx context.Context, modTime time.Time) error {
 			Fields(partialFields).
 			SupportsAllDrives(true).
 			Do()
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return err
@@ -2522,7 +2699,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 				_ = res.Body.Close() // ignore error
 			}
 		}
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return req, nil, err
@@ -2612,7 +2789,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 				Fields("downloadUrl").
 				SupportsAllDrives(true).
 				Do()
-			return shouldRetry(err)
+			return o.fs.shouldRetry(err)
 		})
 		if err == nil {
 			fs.Debugf(o, "Using v2 download: %v", v2File.DownloadUrl)
@@ -2684,7 +2861,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 	src fs.ObjectInfo) (info *drive.File, err error) {
 	// Make the API request to upload metadata and file data.
 	size := src.Size()
-	if size == 0 || size < int64(o.fs.opt.UploadCutoff) {
+	if size >= 0 && size < int64(o.fs.opt.UploadCutoff) {
 		// Don't retry, return a retry error instead
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			info, err = o.fs.svc.Files.Update(o.id, updateInfo).
@@ -2693,7 +2870,7 @@ func (o *baseObject) update(ctx context.Context, updateInfo *drive.File, uploadM
 				SupportsAllDrives(true).
 				KeepRevisionForever(o.fs.opt.KeepRevisionForever).
 				Do()
-			return shouldRetry(err)
+			return o.fs.shouldRetry(err)
 		})
 		return
 	}
@@ -2793,7 +2970,7 @@ func (o *baseObject) Remove(ctx context.Context) error {
 				SupportsAllDrives(true).
 				Do()
 		}
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	return err
 }

backend/drive/upload.go

@@ -11,16 +11,15 @@
 package drive
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
-	"regexp"
 	"strconv"
 
-	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/lib/readers"
@@ -88,13 +87,15 @@ func (f *Fs) Upload(ctx context.Context, in io.Reader, size int64, contentType,
 		})
 		req.Header.Set("Content-Type", "application/json; charset=UTF-8")
 		req.Header.Set("X-Upload-Content-Type", contentType)
-		req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
+		if size >= 0 {
+			req.Header.Set("X-Upload-Content-Length", fmt.Sprintf("%v", size))
+		}
 		res, err = f.client.Do(req)
 		if err == nil {
 			defer googleapi.CloseBody(res)
 			err = googleapi.CheckResponse(res)
 		}
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -116,49 +117,19 @@ func (rx *resumableUpload) makeRequest(ctx context.Context, start int64, body io
 	req, _ := http.NewRequest("POST", rx.URI, body)
 	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 	req.ContentLength = reqSize
+	totalSize := "*"
+	if rx.ContentLength >= 0 {
+		totalSize = strconv.FormatInt(rx.ContentLength, 10)
+	}
 	if reqSize != 0 {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, rx.ContentLength))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes %v-%v/%v", start, start+reqSize-1, totalSize))
 	} else {
-		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", rx.ContentLength))
+		req.Header.Set("Content-Range", fmt.Sprintf("bytes */%v", totalSize))
 	}
 	req.Header.Set("Content-Type", rx.MediaType)
 	return req
 }
 
-// rangeRE matches the transfer status response from the server. $1 is
-// the last byte index uploaded.
-var rangeRE = regexp.MustCompile(`^0\-(\d+)$`)
-
-// Query drive for the amount transferred so far
-//
-// If error is nil, then start should be valid
-func (rx *resumableUpload) transferStatus(ctx context.Context) (start int64, err error) {
-	req := rx.makeRequest(ctx, 0, nil, 0)
-	res, err := rx.f.client.Do(req)
-	if err != nil {
-		return 0, err
-	}
-	defer googleapi.CloseBody(res)
-	if res.StatusCode == http.StatusCreated || res.StatusCode == http.StatusOK {
-		return rx.ContentLength, nil
-	}
-	if res.StatusCode != statusResumeIncomplete {
-		err = googleapi.CheckResponse(res)
-		if err != nil {
-			return 0, err
-		}
-		return 0, errors.Errorf("unexpected http return code %v", res.StatusCode)
-	}
-	Range := res.Header.Get("Range")
-	if m := rangeRE.FindStringSubmatch(Range); len(m) == 2 {
-		start, err = strconv.ParseInt(m[1], 10, 64)
-		if err == nil {
-			return start, nil
-		}
-	}
-	return 0, errors.Errorf("unable to parse range %q", Range)
-}
-
 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
 func (rx *resumableUpload) transferChunk(ctx context.Context, start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
 	_, _ = chunk.Seek(0, io.SeekStart)
@@ -200,18 +171,40 @@ func (rx *resumableUpload) Upload(ctx context.Context) (*drive.File, error) {
 	var StatusCode int
 	var err error
 	buf := make([]byte, int(rx.f.opt.ChunkSize))
-	for start < rx.ContentLength {
-		reqSize := rx.ContentLength - start
-		if reqSize >= int64(rx.f.opt.ChunkSize) {
-			reqSize = int64(rx.f.opt.ChunkSize)
+	for finished := false; !finished; {
+		var reqSize int64
+		var chunk io.ReadSeeker
+		if rx.ContentLength >= 0 {
+			// If size known use repeatable reader for smoother bwlimit
+			if start >= rx.ContentLength {
+				break
+			}
+			reqSize = rx.ContentLength - start
+			if reqSize >= int64(rx.f.opt.ChunkSize) {
+				reqSize = int64(rx.f.opt.ChunkSize)
+			}
+			chunk = readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
+		} else {
+			// If size unknown read into buffer
+			var n int
+			n, err = readers.ReadFill(rx.Media, buf)
+			if err == io.EOF {
+				// Send the last chunk with the correct ContentLength
+				// otherwise Google doesn't know we've finished
+				rx.ContentLength = start + int64(n)
+				finished = true
+			} else if err != nil {
+				return nil, err
+			}
+			reqSize = int64(n)
+			chunk = bytes.NewReader(buf[:reqSize])
 		}
-		chunk := readers.NewRepeatableLimitReaderBuffer(rx.Media, buf, reqSize)
 
 		// Transfer the chunk
 		err = rx.f.pacer.Call(func() (bool, error) {
 			fs.Debugf(rx.remote, "Sending chunk %d length %d", start, reqSize)
 			StatusCode, err = rx.transferChunk(ctx, start, chunk, reqSize)
-			again, err := shouldRetry(err)
+			again, err := rx.f.shouldRetry(err)
 			if StatusCode == statusResumeIncomplete || StatusCode == http.StatusCreated || StatusCode == http.StatusOK {
 				again = false
 				err = nil

backend/dropbox/dropbox.go

@@ -39,6 +39,7 @@ import (
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
 	"github.com/pkg/errors"
+	"github.com/rclone/rclone/backend/dropbox/dbhash"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -46,6 +47,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -102,10 +104,14 @@ var (
 	// A regexp matching path names for files Dropbox ignores
 	// See https://www.dropbox.com/en/help/145 - Ignored files
 	ignoredFiles = regexp.MustCompile(`(?i)(^|/)(desktop\.ini|thumbs\.db|\.ds_store|icon\r|\.dropbox|\.dropbox.attr)$`)
+
+	// DbHashType is the hash.Type for Dropbox
+	DbHashType hash.Type
 )
 
 // Register with Fs
 func init() {
+	DbHashType = hash.RegisterHash("DropboxHash", 64, dbhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "dropbox",
 		Description: "Dropbox",
@@ -139,14 +145,28 @@ memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
 			Help:     "Impersonate this user when using a business account.",
 			Default:  "",
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// https://www.dropbox.com/help/syncing-uploads/files-not-syncing lists / and \
+			// as invalid characters.
+			// Testing revealed names with trailing spaces and the DEL character don't work.
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDel |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize   fs.SizeSuffix `config:"chunk_size"`
-	Impersonate string        `config:"impersonate"`
+	ChunkSize   fs.SizeSuffix        `config:"chunk_size"`
+	Impersonate string               `config:"impersonate"`
+	Enc         encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote dropbox server
@@ -372,14 +392,15 @@ func (f *Fs) setRoot(root string) {
 // getMetadata gets the metadata for a file or directory
 func (f *Fs) getMetadata(objPath string) (entry files.IsMetadata, notFound bool, err error) {
 	err = f.pacer.Call(func() (bool, error) {
-		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{Path: objPath})
+		entry, err = f.srv.GetMetadata(&files.GetMetadataArg{
+			Path: f.opt.Enc.FromStandardPath(objPath),
+		})
 		return shouldRetry(err)
 	})
 	if err != nil {
 		switch e := err.(type) {
 		case files.GetMetadataAPIError:
-			switch e.EndpointError.Path.Tag {
-			case files.LookupErrorNotFound:
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
 				notFound = true
 				err = nil
 			}
@@ -466,7 +487,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	for {
 		if !started {
 			arg := files.ListFolderArg{
-				Path:      root,
+				Path:      f.opt.Enc.FromStandardPath(root),
 				Recursive: false,
 			}
 			if root == "/" {
@@ -479,8 +500,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			if err != nil {
 				switch e := err.(type) {
 				case files.ListFolderAPIError:
-					switch e.EndpointError.Path.Tag {
-					case files.LookupErrorNotFound:
+					if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound {
 						err = fs.ErrorDirNotFound
 					}
 				}
@@ -517,7 +537,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 			// Only the last element is reliably cased in PathDisplay
 			entryPath := metadata.PathDisplay
-			leaf := path.Base(entryPath)
+			leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
 			remote := path.Join(dir, leaf)
 			if folderInfo != nil {
 				d := fs.NewDir(remote, time.Now())
@@ -575,7 +595,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 
 	// create it
 	arg2 := files.CreateFolderArg{
-		Path: root,
+		Path: f.opt.Enc.FromStandardPath(root),
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.CreateFolderV2(&arg2)
@@ -601,6 +621,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return errors.Wrap(err, "Rmdir")
 	}
 
+	root = f.opt.Enc.FromStandardPath(root)
 	// check directory empty
 	arg := files.ListFolderArg{
 		Path:      root,
@@ -657,9 +678,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Copy
-	arg := files.RelocationArg{}
-	arg.FromPath = srcObj.remotePath()
-	arg.ToPath = dstObj.remotePath()
+	arg := files.RelocationArg{
+		RelocationPath: files.RelocationPath{
+			FromPath: f.opt.Enc.FromStandardPath(srcObj.remotePath()),
+			ToPath:   f.opt.Enc.FromStandardPath(dstObj.remotePath()),
+		},
+	}
 	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
@@ -691,7 +715,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 func (f *Fs) Purge(ctx context.Context) (err error) {
 	// Let dropbox delete the filesystem tree
 	err = f.pacer.Call(func() (bool, error) {
-		_, err = f.srv.DeleteV2(&files.DeleteArg{Path: f.slashRoot})
+		_, err = f.srv.DeleteV2(&files.DeleteArg{
+			Path: f.opt.Enc.FromStandardPath(f.slashRoot),
+		})
 		return shouldRetry(err)
 	})
 	return err
@@ -720,9 +746,12 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Do the move
-	arg := files.RelocationArg{}
-	arg.FromPath = srcObj.remotePath()
-	arg.ToPath = dstObj.remotePath()
+	arg := files.RelocationArg{
+		RelocationPath: files.RelocationPath{
+			FromPath: f.opt.Enc.FromStandardPath(srcObj.remotePath()),
+			ToPath:   f.opt.Enc.FromStandardPath(dstObj.remotePath()),
+		},
+	}
 	var err error
 	var result *files.RelocationResult
 	err = f.pacer.Call(func() (bool, error) {
@@ -747,7 +776,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
-	absPath := "/" + path.Join(f.Root(), remote)
+	absPath := f.opt.Enc.FromStandardPath(path.Join(f.slashRoot, remote))
 	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
 	createArg := sharing.CreateSharedLinkWithSettingsArg{
 		Path: absPath,
@@ -758,7 +787,8 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 		return shouldRetry(err)
 	})
 
-	if err != nil && strings.Contains(err.Error(), sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
+	if err != nil && strings.Contains(err.Error(),
+		sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
 		fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
 		listArg := sharing.ListSharedLinksArg{
 			Path:       absPath,
@@ -820,9 +850,12 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	// ...apparently not necessary
 
 	// Do the move
-	arg := files.RelocationArg{}
-	arg.FromPath = srcPath
-	arg.ToPath = dstPath
+	arg := files.RelocationArg{
+		RelocationPath: files.RelocationPath{
+			FromPath: f.opt.Enc.FromStandardPath(srcPath),
+			ToPath:   f.opt.Enc.FromStandardPath(dstPath),
+		},
+	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.srv.MoveV2(&arg)
 		return shouldRetry(err)
@@ -863,7 +896,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.Dropbox)
+	return hash.Set(DbHashType)
 }
 
 // ------------------------------------------------------------
@@ -888,7 +921,7 @@ func (o *Object) Remote() string {
 
 // Hash returns the dropbox special hash
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t != hash.Dropbox {
+	if t != DbHashType {
 		return "", hash.ErrUnsupported
 	}
 	err := o.readMetaData()
@@ -977,7 +1010,10 @@ func (o *Object) Storable() bool {
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	fs.FixRangeOption(options, o.bytes)
 	headers := fs.OpenOptionHeaders(options)
-	arg := files.DownloadArg{Path: o.remotePath(), ExtraHeaders: headers}
+	arg := files.DownloadArg{
+		Path:         o.fs.opt.Enc.FromStandardPath(o.remotePath()),
+		ExtraHeaders: headers,
+	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, in, err = o.fs.srv.Download(&arg)
 		return shouldRetry(err)
@@ -986,7 +1022,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	switch e := err.(type) {
 	case files.DownloadAPIError:
 		// Don't attempt to retry copyright violation errors
-		if e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
+		if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorRestrictedContent {
 			return nil, fserrors.NoRetryError(err)
 		}
 	}
@@ -1087,6 +1123,13 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 			return false, nil
 		}
 		entry, err = o.fs.srv.UploadSessionFinish(args, chunk)
+		// If error is insufficient space then don't retry
+		if e, ok := err.(files.UploadSessionFinishAPIError); ok {
+			if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
+				err = fserrors.NoRetryError(err)
+				return false, err
+			}
+		}
 		// after the first chunk is uploaded, we retry everything
 		return err != nil, err
 	})
@@ -1104,10 +1147,9 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	remote := o.remotePath()
 	if ignoredFiles.MatchString(remote) {
-		fs.Logf(o, "File name disallowed - not uploading")
-		return nil
+		return fserrors.NoRetryError(errors.Errorf("file name %q is disallowed - not uploading", path.Base(remote)))
 	}
-	commitInfo := files.NewCommitInfo(o.remotePath())
+	commitInfo := files.NewCommitInfo(o.fs.opt.Enc.FromStandardPath(o.remotePath()))
 	commitInfo.Mode.Tag = "overwrite"
 	// The Dropbox API only accepts timestamps in UTC with second precision.
 	commitInfo.ClientModified = src.ModTime(ctx).UTC().Round(time.Second)
@@ -1132,7 +1174,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // Remove an object
 func (o *Object) Remove(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{Path: o.remotePath()})
+		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
+			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
+		})
 		return shouldRetry(err)
 	})
 	return err

backend/fichier/api.go

@@ -107,6 +107,10 @@ func (f *Fs) listFiles(ctx context.Context, directoryID int) (filesList *FilesLi
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list files")
 	}
+	for i := range filesList.Items {
+		item := &filesList.Items[i]
+		item.Filename = f.opt.Enc.ToStandardName(item.Filename)
+	}
 
 	return filesList, nil
 }
@@ -131,6 +135,11 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't list folders")
 	}
+	foldersList.Name = f.opt.Enc.ToStandardName(foldersList.Name)
+	for i := range foldersList.SubFolders {
+		folder := &foldersList.SubFolders[i]
+		folder.Name = f.opt.Enc.ToStandardName(folder.Name)
+	}
 
 	// fs.Debugf(f, "Got FoldersList for id `%s`", directoryID)
 
@@ -166,7 +175,6 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
 	entries = make([]fs.DirEntry, len(files.Items)+len(folders.SubFolders))
 
 	for i, item := range files.Items {
-		item.Filename = restoreReservedChars(item.Filename)
 		entries[i] = f.newObjectFromFile(ctx, dir, item)
 	}
 
@@ -176,7 +184,6 @@ func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, er
 			return nil, err
 		}
 
-		folder.Name = restoreReservedChars(folder.Name)
 		fullPath := getRemote(dir, folder.Name)
 		folderID := strconv.Itoa(folder.ID)
 
@@ -206,7 +213,7 @@ func getRemote(dir, fileName string) string {
 }
 
 func (f *Fs) makeFolder(ctx context.Context, leaf string, folderID int) (response *MakeFolderResponse, err error) {
-	name := replaceReservedChars(leaf)
+	name := f.opt.Enc.FromStandardName(leaf)
 	// fs.Debugf(f, "Creating folder `%s` in id `%s`", name, directoryID)
 
 	request := MakeFolderRequest{
@@ -316,7 +323,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) {
 	// fs.Debugf(f, "Uploading File `%s`", fileName)
 
-	fileName = replaceReservedChars(fileName)
+	fileName = f.opt.Enc.FromStandardName(fileName)
 
 	if len(uploadID) > 10 || !isAlphaNumeric(uploadID) {
 		return nil, errors.New("Invalid UploadID")

backend/fichier/fichier.go

@@ -11,11 +11,13 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
 )
@@ -35,25 +37,48 @@ func init() {
 		Config: func(name string, config configmap.Mapper) {
 		},
 		NewFs: NewFs,
-		Options: []fs.Option{
-			{
-				Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
-				Name: "api_key",
-			},
-			{
-				Help:     "If you want to download a shared folder, add this parameter",
-				Name:     "shared_folder",
-				Required: false,
-				Advanced: true,
-			},
-		},
+		Options: []fs.Option{{
+			Help: "Your API Key, get it from https://1fichier.com/console/params.pl",
+			Name: "api_key",
+		}, {
+			Help:     "If you want to download a shared folder, add this parameter",
+			Name:     "shared_folder",
+			Required: false,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Characters that need escaping
+			//
+			// 		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+			// 		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
+			// 		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
+			// 		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
+			// 		'\'': '＇', // FULLWIDTH APOSTROPHE
+			// 		'$':  '＄', // FULLWIDTH DOLLAR SIGN
+			// 		'`':  '｀', // FULLWIDTH GRAVE ACCENT
+			//
+			// Leading space and trailing space
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeSingleQuote |
+				encoder.EncodeBackQuote |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeLtGt |
+				encoder.EncodeDollar |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	APIKey       string `config:"api_key"`
-	SharedFolder string `config:"shared_folder"`
+	APIKey       string               `config:"api_key"`
+	SharedFolder string               `config:"shared_folder"`
+	Enc          encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs is the interface a cloud storage system must provide
@@ -61,9 +86,9 @@ type Fs struct {
 	root       string
 	name       string
 	features   *fs.Features
+	opt        Options
 	dirCache   *dircache.DirCache
 	baseClient *http.Client
-	options    *Options
 	pacer      *fs.Pacer
 	rest       *rest.Client
 }
@@ -141,8 +166,7 @@ func (f *Fs) Features() *fs.Features {
 //
 // On Windows avoid single character remote names as they can be mixed
 // up with drive letters.
-func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error) {
-	root := replaceReservedChars(rootleaf)
+func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) {
 	opt := new(Options)
 	err := configstruct.Set(config, opt)
 	if err != nil {
@@ -160,7 +184,7 @@ func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error)
 	f := &Fs{
 		name:       name,
 		root:       root,
-		options:    opt,
+		opt:        *opt,
 		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		baseClient: &http.Client{},
 	}
@@ -174,7 +198,7 @@ func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error)
 
 	f.rest = rest.NewClient(client).SetRoot(apiBaseURL)
 
-	f.rest.SetHeader("Authorization", "Bearer "+f.options.APIKey)
+	f.rest.SetHeader("Authorization", "Bearer "+f.opt.APIKey)
 
 	f.dirCache = dircache.New(root, rootID, f)
 
@@ -224,8 +248,8 @@ func NewFs(name string, rootleaf string, config configmap.Mapper) (fs.Fs, error)
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	if f.options.SharedFolder != "" {
-		return f.listSharedFiles(ctx, f.options.SharedFolder)
+	if f.opt.SharedFolder != "" {
+		return f.listSharedFiles(ctx, f.opt.SharedFolder)
 	}
 
 	dirContent, err := f.listDir(ctx, dir)
@@ -346,7 +370,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 			Date:        time.Now().Format("2006-01-02 15:04:05"),
 			Filename:    link.Filename,
 			Pass:        0,
-			Size:        int(fileSize),
+			Size:        fileSize,
 			URL:         link.Download,
 		},
 	}, nil

backend/fichier/object.go

@@ -43,7 +43,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 
 // Size returns the size of the file
 func (o *Object) Size() int64 {
-	return int64(o.file.Size)
+	return o.file.Size
 }
 
 // Fs returns read only access to the Fs that this object is part of
@@ -74,7 +74,7 @@ func (o *Object) SetModTime(context.Context, time.Time) error {
 
 // Open opens the file for read.  Call Close() on the returned io.ReadCloser
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
-	fs.FixRangeOption(options, int64(o.file.Size))
+	fs.FixRangeOption(options, o.file.Size)
 	downloadToken, err := o.fs.getDownloadToken(ctx, o.file.URL)
 
 	if err != nil {

backend/fichier/replace.go

@@ -1,71 +0,0 @@
-/*
-Translate file names for 1fichier
-
-1Fichier reserved characters
-
-The following characters are 1Fichier reserved characters, and can't
-be used in 1Fichier  folder and file names.
-
-*/
-
-package fichier
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// 1Fichier has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-		'\'': '＇', // FULLWIDTH APOSTROPHE
-		'$':  '＄', // FULLWIDTH DOLLAR SIGN
-		'`':  '｀', // FULLWIDTH GRAVE ACCENT
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	invCharMap           map[rune]rune
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// file names can't start with space either
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Replace reserved characters
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/fichier/replace_test.go

@@ -1,24 +0,0 @@
-package fichier
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{"\"'<>/\\$`", `＂＇＜＞/＼＄｀`},
-		{" leading space", "␠leading space"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/fichier/structs.go

@@ -69,7 +69,7 @@ type SharedFolderResponse []SharedFile
 type SharedFile struct {
 	Filename string `json:"filename"`
 	Link     string `json:"link"`
-	Size     int    `json:"size"`
+	Size     int64  `json:"size"`
 }
 
 // EndFileUploadResponse is the response structure of the corresponding request
@@ -93,7 +93,7 @@ type File struct {
 	Date        string `json:"date"`
 	Filename    string `json:"filename"`
 	Pass        int    `json:"pass"`
-	Size        int    `json:"size"`
+	Size        int64  `json:"size"`
 	URL         string `json:"url"`
 }
 

backend/ftp/ftp.go

@@ -14,10 +14,12 @@ import (
 	"github.com/jlaffaye/ftp"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 )
@@ -28,54 +30,70 @@ func init() {
 		Name:        "ftp",
 		Description: "FTP Connection",
 		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "host",
-				Help:     "FTP host to connect to",
-				Required: true,
-				Examples: []fs.OptionExample{{
-					Value: "ftp.example.com",
-					Help:  "Connect to ftp.example.com",
-				}},
-			}, {
-				Name: "user",
-				Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
-			}, {
-				Name: "port",
-				Help: "FTP port, leave blank to use default (21)",
-			}, {
-				Name:       "pass",
-				Help:       "FTP password",
-				IsPassword: true,
-				Required:   true,
-			}, {
-				Name:    "tls",
-				Help:    "Use FTP over TLS (Implicit)",
-				Default: false,
-			}, {
-				Name:     "concurrency",
-				Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
-				Default:  0,
-				Advanced: true,
-			}, {
-				Name:     "no_check_certificate",
-				Help:     "Do not verify the TLS certificate of the server",
-				Default:  false,
-				Advanced: true,
-			},
-		},
+		Options: []fs.Option{{
+			Name:     "host",
+			Help:     "FTP host to connect to",
+			Required: true,
+			Examples: []fs.OptionExample{{
+				Value: "ftp.example.com",
+				Help:  "Connect to ftp.example.com",
+			}},
+		}, {
+			Name: "user",
+			Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
+		}, {
+			Name: "port",
+			Help: "FTP port, leave blank to use default (21)",
+		}, {
+			Name:       "pass",
+			Help:       "FTP password",
+			IsPassword: true,
+			Required:   true,
+		}, {
+			Name:    "tls",
+			Help:    "Use FTP over TLS (Implicit)",
+			Default: false,
+		}, {
+			Name:     "concurrency",
+			Help:     "Maximum number of FTP simultaneous connections, 0 for unlimited",
+			Default:  0,
+			Advanced: true,
+		}, {
+			Name:     "no_check_certificate",
+			Help:     "Do not verify the TLS certificate of the server",
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     "disable_epsv",
+			Help:     "Disable using EPSV even if server advertises support",
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// The FTP protocal can't handle trailing spaces (for instance
+			// pureftpd turns them into _)
+			//
+			// proftpd can't handle '*' in file names
+			// pureftpd can't handle '[', ']' or '*'
+			Default: (encoder.Display |
+				encoder.EncodeRightSpace),
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Host              string `config:"host"`
-	User              string `config:"user"`
-	Pass              string `config:"pass"`
-	Port              string `config:"port"`
-	TLS               bool   `config:"tls"`
-	Concurrency       int    `config:"concurrency"`
-	SkipVerifyTLSCert bool   `config:"no_check_certificate"`
+	Host              string               `config:"host"`
+	User              string               `config:"user"`
+	Pass              string               `config:"pass"`
+	Port              string               `config:"port"`
+	TLS               bool                 `config:"tls"`
+	Concurrency       int                  `config:"concurrency"`
+	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
+	DisableEPSV       bool                 `config:"disable_epsv"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote FTP server
@@ -141,6 +159,9 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 		}
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
 	}
+	if f.opt.DisableEPSV {
+		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
+	}
 	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
 	if err != nil {
 		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
@@ -169,7 +190,11 @@ func (f *Fs) getFtpConnection() (c *ftp.ServerConn, err error) {
 	if c != nil {
 		return c, nil
 	}
-	return f.ftpConnection()
+	c, err = f.ftpConnection()
+	if err != nil && f.opt.Concurrency > 0 {
+		f.tokens.Put()
+	}
+	return c, err
 }
 
 // Return an FTP connection to the pool
@@ -182,7 +207,13 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	if f.opt.Concurrency > 0 {
 		defer f.tokens.Put()
 	}
+	if pc == nil {
+		return
+	}
 	c := *pc
+	if c == nil {
+		return
+	}
 	*pc = nil
 	if err != nil {
 		// If not a regular FTP error code then check the connection
@@ -295,10 +326,37 @@ func translateErrorDir(err error) error {
 	return err
 }
 
+// entryToStandard converts an incoming ftp.Entry to Standard encoding
+func (f *Fs) entryToStandard(entry *ftp.Entry) {
+	// Skip . and .. as we don't want these encoded
+	if entry.Name == "." || entry.Name == ".." {
+		return
+	}
+	entry.Name = f.opt.Enc.ToStandardName(entry.Name)
+	entry.Target = f.opt.Enc.ToStandardPath(entry.Target)
+}
+
+// dirFromStandardPath returns dir in encoded form.
+func (f *Fs) dirFromStandardPath(dir string) string {
+	// Skip . and .. as we don't want these encoded
+	if dir == "." || dir == ".." {
+		return dir
+	}
+	return f.opt.Enc.FromStandardPath(dir)
+}
+
 // findItem finds a directory entry for the name in its parent directory
 func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
 	fullPath := path.Join(f.root, remote)
+	if fullPath == "" || fullPath == "." || fullPath == "/" {
+		// if root, assume exists and synthesize an entry
+		return &ftp.Entry{
+			Name: "",
+			Type: ftp.EntryTypeFolder,
+			Time: time.Now(),
+		}, nil
+	}
 	dir := path.Dir(fullPath)
 	base := path.Base(fullPath)
 
@@ -306,12 +364,13 @@ func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "findItem")
 	}
-	files, err := c.List(dir)
+	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
 	for _, file := range files {
+		f.entryToStandard(file)
 		if file.Name == base {
 			return file, nil
 		}
@@ -366,7 +425,7 @@ func (f *Fs) dirExists(remote string) (exists bool, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	// defer fs.Trace(dir, "curlevel=%d", curlevel)("")
+	// defer log.Trace(dir, "dir=%q", dir)("entries=%v, err=%v", &entries, &err)
 	c, err := f.getFtpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "list")
@@ -378,7 +437,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	resultchan := make(chan []*ftp.Entry, 1)
 	errchan := make(chan error, 1)
 	go func() {
-		result, err := c.List(path.Join(f.root, dir))
+		result, err := c.List(f.dirFromStandardPath(path.Join(f.root, dir)))
 		f.putFtpConnection(&c, err)
 		if err != nil {
 			errchan <- err
@@ -415,6 +474,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	for i := range files {
 		object := files[i]
+		f.entryToStandard(object)
 		newremote := path.Join(dir, object.Name)
 		switch object.Type {
 		case ftp.EntryTypeFolder:
@@ -484,19 +544,21 @@ func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "getInfo")
 	}
-	files, err := c.List(dir)
+	files, err := c.List(f.dirFromStandardPath(dir))
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
 
 	for i := range files {
-		if files[i].Name == base {
+		file := files[i]
+		f.entryToStandard(file)
+		if file.Name == base {
 			info := &FileInfo{
 				Name:    remote,
-				Size:    files[i].Size,
-				ModTime: files[i].Time,
-				IsDir:   files[i].Type == ftp.EntryTypeFolder,
+				Size:    file.Size,
+				ModTime: file.Time,
+				IsDir:   file.Type == ftp.EntryTypeFolder,
 			}
 			return info, nil
 		}
@@ -506,6 +568,7 @@ func (f *Fs) getInfo(remote string) (fi *FileInfo, err error) {
 
 // mkdir makes the directory and parents using unrooted paths
 func (f *Fs) mkdir(abspath string) error {
+	abspath = path.Clean(abspath)
 	if abspath == "." || abspath == "/" {
 		return nil
 	}
@@ -527,7 +590,7 @@ func (f *Fs) mkdir(abspath string) error {
 	if connErr != nil {
 		return errors.Wrap(connErr, "mkdir")
 	}
-	err = c.MakeDir(abspath)
+	err = c.MakeDir(f.dirFromStandardPath(abspath))
 	f.putFtpConnection(&c, err)
 	switch errX := err.(type) {
 	case *textproto.Error:
@@ -563,7 +626,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	if err != nil {
 		return errors.Wrap(translateErrorFile(err), "Rmdir")
 	}
-	err = c.RemoveDir(path.Join(f.root, dir))
+	err = c.RemoveDir(f.dirFromStandardPath(path.Join(f.root, dir)))
 	f.putFtpConnection(&c, err)
 	return translateErrorDir(err)
 }
@@ -584,8 +647,8 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, errors.Wrap(err, "Move")
 	}
 	err = c.Rename(
-		path.Join(srcObj.fs.root, srcObj.remote),
-		path.Join(f.root, remote),
+		f.opt.Enc.FromStandardPath(path.Join(srcObj.fs.root, srcObj.remote)),
+		f.opt.Enc.FromStandardPath(path.Join(f.root, remote)),
 	)
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -638,8 +701,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return errors.Wrap(err, "DirMove")
 	}
 	err = c.Rename(
-		srcPath,
-		dstPath,
+		f.dirFromStandardPath(srcPath),
+		f.dirFromStandardPath(dstPath),
 	)
 	f.putFtpConnection(&c, err)
 	if err != nil {
@@ -725,11 +788,13 @@ func (f *ftpReadCloser) Close() error {
 	case <-timer.C:
 		// if timer fired assume no error but connection dead
 		fs.Errorf(f.f, "Timeout when waiting for connection Close")
+		f.f.putFtpConnection(nil, nil)
 		return nil
 	}
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()
+		f.f.putFtpConnection(nil, nil)
 	} else {
 		f.f.putFtpConnection(&f.c, nil)
 	}
@@ -765,7 +830,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (rc io.Read
 	if err != nil {
 		return nil, errors.Wrap(err, "open")
 	}
-	fd, err := c.RetrFrom(path, uint64(offset))
+	fd, err := c.RetrFrom(o.fs.opt.Enc.FromStandardPath(path), uint64(offset))
 	if err != nil {
 		o.fs.putFtpConnection(&c, err)
 		return nil, errors.Wrap(err, "open")
@@ -800,10 +865,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return errors.Wrap(err, "Update")
 	}
-	err = c.Stor(path, in)
+	err = c.Stor(o.fs.opt.Enc.FromStandardPath(path), in)
 	if err != nil {
 		_ = c.Quit() // toss this connection to avoid sync errors
 		remove()
+		o.fs.putFtpConnection(nil, err)
 		return errors.Wrap(err, "update stor")
 	}
 	o.fs.putFtpConnection(&c, nil)
@@ -830,7 +896,7 @@ func (o *Object) Remove(ctx context.Context) (err error) {
 		if err != nil {
 			return errors.Wrap(err, "Remove")
 		}
-		err = c.Delete(path)
+		err = c.Delete(o.fs.opt.Enc.FromStandardPath(path))
 		o.fs.putFtpConnection(&c, err)
 	}
 	return err

backend/ftp/ftp_test.go

@@ -5,13 +5,44 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/ftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestFTP:",
+		RemoteName: "TestFTPProftpd:",
 		NilObject:  (*ftp.Object)(nil),
 	})
 }
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPRclone:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTPPureftpd:",
+		NilObject:  (*ftp.Object)(nil),
+	})
+}
+
+// func TestIntegration4(t *testing.T) {
+// 	if *fstest.RemoteName != "" {
+// 		t.Skip("skipping as -remote is set")
+// 	}
+// 	fstests.Run(t, &fstests.Opt{
+// 		RemoteName: "TestFTPVsftpd:",
+// 		NilObject:  (*ftp.Object)(nil),
+// 	})
+// }

backend/googlecloudstorage/googlecloudstorage.go

@@ -37,6 +37,7 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"golang.org/x/oauth2"
@@ -245,20 +246,28 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 				Value: "DURABLE_REDUCED_AVAILABILITY",
 				Help:  "Durable reduced availability storage class",
 			}},
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.Base |
+				encoder.EncodeCrLf |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ProjectNumber             string `config:"project_number"`
-	ServiceAccountFile        string `config:"service_account_file"`
-	ServiceAccountCredentials string `config:"service_account_credentials"`
-	ObjectACL                 string `config:"object_acl"`
-	BucketACL                 string `config:"bucket_acl"`
-	BucketPolicyOnly          bool   `config:"bucket_policy_only"`
-	Location                  string `config:"location"`
-	StorageClass              string `config:"storage_class"`
+	ProjectNumber             string               `config:"project_number"`
+	ServiceAccountFile        string               `config:"service_account_file"`
+	ServiceAccountCredentials string               `config:"service_account_credentials"`
+	ObjectACL                 string               `config:"object_acl"`
+	BucketACL                 string               `config:"bucket_acl"`
+	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`
+	Location                  string               `config:"location"`
+	StorageClass              string               `config:"storage_class"`
+	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote storage server
@@ -349,7 +358,8 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
 // split returns bucket and bucketPath from the object
@@ -438,8 +448,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	if f.rootBucket != "" && f.rootDirectory != "" {
 		// Check to see if the object exists
+		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
-			_, err = f.svc.Objects.Get(f.rootBucket, f.rootDirectory).Context(ctx).Do()
+			_, err = f.svc.Objects.Get(f.rootBucket, encodedDirectory).Context(ctx).Do()
 			return shouldRetry(err)
 		})
 		if err == nil {
@@ -522,6 +533,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				if !strings.HasSuffix(remote, "/") {
 					continue
 				}
+				remote = f.opt.Enc.ToStandardPath(remote)
 				if !strings.HasPrefix(remote, prefix) {
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
@@ -537,11 +549,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			}
 		}
 		for _, object := range objects.Items {
-			if !strings.HasPrefix(object.Name, prefix) {
+			remote := f.opt.Enc.ToStandardPath(object.Name)
+			if !strings.HasPrefix(remote, prefix) {
 				fs.Logf(f, "Odd name received %q", object.Name)
 				continue
 			}
-			remote := object.Name[len(prefix):]
+			remote = remote[len(prefix):]
 			isDirectory := strings.HasSuffix(remote, "/")
 			if addBucket {
 				remote = path.Join(bucket, remote)
@@ -613,7 +626,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 			return nil, err
 		}
 		for _, bucket := range buckets.Items {
-			d := fs.NewDir(bucket.Name, time.Time{})
+			d := fs.NewDir(f.opt.Enc.ToStandardName(bucket.Name), time.Time{})
 			entries = append(entries, d)
 		}
 		if buckets.NextPageToken == "" {

backend/googlephotos/googlephotos_test.go

@@ -12,6 +12,7 @@ import (
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
@@ -26,17 +27,6 @@ const (
 	fileNameUpload = "rclone-test-image2.jpg"
 )
 
-// Wrapper to override the remote for an object
-type overrideRemoteObject struct {
-	fs.Object
-	remote string
-}
-
-// Remote returns the overridden remote name
-func (o *overrideRemoteObject) Remote() string {
-	return o.remote
-}
-
 func TestIntegration(t *testing.T) {
 	ctx := context.Background()
 	fstest.Initialise()
@@ -66,7 +56,7 @@ func TestIntegration(t *testing.T) {
 			require.NoError(t, err)
 			in, err := srcObj.Open(ctx)
 			require.NoError(t, err)
-			dstObj, err := f.Put(ctx, in, &overrideRemoteObject{srcObj, remote})
+			dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 			require.NoError(t, err)
 			assert.Equal(t, remote, dstObj.Remote())
 			_ = in.Close()
@@ -231,7 +221,7 @@ func TestIntegration(t *testing.T) {
 		require.NoError(t, err)
 		in, err := srcObj.Open(ctx)
 		require.NoError(t, err)
-		dstObj, err := f.Put(ctx, in, &overrideRemoteObject{srcObj, remote})
+		dstObj, err := f.Put(ctx, in, operations.NewOverrideRemote(srcObj, remote))
 		require.NoError(t, err)
 		assert.Equal(t, remote, dstObj.Remote())
 		_ = in.Close()

backend/jottacloud/api/types.go

@@ -46,13 +46,57 @@ func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 // APIString returns Time string in Jottacloud API format
 func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
 
+// LoginToken is struct representing the login token generated in the WebUI
+type LoginToken struct {
+	Username      string `json:"username"`
+	Realm         string `json:"realm"`
+	WellKnownLink string `json:"well_known_link"`
+	AuthToken     string `json:"auth_token"`
+}
+
+// WellKnown contains some configuration parameters for setting up endpoints
+type WellKnown struct {
+	Issuer                                     string   `json:"issuer"`
+	AuthorizationEndpoint                      string   `json:"authorization_endpoint"`
+	TokenEndpoint                              string   `json:"token_endpoint"`
+	TokenIntrospectionEndpoint                 string   `json:"token_introspection_endpoint"`
+	UserinfoEndpoint                           string   `json:"userinfo_endpoint"`
+	EndSessionEndpoint                         string   `json:"end_session_endpoint"`
+	JwksURI                                    string   `json:"jwks_uri"`
+	CheckSessionIframe                         string   `json:"check_session_iframe"`
+	GrantTypesSupported                        []string `json:"grant_types_supported"`
+	ResponseTypesSupported                     []string `json:"response_types_supported"`
+	SubjectTypesSupported                      []string `json:"subject_types_supported"`
+	IDTokenSigningAlgValuesSupported           []string `json:"id_token_signing_alg_values_supported"`
+	UserinfoSigningAlgValuesSupported          []string `json:"userinfo_signing_alg_values_supported"`
+	RequestObjectSigningAlgValuesSupported     []string `json:"request_object_signing_alg_values_supported"`
+	ResponseNodesSupported                     []string `json:"response_modes_supported"`
+	RegistrationEndpoint                       string   `json:"registration_endpoint"`
+	TokenEndpointAuthMethodsSupported          []string `json:"token_endpoint_auth_methods_supported"`
+	TokenEndpointAuthSigningAlgValuesSupported []string `json:"token_endpoint_auth_signing_alg_values_supported"`
+	ClaimsSupported                            []string `json:"claims_supported"`
+	ClaimTypesSupported                        []string `json:"claim_types_supported"`
+	ClaimsParameterSupported                   bool     `json:"claims_parameter_supported"`
+	ScopesSupported                            []string `json:"scopes_supported"`
+	RequestParameterSupported                  bool     `json:"request_parameter_supported"`
+	RequestURIParameterSupported               bool     `json:"request_uri_parameter_supported"`
+	CodeChallengeMethodsSupported              []string `json:"code_challenge_methods_supported"`
+	TLSClientCertificateBoundAccessTokens      bool     `json:"tls_client_certificate_bound_access_tokens"`
+	IntrospectionEndpoint                      string   `json:"introspection_endpoint"`
+}
+
 // TokenJSON is the struct representing the HTTP response from OAuth2
 // providers returning a token in JSON form.
 type TokenJSON struct {
-	AccessToken  string `json:"access_token"`
-	TokenType    string `json:"token_type"`
-	RefreshToken string `json:"refresh_token"`
-	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
+	AccessToken      string `json:"access_token"`
+	ExpiresIn        int32  `json:"expires_in"` // at least PayPal returns string, while most return number
+	RefreshExpiresIn int32  `json:"refresh_expires_in"`
+	RefreshToken     string `json:"refresh_token"`
+	TokenType        string `json:"token_type"`
+	IDToken          string `json:"id_token"`
+	NotBeforePolicy  int32  `json:"not-before-policy"`
+	SessionState     string `json:"session_state"`
+	Scope            string `json:"scope"`
 }
 
 // JSON structures returned by new API

backend/jottacloud/jottacloud.go

@@ -4,12 +4,13 @@ import (
 	"bytes"
 	"context"
 	"crypto/md5"
+	"encoding/base64"
 	"encoding/hex"
+	"encoding/json"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -25,11 +26,11 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
-	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
@@ -38,32 +39,29 @@ import (
 
 // Globals
 const (
-	minSleep                    = 10 * time.Millisecond
-	maxSleep                    = 2 * time.Second
-	decayConstant               = 2 // bigger for slower decay, exponential
-	defaultDevice               = "Jotta"
-	defaultMountpoint           = "Archive"
-	rootURL                     = "https://www.jottacloud.com/jfs/"
-	apiURL                      = "https://api.jottacloud.com/"
-	baseURL                     = "https://www.jottacloud.com/"
-	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
-	registerURL                 = "https://api.jottacloud.com/auth/v1/register"
-	cachePrefix                 = "rclone-jcmd5-"
-	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
-	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
-	configClientID              = "client_id"
-	configClientSecret          = "client_secret"
-	configDevice                = "device"
-	configMountpoint            = "mountpoint"
-	charset                     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	minSleep          = 10 * time.Millisecond
+	maxSleep          = 2 * time.Second
+	decayConstant     = 2 // bigger for slower decay, exponential
+	defaultDevice     = "Jotta"
+	defaultMountpoint = "Archive"
+	rootURL           = "https://www.jottacloud.com/jfs/"
+	apiURL            = "https://api.jottacloud.com/"
+	baseURL           = "https://www.jottacloud.com/"
+	defaultTokenURL   = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
+	cachePrefix       = "rclone-jcmd5-"
+	configDevice      = "device"
+	configMountpoint  = "mountpoint"
+	configTokenURL    = "tokenURL"
+	configVersion     = 1
 )
 
 var (
 	// Description of how to auth for this app for a personal account
 	oauthConfig = &oauth2.Config{
+		ClientID: "jottacli",
 		Endpoint: oauth2.Endpoint{
-			AuthURL:  tokenURL,
-			TokenURL: tokenURL,
+			AuthURL:  defaultTokenURL,
+			TokenURL: defaultTokenURL,
 		},
 		RedirectURL: oauthutil.RedirectLocalhostURL,
 	}
@@ -78,43 +76,37 @@ func init() {
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
 			ctx := context.TODO()
-			tokenString, ok := m.Get("token")
-			if ok && tokenString != "" {
-				fmt.Printf("Already have a token - refresh?\n")
-				if !config.Confirm() {
-					return
-				}
-			}
 
-			srv := rest.NewClient(fshttp.NewClient(fs.Config))
-			fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
-			if config.Confirm() {
-				deviceRegistration, err := registerDevice(ctx, srv)
+			refresh := false
+			if version, ok := m.Get("configVersion"); ok {
+				ver, err := strconv.Atoi(version)
 				if err != nil {
-					log.Fatalf("Failed to register device: %v", err)
+					log.Fatalf("Failed to parse config version - corrupted config")
 				}
-
-				m.Set(configClientID, deviceRegistration.ClientID)
-				m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
-				fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+				refresh = ver != configVersion
 			}
 
-			clientID, ok := m.Get(configClientID)
-			if !ok {
-				clientID = rcloneClientID
+			if refresh {
+				fmt.Printf("Config outdated - refreshing\n")
+			} else {
+				tokenString, ok := m.Get("token")
+				if ok && tokenString != "" {
+					fmt.Printf("Already have a token - refresh?\n")
+					if !config.Confirm(false) {
+						return
+					}
+				}
 			}
-			clientSecret, ok := m.Get(configClientSecret)
-			if !ok {
-				clientSecret = rcloneEncryptedClientSecret
-			}
-			oauthConfig.ClientID = clientID
-			oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
-			fmt.Printf("Username> ")
-			username := config.ReadLine()
-			password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
+			clientConfig := *fs.Config
+			clientConfig.UserAgent = "JottaCli 0.6.18626 windows-amd64"
+			srv := rest.NewClient(fshttp.NewClient(&clientConfig))
 
-			token, err := doAuth(ctx, srv, username, password)
+			fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
+			fmt.Printf("Login Token> ")
+			loginToken := config.ReadLine()
+
+			token, err := doAuth(ctx, srv, loginToken, m)
 			if err != nil {
 				log.Fatalf("Failed to get oauth token: %s", err)
 			}
@@ -124,7 +116,7 @@ func init() {
 			}
 
 			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
-			if config.Confirm() {
+			if config.Confirm(false) {
 				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
 				if err != nil {
 					log.Fatalf("Failed to load oAuthClient: %s", err)
@@ -140,6 +132,8 @@ func init() {
 				m.Set(configDevice, device)
 				m.Set(configMountpoint, mountpoint)
 			}
+
+			m.Set("configVersion", strconv.Itoa(configVersion))
 		},
 		Options: []fs.Option{{
 			Name:     "md5_memory_limit",
@@ -161,18 +155,29 @@ func init() {
 			Help:     "Files bigger than this can be resumed if the upload fail's.",
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as xml doesn't handle them properly.
+			//
+			// Also: '*', '/', ':', '<', '>', '?', '\"', '\x00', '|'
+			Default: (encoder.Display |
+				encoder.EncodeWin | // :?"*<>|
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Device             string        `config:"device"`
-	Mountpoint         string        `config:"mountpoint"`
-	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
-	HardDelete         bool          `config:"hard_delete"`
-	Unlink             bool          `config:"unlink"`
-	UploadThreshold    fs.SizeSuffix `config:"upload_resume_limit"`
+	Device             string               `config:"device"`
+	Mountpoint         string               `config:"mountpoint"`
+	MD5MemoryThreshold fs.SizeSuffix        `config:"md5_memory_limit"`
+	HardDelete         bool                 `config:"hard_delete"`
+	Unlink             bool                 `config:"unlink"`
+	UploadThreshold    fs.SizeSuffix        `config:"upload_resume_limit"`
+	Enc                encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote jottacloud
@@ -246,67 +251,57 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// registerDevice register a new device for use with the jottacloud API
-func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
-	// random generator to generate random device names
-	seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
-	randonDeviceNamePartLength := 21
-	randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
-	for i := range randomDeviceNamePart {
-		randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
-	}
-	randomDeviceName := "rclone-" + string(randomDeviceNamePart)
-	fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
-
-	values := url.Values{}
-	values.Set("device_id", randomDeviceName)
-
-	opts := rest.Opts{
-		Method:       "POST",
-		RootURL:      registerURL,
-		ContentType:  "application/x-www-form-urlencoded",
-		ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
-		Parameters:   values,
-	}
-
-	var deviceRegistration *api.DeviceRegistrationResponse
-	_, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
-	return deviceRegistration, err
-}
-
 // doAuth runs the actual token request
-func doAuth(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
+func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
+	loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
+	if err != nil {
+		return token, err
+	}
+
+	// decode login token
+	var loginToken api.LoginToken
+	decoder := json.NewDecoder(bytes.NewReader(loginTokenBytes))
+	err = decoder.Decode(&loginToken)
+	if err != nil {
+		return token, err
+	}
+
+	// retrieve endpoint urls
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: loginToken.WellKnownLink,
+	}
+	var wellKnown api.WellKnown
+	_, err = srv.CallJSON(ctx, &opts, nil, &wellKnown)
+	if err != nil {
+		return token, err
+	}
+
+	// save the tokenurl
+	oauthConfig.Endpoint.AuthURL = wellKnown.TokenEndpoint
+	oauthConfig.Endpoint.TokenURL = wellKnown.TokenEndpoint
+	m.Set(configTokenURL, wellKnown.TokenEndpoint)
+
 	// prepare out token request with username and password
 	values := url.Values{}
-	values.Set("grant_type", "PASSWORD")
-	values.Set("password", password)
-	values.Set("username", username)
-	values.Set("client_id", oauthConfig.ClientID)
-	values.Set("client_secret", oauthConfig.ClientSecret)
-	opts := rest.Opts{
+	values.Set("client_id", "jottacli")
+	values.Set("grant_type", "password")
+	values.Set("password", loginToken.AuthToken)
+	values.Set("scope", "offline_access+openid")
+	values.Set("username", loginToken.Username)
+	values.Encode()
+	opts = rest.Opts{
 		Method:      "POST",
 		RootURL:     oauthConfig.Endpoint.AuthURL,
 		ContentType: "application/x-www-form-urlencoded",
-		Parameters:  values,
+		Body:        strings.NewReader(values.Encode()),
 	}
 
 	// do the first request
 	var jsonToken api.TokenJSON
-	resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
+	_, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
 	if err != nil {
-		// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
-		if resp != nil {
-			if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
-				fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
-				fmt.Printf("Enter verification code> ")
-				authCode := config.ReadLine()
-
-				authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
-				opts.ExtraHeaders = make(map[string]string)
-				opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-				resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
-			}
-		}
+		return token, err
 	}
 
 	token.AccessToken = jsonToken.AccessToken
@@ -460,7 +455,7 @@ func urlPathEscape(in string) string {
 
 // filePathRaw returns an unescaped file path (f.root, file)
 func (f *Fs) filePathRaw(file string) string {
-	return path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, file)))
+	return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
 // filePath returns a escaped file path (f.root, file)
@@ -468,29 +463,6 @@ func (f *Fs) filePath(file string) string {
 	return urlPathEscape(f.filePathRaw(file))
 }
 
-// Jottacloud requires the grant_type 'refresh_token' string
-// to be uppercase and throws a 400 Bad Request if we use the
-// lower case used by the oauth2 module
-//
-// This filter catches all refresh requests, reads the body,
-// changes the case and then sends it on
-func grantTypeFilter(req *http.Request) {
-	if tokenURL == req.URL.String() {
-		// read the entire body
-		refreshBody, err := ioutil.ReadAll(req.Body)
-		if err != nil {
-			return
-		}
-		_ = req.Body.Close()
-
-		// make the refresh token upper case
-		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
-
-		// set the new ReadCloser (with a dummy Close())
-		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
-	}
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	ctx := context.TODO()
@@ -501,35 +473,37 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 
-	rootIsDir := strings.HasSuffix(root, "/")
-	root = parsePath(root)
-
-	clientID, ok := m.Get(configClientID)
-	if !ok {
-		clientID = rcloneClientID
+	// Check config version
+	var ok bool
+	var version string
+	if version, ok = m.Get("configVersion"); ok {
+		ver, err := strconv.Atoi(version)
+		if err != nil {
+			return nil, errors.New("Failed to parse config version")
+		}
+		ok = ver == configVersion
 	}
-	clientSecret, ok := m.Get(configClientSecret)
 	if !ok {
-		clientSecret = rcloneEncryptedClientSecret
+		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
-	oauthConfig.ClientID = clientID
-	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
 
-	// the oauth client for the api servers needs
-	// a filter to fix the grant_type issues (see above)
+	// if custome endpoints are set use them else stick with defaults
+	if tokenURL, ok := m.Get(configTokenURL); ok {
+		oauthConfig.Endpoint.TokenURL = tokenURL
+		// jottacloud is weird. we need to use the tokenURL as authURL
+		oauthConfig.Endpoint.AuthURL = tokenURL
+	}
+
+	// Create OAuth Client
 	baseClient := fshttp.NewClient(fs.Config)
-	if do, ok := baseClient.Transport.(interface {
-		SetRequestFilter(f func(req *http.Request))
-	}); ok {
-		do.SetRequestFilter(grantTypeFilter)
-	} else {
-		fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
-	}
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
 	}
 
+	rootIsDir := strings.HasSuffix(root, "/")
+	root = parsePath(root)
+
 	f := &Fs{
 		name:   name,
 		root:   root,
@@ -673,7 +647,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		if item.Deleted {
 			continue
 		}
-		remote := path.Join(dir, restoreReservedChars(item.Name))
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
 		d := fs.NewDir(remote, time.Time(item.ModifiedAt))
 		entries = append(entries, d)
 	}
@@ -683,7 +657,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		if item.Deleted || item.State != "COMPLETED" {
 			continue
 		}
-		remote := path.Join(dir, restoreReservedChars(item.Name))
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
 		o, err := f.newObjectWithInfo(ctx, remote, item)
 		if err != nil {
 			continue
@@ -708,7 +682,7 @@ func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolde
 		if folder.Deleted {
 			return nil
 		}
-		folderPath := restoreReservedChars(path.Join(folder.Path, folder.Name))
+		folderPath := f.opt.Enc.ToStandardPath(path.Join(folder.Path, folder.Name))
 		folderPathLength := len(folderPath)
 		var remoteDir string
 		if folderPathLength > pathPrefixLength {
@@ -726,7 +700,7 @@ func (f *Fs) listFileDir(ctx context.Context, remoteStartPath string, startFolde
 			if file.Deleted || file.State != "COMPLETED" {
 				continue
 			}
-			remoteFile := path.Join(remoteDir, restoreReservedChars(file.Name))
+			remoteFile := path.Join(remoteDir, f.opt.Enc.ToStandardName(file.Name))
 			o, err := f.newObjectWithInfo(ctx, remoteFile, file)
 			if err != nil {
 				return err
@@ -897,7 +871,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, replaceReservedChars(path.Join(f.root, dest))))
+	opts.Parameters.Set(method, "/"+path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, dest))))
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -1004,7 +978,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorDirExists
 	}
 
-	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, replaceReservedChars(srcPath))+"/", dstRemote)
+	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
 	if err != nil {
 		return errors.Wrap(err, "couldn't move directory")
@@ -1295,7 +1269,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Created:  fileDate,
 		Modified: fileDate,
 		Md5:      md5String,
-		Path:     path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
+		Path:     path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
 	}
 
 	// send it

backend/jottacloud/replace.go

@@ -1,77 +0,0 @@
-/*
-Translate file names for JottaCloud adapted from OneDrive
-
-
-The following characters are JottaCloud reserved characters, and can't
-be used in JottaCloud folder and file names.
-
-  jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"
-
-
-*/
-
-package jottacloud
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// Onedrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		'*':  '＊', // FULLWIDTH ASTERISK
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'?':  '？', // FULLWIDTH QUESTION MARK
-		':':  '：', // FULLWIDTH COLON
-		';':  '；', // FULLWIDTH SEMICOLON
-		'|':  '｜', // FULLWIDTH VERTICAL LINE
-		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	invCharMap           map[rune]rune
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-	fixEndingWithSpace   = regexp.MustCompile(` (/|$)`)
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// Filenames can't start with space
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Filenames can't end with space
-	in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/jottacloud/replace_test.go

@@ -1,28 +0,0 @@
-package jottacloud
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{`\*<>?:;|"`, `＼＊＜＞？：；｜＂`},
-		{`\*<>?:;|"\*<>?:;|"`, `＼＊＜＞？：；｜＂＼＊＜＞？：；｜＂`},
-		{" leading space", "␠leading space"},
-		{"trailing space ", "trailing space␠"},
-		{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
-		{"trailing space /trailing space /trailing space ", "trailing space␠/trailing space␠/trailing space␠"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/koofr/koofr.go

@@ -12,10 +12,13 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 
 	httpclient "github.com/koofr/go-httpclient"
 	koofrclient "github.com/koofr/go-koofrclient"
@@ -27,46 +30,53 @@ func init() {
 		Name:        "koofr",
 		Description: "Koofr",
 		NewFs:       NewFs,
-		Options: []fs.Option{
-			{
-				Name:     "endpoint",
-				Help:     "The Koofr API endpoint to use",
-				Default:  "https://app.koofr.net",
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "mountid",
-				Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
-				Required: false,
-				Default:  "",
-				Advanced: true,
-			}, {
-				Name:     "setmtime",
-				Help:     "Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend.",
-				Default:  true,
-				Required: true,
-				Advanced: true,
-			}, {
-				Name:     "user",
-				Help:     "Your Koofr user name",
-				Required: true,
-			}, {
-				Name:       "password",
-				Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
-				IsPassword: true,
-				Required:   true,
-			},
-		},
+		Options: []fs.Option{{
+			Name:     "endpoint",
+			Help:     "The Koofr API endpoint to use",
+			Default:  "https://app.koofr.net",
+			Required: true,
+			Advanced: true,
+		}, {
+			Name:     "mountid",
+			Help:     "Mount ID of the mount to use. If omitted, the primary mount is used.",
+			Required: false,
+			Default:  "",
+			Advanced: true,
+		}, {
+			Name:     "setmtime",
+			Help:     "Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend.",
+			Default:  true,
+			Required: true,
+			Advanced: true,
+		}, {
+			Name:     "user",
+			Help:     "Your Koofr user name",
+			Required: true,
+		}, {
+			Name:       "password",
+			Help:       "Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)",
+			IsPassword: true,
+			Required:   true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
+		}},
 	})
 }
 
 // Options represent the configuration of the Koofr backend
 type Options struct {
-	Endpoint string `config:"endpoint"`
-	MountID  string `config:"mountid"`
-	User     string `config:"user"`
-	Password string `config:"password"`
-	SetMTime bool   `config:"setmtime"`
+	Endpoint string               `config:"endpoint"`
+	MountID  string               `config:"mountid"`
+	User     string               `config:"user"`
+	Password string               `config:"password"`
+	SetMTime bool                 `config:"setmtime"`
+	Enc      encoder.MultiEncoder `config:"encoding"`
 }
 
 // A Fs is a representation of a remote Koofr Fs
@@ -242,7 +252,7 @@ func (f *Fs) Hashes() hash.Set {
 
 // fullPath constructs a full, absolute path from a Fs root relative path,
 func (f *Fs) fullPath(part string) string {
-	return path.Join("/", f.root, part)
+	return f.opt.Enc.FromStandardPath(path.Join("/", f.root, part))
 }
 
 // NewFs constructs a new filesystem given a root path and configuration options
@@ -256,7 +266,9 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	if err != nil {
 		return nil, err
 	}
-	client := koofrclient.NewKoofrClient(opt.Endpoint, false)
+	httpClient := httpclient.New()
+	httpClient.Client = fshttp.NewClient(fs.Config)
+	client := koofrclient.NewKoofrClientWithHTTPClient(opt.Endpoint, httpClient)
 	basicAuth := fmt.Sprintf("Basic %s",
 		base64.StdEncoding.EncodeToString([]byte(opt.User+":"+pass)))
 	client.HTTPClient.Headers.Set("Authorization", basicAuth)
@@ -293,7 +305,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 		}
 		return nil, errors.New("Failed to find mount " + opt.MountID)
 	}
-	rootFile, err := f.client.FilesInfo(f.mountID, "/"+f.root)
+	rootFile, err := f.client.FilesInfo(f.mountID, f.opt.Enc.FromStandardPath("/"+f.root))
 	if err == nil && rootFile.Type != "dir" {
 		f.root = dir(f.root)
 		err = fs.ErrorIsFile
@@ -311,13 +323,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	entries = make([]fs.DirEntry, len(files))
 	for i, file := range files {
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(file.Name))
 		if file.Type == "dir" {
-			entries[i] = fs.NewDir(path.Join(dir, file.Name), time.Unix(0, 0))
+			entries[i] = fs.NewDir(remote, time.Unix(0, 0))
 		} else {
 			entries[i] = &Object{
 				fs:     f,
 				info:   file,
-				remote: path.Join(dir, file.Name),
+				remote: remote,
 			}
 		}
 	}

backend/local/encode_darwin.go

@@ -0,0 +1,11 @@
+//+build darwin
+
+package local
+
+import "github.com/rclone/rclone/lib/encoder"
+
+// This is the encoding used by the local backend for macOS
+//
+// macOS can't store invalid UTF-8, it converts them into %XX encoding
+const defaultEnc = (encoder.Base |
+	encoder.EncodeInvalidUtf8)

backend/local/encode_other.go

@@ -0,0 +1,8 @@
+//+build !windows,!darwin
+
+package local
+
+import "github.com/rclone/rclone/lib/encoder"
+
+// This is the encoding used by the local backend for non windows platforms
+const defaultEnc = encoder.Base

backend/local/encode_windows.go

@@ -0,0 +1,33 @@
+//+build windows
+
+package local
+
+import "github.com/rclone/rclone/lib/encoder"
+
+// This is the encoding used by the local backend for windows platforms
+//
+// List of replaced characters:
+//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+//   : (colon)         -> '：' // FULLWIDTH COLON
+//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+//
+// Additionally names can't end with a period (.) or space ( ).
+// List of replaced characters:
+//   . (period)        -> '．' // FULLWIDTH FULL STOP
+//     (space)         -> '␠' // SYMBOL FOR SPACE
+//
+// Also encode invalid UTF-8 bytes as Go can't convert them to UTF-16.
+//
+// https://docs.microsoft.com/de-de/windows/desktop/FileIO/naming-a-file#naming-conventions
+const defaultEnc = (encoder.Base |
+	encoder.EncodeWin |
+	encoder.EncodeBackSlash |
+	encoder.EncodeCtl |
+	encoder.EncodeRightSpace |
+	encoder.EncodeRightPeriod |
+	encoder.EncodeInvalidUtf8)

backend/local/local.go

@@ -20,10 +20,12 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/file"
 	"github.com/rclone/rclone/lib/readers"
 )
@@ -115,6 +117,11 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default:  defaultEnc,
 		}},
 	}
 	fs.Register(fsi)
@@ -122,15 +129,16 @@ to override the default choice.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks    bool `config:"copy_links"`
-	TranslateSymlinks bool `config:"links"`
-	SkipSymlinks      bool `config:"skip_links"`
-	NoUTFNorm         bool `config:"no_unicode_normalization"`
-	NoCheckUpdated    bool `config:"no_check_updated"`
-	NoUNC             bool `config:"nounc"`
-	OneFileSystem     bool `config:"one_file_system"`
-	CaseSensitive     bool `config:"case_sensitive"`
-	CaseInsensitive   bool `config:"case_insensitive"`
+	FollowSymlinks    bool                 `config:"copy_links"`
+	TranslateSymlinks bool                 `config:"links"`
+	SkipSymlinks      bool                 `config:"skip_links"`
+	NoUTFNorm         bool                 `config:"no_unicode_normalization"`
+	NoCheckUpdated    bool                 `config:"no_check_updated"`
+	NoUNC             bool                 `config:"nounc"`
+	OneFileSystem     bool                 `config:"one_file_system"`
+	CaseSensitive     bool                 `config:"case_sensitive"`
+	CaseInsensitive   bool                 `config:"case_insensitive"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -142,19 +150,19 @@ type Fs struct {
 	dev         uint64              // device number of root node
 	precisionOk sync.Once           // Whether we need to read the precision
 	precision   time.Duration       // precision of local filesystem
-	wmu         sync.Mutex          // used for locking access to 'warned'.
+	warnedMu    sync.Mutex          // used for locking access to 'warned'.
 	warned      map[string]struct{} // whether we have warned about this string
+
 	// do os.Lstat or os.Stat
 	lstat          func(name string) (os.FileInfo, error)
-	dirNames       *mapper    // directory name mapping
 	objectHashesMu sync.Mutex // global lock for Object.hashes
 }
 
 // Object represents a local filesystem object
 type Object struct {
 	fs             *Fs    // The Fs this object is part of
-	remote         string // The remote path - properly UTF-8 encoded - for rclone
-	path           string // The local path - may not be properly UTF-8 encoded - for OS
+	remote         string // The remote path (encoded path)
+	path           string // The local path (OS path)
 	size           int64  // file metadata - always present
 	mode           os.FileMode
 	modTime        time.Time
@@ -183,14 +191,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	f := &Fs{
-		name:     name,
-		opt:      *opt,
-		warned:   make(map[string]struct{}),
-		dev:      devUnset,
-		lstat:    os.Lstat,
-		dirNames: newMapper(),
+		name:   name,
+		opt:    *opt,
+		warned: make(map[string]struct{}),
+		dev:    devUnset,
+		lstat:  os.Lstat,
 	}
-	f.root = f.cleanPath(root)
+	f.root = cleanRootPath(root, f.opt.NoUNC, f.opt.Enc)
 	f.features = (&fs.Features{
 		CaseInsensitive:         f.caseInsensitive(),
 		CanHaveEmptyDirectories: true,
@@ -235,12 +242,12 @@ func (f *Fs) Name() string {
 
 // Root of the remote (as passed into NewFs)
 func (f *Fs) Root() string {
-	return f.root
+	return f.opt.Enc.ToStandardPath(filepath.ToSlash(f.root))
 }
 
 // String converts this Fs to a string
 func (f *Fs) String() string {
-	return fmt.Sprintf("Local file system at %s", f.root)
+	return fmt.Sprintf("Local file system at %s", f.Root())
 }
 
 // Features returns the optional features of this Fs
@@ -268,33 +275,27 @@ func (f *Fs) caseInsensitive() bool {
 // and returns a new path, removing the suffix as needed,
 // It also returns whether this is a translated link at all
 //
-// for regular files, dstPath is returned unchanged
-func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
+// for regular files, localPath is returned unchanged
+func translateLink(remote, localPath string) (newLocalPath string, isTranslatedLink bool) {
 	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
-	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
-	return newDstPath, isTranslatedLink
+	newLocalPath = strings.TrimSuffix(localPath, linkSuffix)
+	return newLocalPath, isTranslatedLink
 }
 
 // newObject makes a half completed Object
-//
-// if dstPath is empty then it is made from remote
-func (f *Fs) newObject(remote, dstPath string) *Object {
+func (f *Fs) newObject(remote string) *Object {
 	translatedLink := false
-
-	if dstPath == "" {
-		dstPath = f.cleanPath(filepath.Join(f.root, remote))
-	}
-	remote = f.cleanRemote(remote)
+	localPath := f.localPath(remote)
 
 	if f.opt.TranslateSymlinks {
-		// Possibly receive a new name for dstPath
-		dstPath, translatedLink = translateLink(remote, dstPath)
+		// Possibly receive a new name for localPath
+		localPath, translatedLink = translateLink(remote, localPath)
 	}
 
 	return &Object{
 		fs:             f,
 		remote:         remote,
-		path:           dstPath,
+		path:           localPath,
 		translatedLink: translatedLink,
 	}
 }
@@ -302,8 +303,8 @@ func (f *Fs) newObject(remote, dstPath string) *Object {
 // Return an Object from a path
 //
 // May return nil if an error occurred
-func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Object, error) {
-	o := f.newObject(remote, dstPath)
+func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) {
+	o := f.newObject(remote)
 	if info != nil {
 		o.setMetadata(info)
 	} else {
@@ -332,7 +333,7 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	return f.newObjectWithInfo(remote, "", nil)
+	return f.newObjectWithInfo(remote, nil)
 }
 
 // List the objects and directories in dir into entries.  The
@@ -345,10 +346,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-
-	dir = f.dirNames.Load(dir)
-	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
-	remote := f.cleanRemote(dir)
+	fsDirPath := f.localPath(dir)
 	_, err = os.Stat(fsDirPath)
 	if err != nil {
 		return nil, fs.ErrorDirNotFound
@@ -360,7 +358,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		err = errors.Wrapf(err, "failed to open directory %q", dir)
 		fs.Errorf(dir, "%v", err)
 		if isPerm {
-			accounting.Stats(ctx).Error(fserrors.NoRetryError(err))
+			_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(err))
 			err = nil // ignore error but fail sync
 		}
 		return nil, err
@@ -396,7 +394,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 					if fierr != nil {
 						err = errors.Wrapf(err, "failed to read directory %q", namepath)
 						fs.Errorf(dir, "%v", fierr)
-						accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
+						_ = accounting.Stats(ctx).Error(fserrors.NoRetryError(fierr)) // fail the sync
 						continue
 					}
 					fis = append(fis, fi)
@@ -410,16 +408,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		for _, fi := range fis {
 			name := fi.Name()
 			mode := fi.Mode()
-			newRemote := path.Join(remote, name)
-			newPath := filepath.Join(fsDirPath, name)
+			newRemote := f.cleanRemote(dir, name)
 			// Follow symlinks if required
 			if f.opt.FollowSymlinks && (mode&os.ModeSymlink) != 0 {
-				fi, err = os.Stat(newPath)
+				localPath := filepath.Join(fsDirPath, name)
+				fi, err = os.Stat(localPath)
 				if os.IsNotExist(err) {
 					// Skip bad symlinks
 					err = fserrors.NoRetryError(errors.Wrap(err, "symlink"))
 					fs.Errorf(newRemote, "Listing error: %v", err)
-					accounting.Stats(ctx).Error(err)
+					err = accounting.Stats(ctx).Error(err)
 					continue
 				}
 				if err != nil {
@@ -431,7 +429,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				// Ignore directories which are symlinks.  These are junction points under windows which
 				// are kind of a souped up symlink. Unix doesn't have directories which are symlinks.
 				if (mode&os.ModeSymlink) == 0 && f.dev == readDevice(fi, f.opt.OneFileSystem) {
-					d := fs.NewDir(f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), fi.ModTime())
+					d := fs.NewDir(newRemote, fi.ModTime())
 					entries = append(entries, d)
 				}
 			} else {
@@ -439,7 +437,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
 					newRemote += linkSuffix
 				}
-				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
+				fso, err := f.newObjectWithInfo(newRemote, fi)
 				if err != nil {
 					return nil, err
 				}
@@ -452,67 +450,28 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	return entries, nil
 }
 
-// cleanRemote makes string a valid UTF-8 string for remote strings.
-//
-// Any invalid UTF-8 characters will be replaced with utf8.RuneError
-// It also normalises the UTF-8 and converts the slashes if necessary.
-func (f *Fs) cleanRemote(name string) string {
-	if !utf8.ValidString(name) {
-		f.wmu.Lock()
-		if _, ok := f.warned[name]; !ok {
-			fs.Logf(f, "Replacing invalid UTF-8 characters in %q", name)
-			f.warned[name] = struct{}{}
+func (f *Fs) cleanRemote(dir, filename string) (remote string) {
+	remote = path.Join(dir, f.opt.Enc.ToStandardName(filename))
+
+	if !utf8.ValidString(filename) {
+		f.warnedMu.Lock()
+		if _, ok := f.warned[remote]; !ok {
+			fs.Logf(f, "Replacing invalid UTF-8 characters in %q", remote)
+			f.warned[remote] = struct{}{}
 		}
-		f.wmu.Unlock()
-		name = string([]rune(name))
+		f.warnedMu.Unlock()
 	}
-	name = filepath.ToSlash(name)
-	return name
+	return
 }
 
-// mapper maps raw to cleaned directory names
-type mapper struct {
-	mu sync.RWMutex      // mutex to protect the below
-	m  map[string]string // map of un-normalised directory names
-}
-
-func newMapper() *mapper {
-	return &mapper{
-		m: make(map[string]string),
-	}
-}
-
-// Lookup a directory name to make a local name (reverses
-// cleanDirName)
-//
-// FIXME this is temporary before we make a proper Directory object
-func (m *mapper) Load(in string) string {
-	m.mu.RLock()
-	out, ok := m.m[in]
-	m.mu.RUnlock()
-	if ok {
-		return out
-	}
-	return in
-}
-
-// Cleans a directory name recording if it needed to be altered
-//
-// FIXME this is temporary before we make a proper Directory object
-func (m *mapper) Save(in, out string) string {
-	if in != out {
-		m.mu.Lock()
-		m.m[out] = in
-		m.mu.Unlock()
-	}
-	return out
+func (f *Fs) localPath(name string) string {
+	return filepath.Join(f.root, filepath.FromSlash(f.opt.Enc.FromStandardPath(name)))
 }
 
 // Put the Object to the local filesystem
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	remote := src.Remote()
 	// Temporary Object under construction - info filled in by Update()
-	o := f.newObject(remote, "")
+	o := f.newObject(src.Remote())
 	err := o.Update(ctx, in, src, options...)
 	if err != nil {
 		return nil, err
@@ -528,13 +487,13 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // Mkdir creates the directory if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	// FIXME: https://github.com/syncthing/syncthing/blob/master/lib/osutil/mkdirall_windows.go
-	root := f.cleanPath(filepath.Join(f.root, dir))
-	err := os.MkdirAll(root, 0777)
+	localPath := f.localPath(dir)
+	err := os.MkdirAll(localPath, 0777)
 	if err != nil {
 		return err
 	}
 	if dir == "" {
-		fi, err := f.lstat(root)
+		fi, err := f.lstat(localPath)
 		if err != nil {
 			return err
 		}
@@ -547,8 +506,7 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 //
 // If it isn't empty it will return an error
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	root := f.cleanPath(filepath.Join(f.root, dir))
-	return os.Remove(root)
+	return os.Remove(f.localPath(dir))
 }
 
 // Precision of the file system
@@ -644,7 +602,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// Temporary Object under construction
-	dstObj := f.newObject(remote, "")
+	dstObj := f.newObject(remote)
 
 	// Check it is a file if it exists
 	err := dstObj.lstat()
@@ -701,8 +659,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := f.cleanPath(filepath.Join(srcFs.root, srcRemote))
-	dstPath := f.cleanPath(filepath.Join(f.root, dstRemote))
+	srcPath := srcFs.localPath(srcRemote)
+	dstPath := f.localPath(dstRemote)
 
 	// Check if destination exists
 	_, err := os.Lstat(dstPath)
@@ -736,7 +694,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	return hash.Supported
+	return hash.Supported()
 }
 
 // ------------------------------------------------------------
@@ -836,13 +794,6 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 
 // Storable returns a boolean showing if this object is storable
 func (o *Object) Storable() bool {
-	// Check for control characters in the remote name and show non storable
-	for _, c := range o.Remote() {
-		if c >= 0x00 && c < 0x20 || c == 0x7F {
-			fs.Logf(o.fs, "Can't store file with control characters: %q", o.Remote())
-			return false
-		}
-	}
 	mode := o.mode
 	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
@@ -877,10 +828,10 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 			return 0, errors.Wrap(err, "can't read status of source file while transferring")
 		}
 		if file.o.size != fi.Size() {
-			return 0, errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size())
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size()))
 		}
 		if !file.o.modTime.Equal(fi.ModTime()) {
-			return 0, errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime())
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime()))
 		}
 	}
 
@@ -1013,7 +964,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if !o.translatedLink {
 		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
 		if err != nil {
-			return err
+			if runtime.GOOS == "windows" && os.IsPermission(err) {
+				// If permission denied on Windows might be trying to update a
+				// hidden file, in which case try opening without CREATE
+				// See: https://stackoverflow.com/questions/13215716/ioerror-errno-13-permission-denied-when-trying-to-open-hidden-file-in-w-mod
+				f, err = file.OpenFile(o.path, os.O_WRONLY|os.O_TRUNC, 0666)
+				if err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
 		}
 		// Pre-allocate the file for performance reasons
 		err = preAllocate(src.Size(), f)
@@ -1087,7 +1048,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 // It truncates any existing object
 func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.WriterAtCloser, error) {
 	// Temporary Object under construction
-	o := f.newObject(remote, "")
+	o := f.newObject(remote)
 
 	err := o.mkdirAll()
 	if err != nil {
@@ -1139,29 +1100,7 @@ func (o *Object) Remove(ctx context.Context) error {
 	return remove(o.path)
 }
 
-// cleanPathFragment cleans an OS path fragment which is part of a
-// bigger path and not necessarily absolute
-func cleanPathFragment(s string) string {
-	if s == "" {
-		return s
-	}
-	s = filepath.Clean(s)
-	if runtime.GOOS == "windows" {
-		s = strings.Replace(s, `/`, `\`, -1)
-	}
-	return s
-}
-
-// cleanPath cleans and makes absolute the path passed in and returns
-// an OS path.
-//
-// The input might be in OS form or rclone form or a mixture, but the
-// output is in OS form.
-//
-// On windows it makes the path UNC also and replaces any characters
-// Windows can't deal with with their replacements.
-func (f *Fs) cleanPath(s string) string {
-	s = cleanPathFragment(s)
+func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 	if runtime.GOOS == "windows" {
 		if !filepath.IsAbs(s) && !strings.HasPrefix(s, "\\") {
 			s2, err := filepath.Abs(s)
@@ -1169,19 +1108,24 @@ func (f *Fs) cleanPath(s string) string {
 				s = s2
 			}
 		}
-		if !f.opt.NoUNC {
+		s = filepath.ToSlash(s)
+		vol := filepath.VolumeName(s)
+		s = vol + enc.FromStandardPath(s[len(vol):])
+		s = filepath.FromSlash(s)
+
+		if !noUNC {
 			// Convert to UNC
 			s = uncPath(s)
 		}
-		s = cleanWindowsName(f, s)
-	} else {
-		if !filepath.IsAbs(s) {
-			s2, err := filepath.Abs(s)
-			if err == nil {
-				s = s2
-			}
+		return s
+	}
+	if !filepath.IsAbs(s) {
+		s2, err := filepath.Abs(s)
+		if err == nil {
+			s = s2
 		}
 	}
+	s = enc.FromStandardPath(s)
 	return s
 }
 
@@ -1190,63 +1134,21 @@ var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
 
 // uncPath converts an absolute Windows path
 // to a UNC long path.
-func uncPath(s string) string {
-	// UNC can NOT use "/", so convert all to "\"
-	s = strings.Replace(s, `/`, `\`, -1)
-
+func uncPath(l string) string {
 	// If prefix is "\\", we already have a UNC path or server.
-	if strings.HasPrefix(s, `\\`) {
+	if strings.HasPrefix(l, `\\`) {
 		// If already long path, just keep it
-		if strings.HasPrefix(s, `\\?\`) {
-			return s
+		if strings.HasPrefix(l, `\\?\`) {
+			return l
 		}
 
 		// Trim "\\" from path and add UNC prefix.
-		return `\\?\UNC\` + strings.TrimPrefix(s, `\\`)
+		return `\\?\UNC\` + strings.TrimPrefix(l, `\\`)
 	}
-	if isAbsWinDrive.MatchString(s) {
-		return `\\?\` + s
+	if isAbsWinDrive.MatchString(l) {
+		return `\\?\` + l
 	}
-	return s
-}
-
-// cleanWindowsName will clean invalid Windows characters replacing them with _
-func cleanWindowsName(f *Fs, name string) string {
-	original := name
-	var name2 string
-	if strings.HasPrefix(name, `\\?\`) {
-		name2 = `\\?\`
-		name = strings.TrimPrefix(name, `\\?\`)
-	}
-	if strings.HasPrefix(name, `//?/`) {
-		name2 = `//?/`
-		name = strings.TrimPrefix(name, `//?/`)
-	}
-	// Colon is allowed as part of a drive name X:\
-	colonAt := strings.Index(name, ":")
-	if colonAt > 0 && colonAt < 3 && len(name) > colonAt+1 {
-		// Copy to name2, which is unfiltered
-		name2 += name[0 : colonAt+1]
-		name = name[colonAt+1:]
-	}
-
-	name2 += strings.Map(func(r rune) rune {
-		switch r {
-		case '<', '>', '"', '|', '?', '*', ':':
-			return '_'
-		}
-		return r
-	}, name)
-
-	if name2 != original && f != nil {
-		f.wmu.Lock()
-		if _, ok := f.warned[name]; !ok {
-			fs.Logf(f, "Replacing invalid characters in %q to %q", name, name2)
-			f.warned[name] = struct{}{}
-		}
-		f.wmu.Unlock()
-	}
-	return name2
+	return l
 }
 
 // Check the interfaces are satisfied

backend/local/local_internal_test.go

@@ -25,19 +25,6 @@ func TestMain(m *testing.M) {
 	fstest.TestMain(m)
 }
 
-func TestMapper(t *testing.T) {
-	m := newMapper()
-	assert.Equal(t, m.m, map[string]string{})
-	assert.Equal(t, "potato", m.Save("potato", "potato"))
-	assert.Equal(t, m.m, map[string]string{})
-	assert.Equal(t, "-r'áö", m.Save("-r?'a´o¨", "-r'áö"))
-	assert.Equal(t, m.m, map[string]string{
-		"-r'áö": "-r?'a´o¨",
-	})
-	assert.Equal(t, "potato", m.Load("potato"))
-	assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö"))
-}
-
 // Test copy with source file that's updating
 func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
@@ -57,7 +44,7 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 	o := &Object{size: fi.Size(), modTime: fi.ModTime(), fs: &Fs{}}
 	wrappedFd := readers.NewLimitedReadCloser(fd, -1)
-	hash, err := hash.NewMultiHasherTypes(hash.Supported)
+	hash, err := hash.NewMultiHasherTypes(hash.Supported())
 	require.NoError(t, err)
 	in := localOpenFile{
 		o:    o,

backend/local/tests_test.go

@@ -1,29 +1,26 @@
 package local
 
 import (
+	"runtime"
 	"testing"
 )
 
 var uncTestPaths = []string{
-	"C:\\Ba*d\\P|a?t<h>\\Windows\\Folder",
-	"C:/Ba*d/P|a?t<h>/Windows\\Folder",
-	"C:\\Windows\\Folder",
-	"\\\\?\\C:\\Windows\\Folder",
-	"//?/C:/Windows/Folder",
-	"\\\\?\\UNC\\server\\share\\Desktop",
-	"\\\\?\\unC\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"\\\\server\\share\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"C:\\Desktop\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"C:\\AbsoluteToRoot\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path\\Very Long path",
-	"\\\\server\\share\\Desktop",
-	"\\\\?\\UNC\\\\share\\folder\\Desktop",
-	"\\\\server\\share",
+	`C:\Ba*d\P|a?t<h>\Windows\Folder`,
+	`C:\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop`,
+	`\\?\UNC\\share\folder\Desktop`,
+	`\\server\share`,
 }
 
 var uncTestPathsResults = []string{
 	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
-	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
-	`\\?\C:\Windows\Folder`,
 	`\\?\C:\Windows\Folder`,
 	`\\?\C:\Windows\Folder`,
 	`\\?\UNC\server\share\Desktop`,
@@ -51,38 +48,23 @@ func TestUncPaths(t *testing.T) {
 	}
 }
 
-var utf8Tests = [][2]string{
-	{"ABC", "ABC"},
-	{string([]byte{0x80}), "�"},
-	{string([]byte{'a', 0x80, 'b'}), "a�b"},
-}
-
-func TestCleanRemote(t *testing.T) {
-	f := &Fs{}
-	f.warned = make(map[string]struct{})
-	for _, test := range utf8Tests {
-		got := f.cleanRemote(test[0])
-		expect := test[1]
-		if got != expect {
-			t.Fatalf("got %q, expected %q", got, expect)
-		}
-	}
-}
-
 // Test Windows character replacements
 var testsWindows = [][2]string{
 	{`c:\temp`, `c:\temp`},
 	{`\\?\UNC\theserver\dir\file.txt`, `\\?\UNC\theserver\dir\file.txt`},
-	{`//?/UNC/theserver/dir\file.txt`, `//?/UNC/theserver/dir\file.txt`},
-	{"c:/temp", "c:/temp"},
-	{"/temp/file.txt", "/temp/file.txt"},
-	{`!\"#¤%&/()=;:*^?+-`, "!\\_#¤%&/()=;__^_+-"},
-	{`<>"|?*:&\<>"|?*:&\<>"|?*:&`, "_______&\\_______&\\_______&"},
+	{`//?/UNC/theserver/dir\file.txt`, `\\?\UNC\theserver\dir\file.txt`},
+	{`c:/temp`, `c:\temp`},
+	{`C:/temp/file.txt`, `C:\temp\file.txt`},
+	{`c:\!\"#¤%&/()=;:*^?+-`, `c:\!\＂#¤%&\()=;：＊^？+-`},
+	{`c:\<>"|?*:&\<>"|?*:&\<>"|?*:&`, `c:\＜＞＂｜？＊：&\＜＞＂｜？＊：&\＜＞＂｜？＊：&`},
 }
 
 func TestCleanWindows(t *testing.T) {
+	if runtime.GOOS != "windows" {
+		t.Skipf("windows only")
+	}
 	for _, test := range testsWindows {
-		got := cleanWindowsName(nil, test[0])
+		got := cleanRootPath(test[0], true, defaultEnc)
 		expect := test[1]
 		if got != expect {
 			t.Fatalf("got %q, expected %q", got, expect)

backend/mailru/mailru.go

@@ -24,6 +24,7 @@ import (
 	"github.com/rclone/rclone/backend/mailru/mrhash"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -33,6 +34,7 @@ import (
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/operations"
 
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
@@ -59,6 +61,9 @@ var (
 	ErrorDirAlreadyExists   = errors.New("directory already exists")
 	ErrorDirSourceNotExists = errors.New("directory source does not exist")
 	ErrorInvalidName        = errors.New("invalid characters in object name")
+
+	// MrHashType is the hash.Type for Mailru
+	MrHashType hash.Type
 )
 
 // Description of how to authorize
@@ -74,6 +79,7 @@ var oauthConfig = &oauth2.Config{
 
 // Register with Fs
 func init() {
+	MrHashType = hash.RegisterHash("MailruHash", 40, mrhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "mailru",
 		Description: "Mail.ru Cloud",
@@ -186,21 +192,31 @@ facilitate remote troubleshooting of backend issues. Strict meaning of
 flags is not documented and not guaranteed to persist between releases.
 Quirks will be removed when the backend grows stable.
 Supported quirks: atomicmkdir binlist gzip insecure retry400`,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeWin | // :?"*<>|
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Username        string        `config:"user"`
-	Password        string        `config:"pass"`
-	UserAgent       string        `config:"user_agent"`
-	CheckHash       bool          `config:"check_hash"`
-	SpeedupEnable   bool          `config:"speedup_enable"`
-	SpeedupPatterns string        `config:"speedup_file_patterns"`
-	SpeedupMaxDisk  fs.SizeSuffix `config:"speedup_max_disk"`
-	SpeedupMaxMem   fs.SizeSuffix `config:"speedup_max_memory"`
-	Quirks          string        `config:"quirks"`
+	Username        string               `config:"user"`
+	Password        string               `config:"pass"`
+	UserAgent       string               `config:"user_agent"`
+	CheckHash       bool                 `config:"check_hash"`
+	SpeedupEnable   bool                 `config:"speedup_enable"`
+	SpeedupPatterns string               `config:"speedup_file_patterns"`
+	SpeedupMaxDisk  fs.SizeSuffix        `config:"speedup_max_disk"`
+	SpeedupMaxMem   fs.SizeSuffix        `config:"speedup_max_memory"`
+	Quirks          string               `config:"quirks"`
+	Enc             encoder.MultiEncoder `config:"encoding"`
 }
 
 // retryErrorCodes is a slice of error codes that we will retry
@@ -280,7 +296,7 @@ type Fs struct {
 
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	fs.Debugf(nil, ">>> NewFs %q %q", name, root)
+	// fs.Debugf(nil, ">>> NewFs %q %q", name, root)
 	ctx := context.Background() // Note: NewFs does not pass context!
 
 	// Parse config into Options struct
@@ -515,7 +531,7 @@ func (f *Fs) accessToken() (string, error) {
 
 // absPath converts root-relative remote to absolute home path
 func (f *Fs) absPath(remote string) string {
-	return "/" + path.Join(f.root, strings.Trim(remote, "/"))
+	return path.Join("/", f.root, remote)
 }
 
 // relPath converts absolute home path to root-relative remote
@@ -600,7 +616,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 		Path:   "/api/m1/file",
 		Parameters: url.Values{
 			"access_token": {token},
-			"home":         {path},
+			"home":         {f.opt.Enc.FromStandardPath(path)},
 			"offset":       {"0"},
 			"limit":        {strconv.Itoa(maxInt32)},
 		},
@@ -635,7 +651,7 @@ func (f *Fs) readItemMetaData(ctx context.Context, path string) (entry fs.DirEnt
 //   =0 - for an empty directory
 //   >0 - for a non-empty directory
 func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.DirEntry, dirSize int, err error) {
-	remote, err := f.relPath(item.Home)
+	remote, err := f.relPath(f.opt.Enc.ToStandardPath(item.Home))
 	if err != nil {
 		return nil, -1, err
 	}
@@ -668,7 +684,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
 // dir should be "" to list the root, and should not have trailing slashes.
 // This should return ErrDirNotFound if the directory isn't found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	fs.Debugf(f, ">>> List: %q", dir)
+	// fs.Debugf(f, ">>> List: %q", dir)
 
 	if f.quirks.binlist {
 		entries, err = f.listBin(ctx, f.absPath(dir), 1)
@@ -682,7 +698,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			names = append(names, entry.Remote())
 		}
 		sort.Strings(names)
-		fs.Debugf(f, "List(%q): %v", dir, names)
+		// fs.Debugf(f, "List(%q): %v", dir, names)
 	}
 
 	return
@@ -701,7 +717,7 @@ func (f *Fs) listM1(ctx context.Context, dirPath string, offset int, limit int)
 	params.Set("limit", strconv.Itoa(limit))
 
 	data := url.Values{}
-	data.Set("home", dirPath)
+	data.Set("home", f.opt.Enc.FromStandardPath(dirPath))
 
 	opts := rest.Opts{
 		Method:      "POST",
@@ -749,7 +765,7 @@ func (f *Fs) listBin(ctx context.Context, dirPath string, depth int) (entries fs
 
 	req := api.NewBinWriter()
 	req.WritePu16(api.OperationFolderList)
-	req.WriteString(dirPath)
+	req.WriteString(f.opt.Enc.FromStandardPath(dirPath))
 	req.WritePu32(int64(depth))
 	req.WritePu32(int64(options))
 	req.WritePu32(0)
@@ -885,7 +901,7 @@ func (t *treeState) NextRecord() (fs.DirEntry, error) {
 	if (head & 4096) != 0 {
 		t.dunnoNodeID = r.ReadNBytes(api.DunnoNodeIDLength)
 	}
-	name := string(r.ReadBytesByLength())
+	name := t.f.opt.Enc.FromStandardPath(string(r.ReadBytesByLength()))
 	t.dunno1 = int(r.ReadULong())
 	t.dunno2 = 0
 	t.dunno3 = 0
@@ -1019,12 +1035,12 @@ func (rev *treeRevision) Read(data *api.BinReader) error {
 
 // CreateDir makes a directory (parent must exist)
 func (f *Fs) CreateDir(ctx context.Context, path string) error {
-	fs.Debugf(f, ">>> CreateDir %q", path)
+	// fs.Debugf(f, ">>> CreateDir %q", path)
 
 	req := api.NewBinWriter()
 	req.WritePu16(api.OperationCreateFolder)
 	req.WritePu16(0) // revision
-	req.WriteString(path)
+	req.WriteString(f.opt.Enc.FromStandardPath(path))
 	req.WritePu32(0)
 
 	token, err := f.accessToken()
@@ -1081,7 +1097,7 @@ func (f *Fs) CreateDir(ctx context.Context, path string) error {
 // already exists. As a workaround, users can add string "atomicmkdir" in the
 // hidden `quirks` parameter or in the `--mailru-quirks` command-line option.
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	fs.Debugf(f, ">>> Mkdir %q", dir)
+	// fs.Debugf(f, ">>> Mkdir %q", dir)
 	err := f.mkDirs(ctx, f.absPath(dir))
 	if err == ErrorDirAlreadyExists && !f.quirks.atomicmkdir {
 		return nil
@@ -1142,7 +1158,7 @@ func (f *Fs) mkParentDirs(ctx context.Context, path string) error {
 // Rmdir deletes a directory.
 // Returns an error if it isn't empty.
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	fs.Debugf(f, ">>> Rmdir %q", dir)
+	// fs.Debugf(f, ">>> Rmdir %q", dir)
 	return f.purgeWithCheck(ctx, dir, true, "rmdir")
 }
 
@@ -1150,7 +1166,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 // Optional interface: Only implement this if you have a way of deleting
 // all the files quicker than just running Remove() on the result of List()
 func (f *Fs) Purge(ctx context.Context) error {
-	fs.Debugf(f, ">>> Purge")
+	// fs.Debugf(f, ">>> Purge")
 	return f.purgeWithCheck(ctx, "", false, "purge")
 }
 
@@ -1179,7 +1195,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error {
 		return err
 	}
 
-	data := url.Values{"home": {path}}
+	data := url.Values{"home": {f.opt.Enc.FromStandardPath(path)}}
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/api/m1/file/remove",
@@ -1212,7 +1228,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) error {
 // Will only be called if src.Fs().Name() == f.Name()
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	fs.Debugf(f, ">>> Copy %q %q", src.Remote(), remote)
+	// fs.Debugf(f, ">>> Copy %q %q", src.Remote(), remote)
 
 	srcObj, ok := src.(*Object)
 	if !ok {
@@ -1228,7 +1244,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	srcPath := srcObj.absPath()
 	dstPath := f.absPath(remote)
 	overwrite := false
-	fs.Debugf(f, "copy %q -> %q\n", srcPath, dstPath)
+	// fs.Debugf(f, "copy %q -> %q\n", srcPath, dstPath)
 
 	err := f.mkParentDirs(ctx, dstPath)
 	if err != nil {
@@ -1236,8 +1252,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	data := url.Values{}
-	data.Set("home", srcPath)
-	data.Set("folder", parentDir(dstPath))
+	data.Set("home", f.opt.Enc.FromStandardPath(srcPath))
+	data.Set("folder", f.opt.Enc.FromStandardPath(parentDir(dstPath)))
 	data.Set("email", f.opt.Username)
 	data.Set("x-email", f.opt.Username)
 
@@ -1275,9 +1291,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fmt.Errorf("copy failed with code %d", response.Status)
 	}
 
-	tmpPath := response.Body
+	tmpPath := f.opt.Enc.ToStandardPath(response.Body)
 	if tmpPath != dstPath {
-		fs.Debugf(f, "rename temporary file %q -> %q\n", tmpPath, dstPath)
+		// fs.Debugf(f, "rename temporary file %q -> %q\n", tmpPath, dstPath)
 		err = f.moveItemBin(ctx, tmpPath, dstPath, "rename temporary file")
 		if err != nil {
 			_ = f.delete(ctx, tmpPath, false) // ignore error
@@ -1307,7 +1323,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // Will only be called if src.Fs().Name() == f.Name()
 // If it isn't possible then return fs.ErrorCantMove
 func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	fs.Debugf(f, ">>> Move %q %q", src.Remote(), remote)
+	// fs.Debugf(f, ">>> Move %q %q", src.Remote(), remote)
 
 	srcObj, ok := src.(*Object)
 	if !ok {
@@ -1350,9 +1366,9 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
 	req := api.NewBinWriter()
 	req.WritePu16(api.OperationRename)
 	req.WritePu32(0) // old revision
-	req.WriteString(srcPath)
+	req.WriteString(f.opt.Enc.FromStandardPath(srcPath))
 	req.WritePu32(0) // new revision
-	req.WriteString(dstPath)
+	req.WriteString(f.opt.Enc.FromStandardPath(dstPath))
 	req.WritePu32(0) // dunno
 
 	opts := rest.Opts{
@@ -1393,7 +1409,7 @@ func (f *Fs) moveItemBin(ctx context.Context, srcPath, dstPath, opName string) e
 // If it isn't possible then return fs.ErrorCantDirMove
 // If destination exists then return fs.ErrorDirExists
 func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string) error {
-	fs.Debugf(f, ">>> DirMove %q %q", srcRemote, dstRemote)
+	// fs.Debugf(f, ">>> DirMove %q %q", srcRemote, dstRemote)
 
 	srcFs, ok := src.(*Fs)
 	if !ok {
@@ -1407,7 +1423,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	}
 	srcPath := srcFs.absPath(srcRemote)
 	dstPath := f.absPath(dstRemote)
-	fs.Debugf(srcFs, "DirMove [%s]%q --> [%s]%q\n", srcRemote, srcPath, dstRemote, dstPath)
+	// fs.Debugf(srcFs, "DirMove [%s]%q --> [%s]%q\n", srcRemote, srcPath, dstRemote, dstPath)
 
 	// Refuse to move to or from the root
 	if len(srcPath) <= len(srcFs.root) || len(dstPath) <= len(f.root) {
@@ -1435,7 +1451,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
 func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
-	fs.Debugf(f, ">>> PublicLink %q", remote)
+	// fs.Debugf(f, ">>> PublicLink %q", remote)
 
 	token, err := f.accessToken()
 	if err != nil {
@@ -1443,7 +1459,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 	}
 
 	data := url.Values{}
-	data.Set("home", f.absPath(remote))
+	data.Set("home", f.opt.Enc.FromStandardPath(f.absPath(remote)))
 	data.Set("email", f.opt.Username)
 	data.Set("x-email", f.opt.Username)
 
@@ -1477,7 +1493,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 
 // CleanUp permanently deletes all trashed files/folders
 func (f *Fs) CleanUp(ctx context.Context) error {
-	fs.Debugf(f, ">>> CleanUp")
+	// fs.Debugf(f, ">>> CleanUp")
 
 	token, err := f.accessToken()
 	if err != nil {
@@ -1517,7 +1533,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
-	fs.Debugf(f, ">>> About")
+	// fs.Debugf(f, ">>> About")
 
 	token, err := f.accessToken()
 	if err != nil {
@@ -1561,7 +1577,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		size:    src.Size(),
 		modTime: src.ModTime(ctx),
 	}
-	fs.Debugf(f, ">>> Put: %q %d '%v'", o.remote, o.size, o.modTime)
+	// fs.Debugf(f, ">>> Put: %q %d '%v'", o.remote, o.size, o.modTime)
 	return o, o.Update(ctx, in, src, options...)
 }
 
@@ -1591,7 +1607,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// Skip an extra speedup request if file fits in hash.
 	if size > mrhash.Size {
 		// Request hash from source.
-		if srcHash, err := src.Hash(ctx, hash.Mailru); err == nil && srcHash != "" {
+		if srcHash, err := src.Hash(ctx, MrHashType); err == nil && srcHash != "" {
 			fileHash, _ = mrhash.DecodeString(srcHash)
 		}
 
@@ -1762,7 +1778,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
 	hashType := hash.SHA1
 
 	// Calculate Mailru and spool verification hashes in transit
-	hashSet := hash.NewHashSet(hash.Mailru, hashType)
+	hashSet := hash.NewHashSet(MrHashType, hashType)
 	hasher, err := hash.NewMultiHasherTypes(hashSet)
 	if err != nil {
 		return nil, nil, err
@@ -1784,7 +1800,7 @@ func makeTempFile(ctx context.Context, tmpFs fs.Fs, wrapIn io.Reader, src fs.Obj
 		return nil, nil, mrhash.ErrorInvalidHash
 	}
 
-	mrHash, err = mrhash.DecodeString(sums[hash.Mailru])
+	mrHash, err = mrhash.DecodeString(sums[MrHashType])
 	return
 }
 
@@ -1888,7 +1904,7 @@ type Object struct {
 // NewObject finds an Object at the remote.
 // If object can't be found it fails with fs.ErrorObjectNotFound
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	fs.Debugf(f, ">>> NewObject %q", remote)
+	// fs.Debugf(f, ">>> NewObject %q", remote)
 	o := &Object{
 		fs:     f,
 		remote: remote,
@@ -1972,7 +1988,7 @@ func (o *Object) Size() int64 {
 // Hash returns the MD5 or SHA1 sum of an object
 // returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if t == hash.Mailru {
+	if t == MrHashType {
 		return hex.EncodeToString(o.mrHash), nil
 	}
 	return "", hash.ErrUnsupported
@@ -1987,7 +2003,7 @@ func (o *Object) Storable() bool {
 //
 // Commits the datastore
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	fs.Debugf(o, ">>> SetModTime [%v]", modTime)
+	// fs.Debugf(o, ">>> SetModTime [%v]", modTime)
 	o.modTime = modTime
 	return o.addFileMetaData(ctx, true)
 }
@@ -2008,7 +2024,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 	req := api.NewBinWriter()
 	req.WritePu16(api.OperationAddFile)
 	req.WritePu16(0) // revision
-	req.WriteString(o.absPath())
+	req.WriteString(o.fs.opt.Enc.FromStandardPath(o.absPath()))
 	req.WritePu64(o.size)
 	req.WritePu64(o.modTime.Unix())
 	req.WritePu32(0)
@@ -2060,7 +2076,7 @@ func (o *Object) addFileMetaData(ctx context.Context, overwrite bool) error {
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) error {
-	fs.Debugf(o, ">>> Remove")
+	// fs.Debugf(o, ">>> Remove")
 	return o.fs.delete(ctx, o.absPath(), false)
 }
 
@@ -2093,7 +2109,7 @@ func getTransferRange(size int64, options ...fs.OpenOption) (start int64, end in
 
 // Open an object for read and download its content
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	fs.Debugf(o, ">>> Open")
+	// fs.Debugf(o, ">>> Open")
 
 	token, err := o.fs.accessToken()
 	if err != nil {
@@ -2106,7 +2122,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	opts := rest.Opts{
 		Method:  "GET",
 		Options: options,
-		Path:    url.PathEscape(strings.TrimLeft(o.absPath(), "/")),
+		Path:    url.PathEscape(strings.TrimLeft(o.fs.opt.Enc.FromStandardPath(o.absPath()), "/")),
 		Parameters: url.Values{
 			"client_id": {api.OAuthClientID},
 			"token":     {token},
@@ -2354,7 +2370,7 @@ func (f *Fs) Precision() time.Duration {
 
 // Hashes returns the supported hash sets
 func (f *Fs) Hashes() hash.Set {
-	return hash.Set(hash.Mailru)
+	return hash.Set(MrHashType)
 }
 
 // Features returns the optional features of this Fs

backend/mega/mega.go

@@ -26,11 +26,13 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	mega "github.com/t3rm1n4l/go-mega"
@@ -80,16 +82,24 @@ than permanently deleting them.  If you specify this then rclone will
 permanently delete objects instead.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Base |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	User       string `config:"user"`
-	Pass       string `config:"pass"`
-	Debug      bool   `config:"debug"`
-	HardDelete bool   `config:"hard_delete"`
+	User       string               `config:"user"`
+	Pass       string               `config:"pass"`
+	Debug      bool                 `config:"debug"`
+	HardDelete bool                 `config:"hard_delete"`
+	Enc        encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote mega
@@ -245,21 +255,22 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 }
 
 // splitNodePath splits nodePath into / separated parts, returning nil if it
-// should refer to the root
-func splitNodePath(nodePath string) (parts []string) {
+// should refer to the root.
+// It also encodes the parts into backend specific encoding
+func (f *Fs) splitNodePath(nodePath string) (parts []string) {
 	nodePath = path.Clean(nodePath)
-	parts = strings.Split(nodePath, "/")
-	if len(parts) == 1 && (parts[0] == "." || parts[0] == "/") {
+	if nodePath == "." || nodePath == "/" {
 		return nil
 	}
-	return parts
+	nodePath = f.opt.Enc.FromStandardPath(nodePath)
+	return strings.Split(nodePath, "/")
 }
 
 // findNode looks up the node for the path of the name given from the root given
 //
 // It returns mega.ENOENT if it wasn't found
 func (f *Fs) findNode(rootNode *mega.Node, nodePath string) (*mega.Node, error) {
-	parts := splitNodePath(nodePath)
+	parts := f.splitNodePath(nodePath)
 	if parts == nil {
 		return rootNode, nil
 	}
@@ -316,7 +327,7 @@ func (f *Fs) mkdir(rootNode *mega.Node, dir string) (node *mega.Node, err error)
 	f.mkdirMu.Lock()
 	defer f.mkdirMu.Unlock()
 
-	parts := splitNodePath(dir)
+	parts := f.splitNodePath(dir)
 	if parts == nil {
 		return rootNode, nil
 	}
@@ -418,7 +429,7 @@ func (f *Fs) CleanUp(ctx context.Context) (err error) {
 	errors := 0
 	// similar to f.deleteNode(trash) but with HardDelete as true
 	for _, item := range items {
-		fs.Debugf(f, "Deleting trash %q", item.GetName())
+		fs.Debugf(f, "Deleting trash %q", f.opt.Enc.ToStandardName(item.GetName()))
 		deleteErr := f.pacer.Call(func() (bool, error) {
 			err := f.srv.Delete(item, true)
 			return shouldRetry(err)
@@ -500,7 +511,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 	var iErr error
 	_, err = f.list(ctx, dirNode, func(info *mega.Node) bool {
-		remote := path.Join(dir, info.GetName())
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(info.GetName()))
 		switch info.GetType() {
 		case mega.FOLDER, mega.ROOT, mega.INBOX, mega.TRASH:
 			d := fs.NewDir(remote, info.GetTimeStamp()).SetID(info.GetHash())
@@ -722,7 +733,7 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 	if srcLeaf != dstLeaf {
 		//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
 		err = f.pacer.Call(func() (bool, error) {
-			err = f.srv.Rename(info, dstLeaf)
+			err = f.srv.Rename(info, f.opt.Enc.FromStandardName(dstLeaf))
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -871,13 +882,13 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 		}
 		// move them into place
 		for _, info := range infos {
-			fs.Infof(srcDir, "merging %q", info.GetName())
+			fs.Infof(srcDir, "merging %q", f.opt.Enc.ToStandardName(info.GetName()))
 			err = f.pacer.Call(func() (bool, error) {
 				err = f.srv.Move(info, dstDirNode)
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", f.opt.Enc.ToStandardName(info.GetName()), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
@@ -1120,7 +1131,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	var u *mega.Upload
 	err = o.fs.pacer.Call(func() (bool, error) {
-		u, err = o.fs.srv.NewUpload(dirNode, leaf, size)
+		u, err = o.fs.srv.NewUpload(dirNode, o.fs.opt.Enc.FromStandardName(leaf), size)
 		return shouldRetry(err)
 	})
 	if err != nil {

backend/memory/memory.go

@@ -0,0 +1,624 @@
+// Package memory provides an interface to an in memory object storage system
+package memory
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/bucket"
+)
+
+var (
+	hashType = hash.MD5
+	// the object storage is persistent
+	buckets = newBucketsInfo()
+)
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "memory",
+		Description: "In memory object storage system.",
+		NewFs:       NewFs,
+		Options:     []fs.Option{},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+}
+
+// Fs represents a remote memory server
+type Fs struct {
+	name          string       // name of this remote
+	root          string       // the path we are working on if any
+	opt           Options      // parsed config options
+	rootBucket    string       // bucket part of root (if any)
+	rootDirectory string       // directory part of root (if any)
+	features      *fs.Features // optional features
+}
+
+// bucketsInfo holds info about all the buckets
+type bucketsInfo struct {
+	mu      sync.RWMutex
+	buckets map[string]*bucketInfo
+}
+
+func newBucketsInfo() *bucketsInfo {
+	return &bucketsInfo{
+		buckets: make(map[string]*bucketInfo, 16),
+	}
+}
+
+// getBucket gets a names bucket or nil
+func (bi *bucketsInfo) getBucket(name string) (b *bucketInfo) {
+	bi.mu.RLock()
+	b = bi.buckets[name]
+	bi.mu.RUnlock()
+	return b
+}
+
+// makeBucket returns the bucket or makes it
+func (bi *bucketsInfo) makeBucket(name string) (b *bucketInfo) {
+	bi.mu.Lock()
+	defer bi.mu.Unlock()
+	b = bi.buckets[name]
+	if b != nil {
+		return b
+	}
+	b = newBucketInfo()
+	bi.buckets[name] = b
+	return b
+}
+
+// deleteBucket deleted the bucket or returns an error
+func (bi *bucketsInfo) deleteBucket(name string) error {
+	bi.mu.Lock()
+	defer bi.mu.Unlock()
+	b := bi.buckets[name]
+	if b == nil {
+		return fs.ErrorDirNotFound
+	}
+	if !b.isEmpty() {
+		return fs.ErrorDirectoryNotEmpty
+	}
+	delete(bi.buckets, name)
+	return nil
+}
+
+// getObjectData gets an object from (bucketName, bucketPath) or nil
+func (bi *bucketsInfo) getObjectData(bucketName, bucketPath string) (od *objectData) {
+	b := bi.getBucket(bucketName)
+	if b == nil {
+		return nil
+	}
+	return b.getObjectData(bucketPath)
+}
+
+// updateObjectData updates an object from (bucketName, bucketPath)
+func (bi *bucketsInfo) updateObjectData(bucketName, bucketPath string, od *objectData) {
+	b := bi.makeBucket(bucketName)
+	b.mu.Lock()
+	b.objects[bucketPath] = od
+	b.mu.Unlock()
+}
+
+// removeObjectData removes an object from (bucketName, bucketPath) returning true if removed
+func (bi *bucketsInfo) removeObjectData(bucketName, bucketPath string) (removed bool) {
+	b := bi.getBucket(bucketName)
+	if b != nil {
+		b.mu.Lock()
+		od := b.objects[bucketPath]
+		if od != nil {
+			delete(b.objects, bucketPath)
+			removed = true
+		}
+		b.mu.Unlock()
+	}
+	return removed
+}
+
+// bucketInfo holds info about a single bucket
+type bucketInfo struct {
+	mu      sync.RWMutex
+	objects map[string]*objectData
+}
+
+func newBucketInfo() *bucketInfo {
+	return &bucketInfo{
+		objects: make(map[string]*objectData, 16),
+	}
+}
+
+// getBucket gets a names bucket or nil
+func (bi *bucketInfo) getObjectData(name string) (od *objectData) {
+	bi.mu.RLock()
+	od = bi.objects[name]
+	bi.mu.RUnlock()
+	return od
+}
+
+// getBucket gets a names bucket or nil
+func (bi *bucketInfo) isEmpty() (empty bool) {
+	bi.mu.RLock()
+	empty = len(bi.objects) == 0
+	bi.mu.RUnlock()
+	return empty
+}
+
+// the object data and metadata
+type objectData struct {
+	modTime  time.Time
+	hash     string
+	mimeType string
+	data     []byte
+}
+
+// Object describes a memory object
+type Object struct {
+	fs     *Fs         // what this object is part of
+	remote string      // The remote path
+	od     *objectData // the object data
+}
+
+// ------------------------------------------------------------
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String converts this Fs to a string
+func (f *Fs) String() string {
+	return fmt.Sprintf("Memory root '%s'", f.root)
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// parsePath parses a remote 'url'
+func parsePath(path string) (root string) {
+	root = strings.Trim(path, "/")
+	return
+}
+
+// split returns bucket and bucketPath from the rootRelativePath
+// relative to f.root
+func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
+	return bucket.Split(path.Join(f.root, rootRelativePath))
+}
+
+// split returns bucket and bucketPath from the object
+func (o *Object) split() (bucket, bucketPath string) {
+	return o.fs.split(o.remote)
+}
+
+// setRoot changes the root of the Fs
+func (f *Fs) setRoot(root string) {
+	f.root = parsePath(root)
+	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
+}
+
+// NewFs contstructs an Fs from the path, bucket:path
+func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
+	// Parse config into Options struct
+	opt := new(Options)
+	err := configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
+	root = strings.Trim(root, "/")
+	f := &Fs{
+		name: name,
+		root: root,
+		opt:  *opt,
+	}
+	f.setRoot(root)
+	f.features = (&fs.Features{
+		ReadMimeType:      true,
+		WriteMimeType:     true,
+		BucketBased:       true,
+		BucketBasedRootOK: true,
+	}).Fill(f)
+	if f.rootBucket != "" && f.rootDirectory != "" {
+		od := buckets.getObjectData(f.rootBucket, f.rootDirectory)
+		if od != nil {
+			newRoot := path.Dir(f.root)
+			if newRoot == "." {
+				newRoot = ""
+			}
+			f.setRoot(newRoot)
+			// return an error with an fs which points to the parent
+			err = fs.ErrorIsFile
+		}
+	}
+	return f, err
+}
+
+// newObject makes an object from a remote and an objectData
+func (f *Fs) newObject(remote string, od *objectData) *Object {
+	return &Object{fs: f, remote: remote, od: od}
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
+	bucket, bucketPath := f.split(remote)
+	od := buckets.getObjectData(bucket, bucketPath)
+	if od == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return f.newObject(remote, od), nil
+}
+
+// listFn is called from list to handle an object.
+type listFn func(remote string, entry fs.DirEntry, isDirectory bool) error
+
+// list the buckets to fn
+func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBucket bool, recurse bool, fn listFn) (err error) {
+	if prefix != "" {
+		prefix += "/"
+	}
+	if directory != "" {
+		directory += "/"
+	}
+	b := buckets.getBucket(bucket)
+	if b == nil {
+		return fs.ErrorDirNotFound
+	}
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	dirs := make(map[string]struct{})
+	for absPath, od := range b.objects {
+		if strings.HasPrefix(absPath, directory) {
+			remote := absPath[len(prefix):]
+			if !recurse {
+				localPath := absPath[len(directory):]
+				slash := strings.IndexRune(localPath, '/')
+				if slash >= 0 {
+					// send a directory if have a slash
+					dir := directory + localPath[:slash]
+					if addBucket {
+						dir = path.Join(bucket, dir)
+					}
+					_, found := dirs[dir]
+					if !found {
+						err = fn(dir, fs.NewDir(dir, time.Time{}), true)
+						if err != nil {
+							return err
+						}
+						dirs[dir] = struct{}{}
+					}
+					continue // don't send this file if not recursing
+				}
+			}
+			// send an object
+			if addBucket {
+				remote = path.Join(bucket, remote)
+			}
+			err = fn(remote, f.newObject(remote, od), false)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// listDir lists the bucket to the entries
+func (f *Fs) listDir(ctx context.Context, bucket, directory, prefix string, addBucket bool) (entries fs.DirEntries, err error) {
+	// List the objects and directories
+	err = f.list(ctx, bucket, directory, prefix, addBucket, false, func(remote string, entry fs.DirEntry, isDirectory bool) error {
+		entries = append(entries, entry)
+		return nil
+	})
+	return entries, err
+}
+
+// listBuckets lists the buckets to entries
+func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
+	buckets.mu.RLock()
+	defer buckets.mu.RUnlock()
+	for name := range buckets.buckets {
+		entries = append(entries, fs.NewDir(name, time.Time{}))
+	}
+	return entries, nil
+}
+
+// List the objects and directories in dir into entries.  The
+// entries can be returned in any order but should be for a
+// complete directory.
+//
+// dir should be "" to list the root, and should not have
+// trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+	// defer fslog.Trace(dir, "")("entries = %q, err = %v", &entries, &err)
+	bucket, directory := f.split(dir)
+	if bucket == "" {
+		if directory != "" {
+			return nil, fs.ErrorListBucketRequired
+		}
+		return f.listBuckets(ctx)
+	}
+	return f.listDir(ctx, bucket, directory, f.rootDirectory, f.rootBucket == "")
+}
+
+// ListR lists the objects and directories of the Fs starting
+// from dir recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order.  If
+// callback returns an error then the listing will stop
+// immediately.
+//
+// Don't implement this unless you have a more efficient way
+// of listing recursively that doing a directory traversal.
+func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+	bucket, directory := f.split(dir)
+	list := walk.NewListRHelper(callback)
+	listR := func(bucket, directory, prefix string, addBucket bool) error {
+		return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {
+			return list.Add(entry)
+		})
+	}
+	if bucket == "" {
+		entries, err := f.listBuckets(ctx)
+		if err != nil {
+			return err
+		}
+		for _, entry := range entries {
+			err = list.Add(entry)
+			if err != nil {
+				return err
+			}
+			bucket := entry.Remote()
+			err = listR(bucket, "", f.rootDirectory, true)
+			if err != nil {
+				return err
+			}
+		}
+	} else {
+		err = listR(bucket, directory, f.rootDirectory, f.rootBucket == "")
+		if err != nil {
+			return err
+		}
+	}
+	return list.Flush()
+}
+
+// Put the object into the bucket
+//
+// Copy the reader in to the new object which is returned
+//
+// The new object may have been created if an error is returned
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	// Temporary Object under construction
+	fs := &Object{
+		fs:     f,
+		remote: src.Remote(),
+		od: &objectData{
+			modTime: src.ModTime(ctx),
+		},
+	}
+	return fs, fs.Update(ctx, in, src, options...)
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
+// Mkdir creates the bucket if it doesn't exist
+func (f *Fs) Mkdir(ctx context.Context, dir string) error {
+	bucket, _ := f.split(dir)
+	buckets.makeBucket(bucket)
+	return nil
+}
+
+// Rmdir deletes the bucket if the fs is at the root
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
+	bucket, directory := f.split(dir)
+	if bucket == "" || directory != "" {
+		return nil
+	}
+	return buckets.deleteBucket(bucket)
+}
+
+// Precision of the remote
+func (f *Fs) Precision() time.Duration {
+	return time.Nanosecond
+}
+
+// Copy src to this remote using server side copy operations.
+//
+// This is stored with the remote path given
+//
+// It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+	dstBucket, dstPath := f.split(remote)
+	_ = buckets.makeBucket(dstBucket)
+	srcObj, ok := src.(*Object)
+	if !ok {
+		fs.Debugf(src, "Can't copy - not same remote type")
+		return nil, fs.ErrorCantCopy
+	}
+	srcBucket, srcPath := srcObj.split()
+	od := buckets.getObjectData(srcBucket, srcPath)
+	if od == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	buckets.updateObjectData(dstBucket, dstPath, od)
+	return f.NewObject(ctx, remote)
+}
+
+// Hashes returns the supported hash sets.
+func (f *Fs) Hashes() hash.Set {
+	return hash.Set(hashType)
+}
+
+// ------------------------------------------------------------
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.Remote()
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// Hash returns the hash of an object returning a lowercase hex string
+func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
+	if t != hashType {
+		return "", hash.ErrUnsupported
+	}
+	if o.od.hash == "" {
+		sum := md5.Sum(o.od.data)
+		o.od.hash = hex.EncodeToString(sum[:])
+	}
+	return o.od.hash, nil
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return int64(len(o.od.data))
+}
+
+// ModTime returns the modification time of the object
+//
+// It attempts to read the objects mtime and if that isn't present the
+// LastModified returned in the http headers
+//
+// SHA-1 will also be updated once the request has completed.
+func (o *Object) ModTime(ctx context.Context) (result time.Time) {
+	return o.od.modTime
+}
+
+// SetModTime sets the modification time of the local fs object
+func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
+	o.od.modTime = modTime
+	return nil
+}
+
+// Storable returns if this object is storable
+func (o *Object) Storable() bool {
+	return true
+}
+
+// Open an object for read
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
+	var offset, limit int64 = 0, -1
+	for _, option := range options {
+		switch x := option.(type) {
+		case *fs.RangeOption:
+			offset, limit = x.Decode(int64(len(o.od.data)))
+		case *fs.SeekOption:
+			offset = x.Offset
+		default:
+			if option.Mandatory() {
+				fs.Logf(o, "Unsupported mandatory option: %v", option)
+			}
+		}
+	}
+	if offset > int64(len(o.od.data)) {
+		offset = int64(len(o.od.data))
+	}
+	data := o.od.data[offset:]
+	if limit >= 0 {
+		if limit > int64(len(data)) {
+			limit = int64(len(data))
+		}
+		data = data[:limit]
+	}
+	return ioutil.NopCloser(bytes.NewBuffer(data)), nil
+}
+
+// Update the object with the contents of the io.Reader, modTime and size
+//
+// The new object may have been created if an error is returned
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	bucket, bucketPath := o.split()
+	data, err := ioutil.ReadAll(in)
+	if err != nil {
+		return errors.Wrap(err, "failed to update memory object")
+	}
+	o.od = &objectData{
+		data:     data,
+		hash:     "",
+		modTime:  src.ModTime(ctx),
+		mimeType: fs.MimeType(ctx, o),
+	}
+	buckets.updateObjectData(bucket, bucketPath, o.od)
+	return nil
+}
+
+// Remove an object
+func (o *Object) Remove(ctx context.Context) error {
+	bucket, bucketPath := o.split()
+	removed := buckets.removeObjectData(bucket, bucketPath)
+	if !removed {
+		return fs.ErrorObjectNotFound
+	}
+	return nil
+}
+
+// MimeType of an Object if known, "" otherwise
+func (o *Object) MimeType(ctx context.Context) string {
+	return o.od.mimeType
+}
+
+// Check the interfaces are satisfied
+var (
+	_ fs.Fs          = &Fs{}
+	_ fs.Copier      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.Object      = &Object{}
+	_ fs.MimeTyper   = &Object{}
+)

backend/memory/memory_test.go

@@ -0,0 +1,16 @@
+// Test memory filesystem interface
+package memory
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: ":memory:",
+		NilObject:  (*Object)(nil),
+	})
+}

backend/onedrive/onedrive.go

@@ -12,13 +12,13 @@ import (
 	"log"
 	"net/http"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/rclone/rclone/lib/atexit"
-
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/onedrive/api"
+	"github.com/rclone/rclone/backend/onedrive/quickxorhash"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
@@ -26,7 +26,9 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -58,15 +60,19 @@ var (
 			AuthURL:  "https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
 			TokenURL: "https://login.microsoftonline.com/common/oauth2/v2.0/token",
 		},
-		Scopes:       []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access"},
+		Scopes:       []string{"Files.Read", "Files.ReadWrite", "Files.Read.All", "Files.ReadWrite.All", "offline_access", "Sites.Read.All"},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
 		RedirectURL:  oauthutil.RedirectLocalhostURL,
 	}
+
+	// QuickXorHashType is the hash.Type for OneDrive
+	QuickXorHashType hash.Type
 )
 
 // Register with Fs
 func init() {
+	QuickXorHashType = hash.RegisterHash("QuickXorHash", 40, quickxorhash.New)
 	fs.Register(&fs.RegInfo{
 		Name:        "onedrive",
 		Description: "Microsoft OneDrive",
@@ -218,9 +224,9 @@ func init() {
 			Help: "Microsoft App Client Secret\nLeave blank normally.",
 		}, {
 			Name: "chunk_size",
-			Help: `Chunk size to upload files with - must be multiple of 320k.
+			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
 
-Above this size files will be chunked - must be multiple of 320k. Note
+Above this size files will be chunked - must be multiple of 320k (327,680 bytes). Note
 that the chunks will be buffered into memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
@@ -245,16 +251,63 @@ delete OneNote files or otherwise want them to show up in directory
 listing, set this option.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// List of replaced characters:
+			//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+			//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+			//   : (colon)         -> '：' // FULLWIDTH COLON
+			//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+			//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+			//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+			//   # (number sign)  -> '＃'  // FULLWIDTH NUMBER SIGN
+			//   % (percent sign) -> '％'  // FULLWIDTH PERCENT SIGN
+			//
+			// Folder names cannot begin with a tilde ('~')
+			// List of replaced characters:
+			//   ~ (tilde)        -> '～'  // FULLWIDTH TILDE
+			//
+			// Additionally names can't begin with a space ( ) or end with a period (.) or space ( ).
+			// List of replaced characters:
+			//   . (period)        -> '．' // FULLWIDTH FULL STOP
+			//     (space)         -> '␠'  // SYMBOL FOR SPACE
+			//
+			// Also encode invalid UTF-8 bytes as json doesn't handle them.
+			//
+			// The OneDrive API documentation lists the set of reserved characters, but
+			// testing showed this list is incomplete. This are the differences:
+			//  - " (double quote) is rejected, but missing in the documentation
+			//  - space at the end of file and folder names is rejected, but missing in the documentation
+			//  - period at the end of file names is rejected, but missing in the documentation
+			//
+			// Adding these restrictions to the OneDrive API documentation yields exactly
+			// the same rules as the Windows naming conventions.
+			//
+			// https://docs.microsoft.com/en-us/onedrive/developer/rest-api/concepts/addressing-driveitems?view=odsp-graph-online#path-encoding
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeHashPercent |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeLeftTilde |
+				encoder.EncodeRightPeriod |
+				encoder.EncodeRightSpace |
+				encoder.EncodeWin |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize          fs.SizeSuffix `config:"chunk_size"`
-	DriveID            string        `config:"drive_id"`
-	DriveType          string        `config:"drive_type"`
-	ExposeOneNoteFiles bool          `config:"expose_onenote_files"`
+	ChunkSize          fs.SizeSuffix        `config:"chunk_size"`
+	DriveID            string               `config:"drive_id"`
+	DriveType          string               `config:"drive_type"`
+	ExposeOneNoteFiles bool                 `config:"expose_onenote_files"`
+	Enc                encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote one drive
@@ -328,13 +381,30 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRetry := false
-
-	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRetry = true
-		fs.Debugf(nil, "Should retry: %v", err)
+	retry := false
+	if resp != nil {
+		switch resp.StatusCode {
+		case 401:
+			if len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
+				retry = true
+				fs.Debugf(nil, "Should retry: %v", err)
+			}
+		case 429: // Too Many Requests.
+			// see https://docs.microsoft.com/en-us/sharepoint/dev/general-development/how-to-avoid-getting-throttled-or-blocked-in-sharepoint-online
+			if values := resp.Header["Retry-After"]; len(values) == 1 && values[0] != "" {
+				retryAfter, parseErr := strconv.Atoi(values[0])
+				if parseErr != nil {
+					fs.Debugf(nil, "Failed to parse Retry-After: %q: %v", values[0], parseErr)
+				} else {
+					duration := time.Second * time.Duration(retryAfter)
+					retry = true
+					err = pacer.RetryAfterError(err, duration)
+					fs.Debugf(nil, "Too many requests. Trying again in %d seconds.", retryAfter)
+				}
+			}
+		}
 	}
-	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return retry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
@@ -344,8 +414,13 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // instead of simply using `drives/driveID/root:/itemPath` because it works for
 // "shared with me" folders in OneDrive Personal (See #2536, #2778)
 // This path pattern comes from https://github.com/OneDrive/onedrive-api-docs/issues/908#issuecomment-417488480
+//
+// If `relPath` == '', do not append the slash (See #3664)
 func (f *Fs) readMetaDataForPathRelativeToID(ctx context.Context, normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
-	opts := newOptsCall(normalizedID, "GET", ":/"+withTrailingColon(rest.URLPathEscape(replaceReservedChars(relPath))))
+	if relPath != "" {
+		relPath = "/" + withTrailingColon(rest.URLPathEscape(f.opt.Enc.FromStandardPath(relPath)))
+	}
+	opts := newOptsCall(normalizedID, "GET", ":"+relPath)
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -368,7 +443,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 		} else {
 			opts = rest.Opts{
 				Method: "GET",
-				Path:   "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
+				Path:   "/root:/" + rest.URLPathEscape(f.opt.Enc.FromStandardPath(path)),
 			}
 		}
 		err = f.pacer.Call(func() (bool, error) {
@@ -616,7 +691,7 @@ func (f *Fs) CreateDir(ctx context.Context, dirID, leaf string) (newID string, e
 	var info *api.Item
 	opts := newOptsCall(dirID, "POST", "/children")
 	mkdir := api.CreateItemRequest{
-		Name:             replaceReservedChars(leaf),
+		Name:             f.opt.Enc.FromStandardName(leaf),
 		ConflictBehavior: "fail",
 	}
 	err = f.pacer.Call(func() (bool, error) {
@@ -676,7 +751,7 @@ OUTER:
 			if item.Deleted != nil {
 				continue
 			}
-			item.Name = restoreReservedChars(item.GetName())
+			item.Name = f.opt.Enc.ToStandardName(item.GetName())
 			if fn(item) {
 				found = true
 				break OUTER
@@ -913,8 +988,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	srcPath := srcObj.fs.rootSlash() + srcObj.remote
-	dstPath := f.rootSlash() + remote
+	srcPath := srcObj.rootPath()
+	dstPath := f.rootPath(remote)
 	if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
 		return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
 	}
@@ -932,7 +1007,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	id, dstDriveID, _ := parseNormalizedID(directoryID)
 
-	replacedLeaf := replaceReservedChars(leaf)
+	replacedLeaf := f.opt.Enc.FromStandardName(leaf)
 	copyReq := api.CopyItemRequest{
 		Name: &replacedLeaf,
 		ParentReference: api.ItemReference{
@@ -1016,7 +1091,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	opts := newOptsCall(srcObj.id, "PATCH", "")
 
 	move := api.MoveItemRequest{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		ParentReference: &api.ItemReference{
 			DriveID: dstDriveID,
 			ID:      id,
@@ -1131,7 +1206,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	// Do the move
 	opts := newOptsCall(srcID, "PATCH", "")
 	move := api.MoveItemRequest{
-		Name: replaceReservedChars(leaf),
+		Name: f.opt.Enc.FromStandardName(leaf),
 		ParentReference: &api.ItemReference{
 			DriveID: dstDriveID,
 			ID:      parsedDstDirID,
@@ -1192,12 +1267,12 @@ func (f *Fs) Hashes() hash.Set {
 	if f.driveType == driveTypePersonal {
 		return hash.Set(hash.SHA1)
 	}
-	return hash.Set(hash.QuickXorHash)
+	return hash.Set(QuickXorHashType)
 }
 
 // PublicLink returns a link for downloading without accout.
 func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
-	info, _, err := f.readMetaDataForPath(ctx, f.srvPath(remote))
+	info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
 	if err != nil {
 		return "", err
 	}
@@ -1241,9 +1316,19 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
+// rootPath returns a path for use in server given a remote
+func (f *Fs) rootPath(remote string) string {
+	return f.rootSlash() + remote
+}
+
+// rootPath returns a path for use in local functions
+func (o *Object) rootPath() string {
+	return o.fs.rootPath(o.remote)
+}
+
 // srvPath returns a path for use in server given a remote
 func (f *Fs) srvPath(remote string) string {
-	return replaceReservedChars(f.rootSlash() + remote)
+	return f.opt.Enc.FromStandardPath(f.rootSlash() + remote)
 }
 
 // srvPath returns a path for use in server
@@ -1258,7 +1343,7 @@ func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 			return o.sha1, nil
 		}
 	} else {
-		if t == hash.QuickXorHash {
+		if t == QuickXorHashType {
 			return o.quickxorhash, nil
 		}
 	}
@@ -1320,7 +1405,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.hasMetaData {
 		return nil
 	}
-	info, _, err := o.fs.readMetaDataForPath(ctx, o.srvPath())
+	info, _, err := o.fs.readMetaDataForPath(ctx, o.rootPath())
 	if err != nil {
 		if apiErr, ok := err.(*api.Error); ok {
 			if apiErr.ErrorInfo.Code == "itemNotFound" {
@@ -1355,7 +1440,7 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(leaf)),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + withTrailingColon(rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 	} else {
 		opts = rest.Opts{
@@ -1429,7 +1514,8 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 		opts = rest.Opts{
 			Method:  "POST",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/items/" + id + ":/" + rest.URLPathEscape(replaceReservedChars(leaf)) + ":/createUploadSession",
+			Path: fmt.Sprintf("/%s/items/%s:/%s:/createUploadSession",
+				drive, id, rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 	} else {
 		opts = rest.Opts{
@@ -1454,21 +1540,74 @@ func (o *Object) createUploadSession(ctx context.Context, modTime time.Time) (re
 	return response, err
 }
 
+// getPosition gets the current position in a multipart upload
+func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err error) {
+	opts := rest.Opts{
+		Method:  "GET",
+		RootURL: url,
+	}
+	var info api.UploadFragmentResponse
+	var resp *http.Response
+	err = o.fs.pacer.Call(func() (bool, error) {
+		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &info)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return 0, err
+	}
+	if len(info.NextExpectedRanges) != 1 {
+		return 0, errors.Errorf("bad number of ranges in upload position: %v", info.NextExpectedRanges)
+	}
+	position := info.NextExpectedRanges[0]
+	i := strings.IndexByte(position, '-')
+	if i < 0 {
+		return 0, errors.Errorf("no '-' in next expected range: %q", position)
+	}
+	position = position[:i]
+	pos, err = strconv.ParseInt(position, 10, 64)
+	if err != nil {
+		return 0, errors.Wrapf(err, "bad expected range: %q", position)
+	}
+	return pos, nil
+}
+
 // uploadFragment uploads a part
 func (o *Object) uploadFragment(ctx context.Context, url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) {
-	opts := rest.Opts{
-		Method:        "PUT",
-		RootURL:       url,
-		ContentLength: &chunkSize,
-		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", start, start+chunkSize-1, totalSize),
-		Body:          chunk,
-	}
 	//	var response api.UploadFragmentResponse
 	var resp *http.Response
 	var body []byte
+	var skip = int64(0)
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, _ = chunk.Seek(0, io.SeekStart)
+		toSend := chunkSize - skip
+		opts := rest.Opts{
+			Method:        "PUT",
+			RootURL:       url,
+			ContentLength: &toSend,
+			ContentRange:  fmt.Sprintf("bytes %d-%d/%d", start+skip, start+chunkSize-1, totalSize),
+			Body:          chunk,
+		}
+		_, _ = chunk.Seek(skip, io.SeekStart)
 		resp, err = o.fs.srv.Call(ctx, &opts)
+		if err != nil && resp != nil && resp.StatusCode == http.StatusRequestedRangeNotSatisfiable {
+			fs.Debugf(o, "Received 416 error - reading current position from server: %v", err)
+			pos, posErr := o.getPosition(ctx, url)
+			if posErr != nil {
+				fs.Debugf(o, "Failed to read position: %v", posErr)
+				return false, posErr
+			}
+			skip = pos - start
+			fs.Debugf(o, "Read position %d, chunk is %d..%d, bytes to skip = %d", pos, start, start+chunkSize, skip)
+			switch {
+			case skip < 0:
+				return false, errors.Wrapf(err, "sent block already (skip %d < 0), can't rewind", skip)
+			case skip > chunkSize:
+				return false, errors.Wrapf(err, "position is in the future (skip %d > chunkSize %d), can't skip forward", skip, chunkSize)
+			case skip == chunkSize:
+				fs.Debugf(o, "Skipping chunk as already sent (skip %d == chunkSize %d)", skip, chunkSize)
+				return false, nil
+			}
+			return true, errors.Wrapf(err, "retry this chunk skipping %d bytes", skip)
+		}
 		if err != nil {
 			return shouldRetry(resp, err)
 		}
@@ -1581,7 +1720,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 		opts = rest.Opts{
 			Method:        "PUT",
 			RootURL:       rootURL,
-			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf) + ":/content",
+			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf)) + ":/content",
 			ContentLength: &size,
 			Body:          in,
 		}

backend/onedrive/replace.go

@@ -1,91 +0,0 @@
-/*
-Translate file names for one drive
-
-OneDrive reserved characters
-
-The following characters are OneDrive reserved characters, and can't
-be used in OneDrive folder and file names.
-
-  onedrive-reserved  = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|"
-  onedrive-business-reserved
-                     = "/" / "\" / "*" / "<" / ">" / "?" / ":" / "|" / "#" / "%"
-
-Note: Folder names can't end with a period (.).
-
-Note: OneDrive for Business file or folder names cannot begin with a
-tilde ('~').
-
-*/
-
-package onedrive
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// Onedrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		'*':  '＊', // FULLWIDTH ASTERISK
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'?':  '？', // FULLWIDTH QUESTION MARK
-		':':  '：', // FULLWIDTH COLON
-		'|':  '｜', // FULLWIDTH VERTICAL LINE
-		'#':  '＃', // FULLWIDTH NUMBER SIGN
-		'%':  '％', // FULLWIDTH PERCENT SIGN
-		'"':  '＂', // FULLWIDTH QUOTATION MARK - not on the list but seems to be reserved
-		'.':  '．', // FULLWIDTH FULL STOP
-		'~':  '～', // FULLWIDTH TILDE
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	invCharMap           map[rune]rune
-	fixEndingInPeriod    = regexp.MustCompile(`\.(/|$)`)
-	fixStartingWithTilde = regexp.MustCompile(`(/|^)~`)
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// Folder names can't end with a period '.'
-	in = fixEndingInPeriod.ReplaceAllString(in, string(charMap['.'])+"$1")
-	// OneDrive for Business file or folder names cannot begin with a tilde '~'
-	in = fixStartingWithTilde.ReplaceAllString(in, "$1"+string(charMap['~']))
-	// Apparently file names can't start with space either
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Replace reserved characters
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != '.' && c != '~' && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/onedrive/replace_test.go

@@ -1,30 +0,0 @@
-package onedrive
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{`\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~`},
-		{`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜＃％＂.~/＼＊＜＞？：｜＃％＂.~`},
-		{" leading space", "␠leading space"},
-		{"~leading tilde", "～leading tilde"},
-		{"trailing dot.", "trailing dot．"},
-		{" leading space/ leading space/ leading space", "␠leading space/␠leading space/␠leading space"},
-		{"~leading tilde/~leading tilde/~leading tilde", "～leading tilde/～leading tilde/～leading tilde"},
-		{"trailing dot./trailing dot./trailing dot.", "trailing dot．/trailing dot．/trailing dot．"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/opendrive/opendrive.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -20,6 +21,7 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
@@ -47,14 +49,57 @@ func init() {
 			Help:       "Password.",
 			IsPassword: true,
 			Required:   true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// List of replaced characters:
+			//   < (less than)     -> '＜' // FULLWIDTH LESS-THAN SIGN
+			//   > (greater than)  -> '＞' // FULLWIDTH GREATER-THAN SIGN
+			//   : (colon)         -> '：' // FULLWIDTH COLON
+			//   " (double quote)  -> '＂' // FULLWIDTH QUOTATION MARK
+			//   \ (backslash)     -> '＼' // FULLWIDTH REVERSE SOLIDUS
+			//   | (vertical line) -> '｜' // FULLWIDTH VERTICAL LINE
+			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
+			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
+			//
+			// Additionally names can't begin or end with a ASCII whitespace.
+			// List of replaced characters:
+			//     (space)           -> '␠'  // SYMBOL FOR SPACE
+			//     (horizontal tab)  -> '␉'  // SYMBOL FOR HORIZONTAL TABULATION
+			//     (line feed)       -> '␊'  // SYMBOL FOR LINE FEED
+			//     (vertical tab)    -> '␋'  // SYMBOL FOR VERTICAL TABULATION
+			//     (carriage return) -> '␍'  // SYMBOL FOR CARRIAGE RETURN
+			//
+			// Also encode invalid UTF-8 bytes as json doesn't handle them properly.
+			//
+			// https://www.opendrive.com/wp-content/uploads/guides/OpenDrive_API_guide.pdf
+			Default: (encoder.Base |
+				encoder.EncodeWin |
+				encoder.EncodeLeftCrLfHtVt |
+				encoder.EncodeRightCrLfHtVt |
+				encoder.EncodeBackSlash |
+				encoder.EncodeLeftSpace |
+				encoder.EncodeRightSpace |
+				encoder.EncodeInvalidUtf8),
+		}, {
+			Name: "chunk_size",
+			Help: `Files will be uploaded in chunks this size.
+
+Note that these chunks are buffered in memory so increasing them will
+increase memory use.`,
+			Default:  10 * fs.MebiByte,
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	UserName string `config:"username"`
-	Password string `config:"password"`
+	UserName  string               `config:"username"`
+	Password  string               `config:"password"`
+	Enc       encoder.MultiEncoder `config:"encoding"`
+	ChunkSize fs.SizeSuffix        `config:"chunk_size"`
 }
 
 // Fs represents a remote server
@@ -585,7 +630,7 @@ func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time,
 		fs:     f,
 		remote: remote,
 	}
-	return o, leaf, directoryID, nil
+	return o, f.opt.Enc.FromStandardName(leaf), directoryID, nil
 }
 
 // readMetaDataForPath reads the metadata from the path
@@ -636,7 +681,11 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		var resp *http.Response
 		response := createFileResponse{}
 		err := o.fs.pacer.Call(func() (bool, error) {
-			createFileData := createFile{SessionID: o.fs.session.SessionID, FolderID: directoryID, Name: replaceReservedChars(leaf)}
+			createFileData := createFile{
+				SessionID: o.fs.session.SessionID,
+				FolderID:  directoryID,
+				Name:      leaf,
+			}
 			opts := rest.Opts{
 				Method: "POST",
 				Path:   "/upload/create_file.json",
@@ -683,7 +732,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	err = f.pacer.Call(func() (bool, error) {
 		createDirData := createFolder{
 			SessionID:           f.session.SessionID,
-			FolderName:          replaceReservedChars(leaf),
+			FolderName:          f.opt.Enc.FromStandardName(leaf),
 			FolderSubParent:     pathID,
 			FolderIsPublic:      0,
 			FolderPublicUpl:     0,
@@ -729,8 +778,8 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 		return "", false, errors.Wrap(err, "failed to get folder list")
 	}
 
+	leaf = f.opt.Enc.FromStandardName(leaf)
 	for _, folder := range folderList.Folders {
-		folder.Name = restoreReservedChars(folder.Name)
 		// fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID)
 
 		if leaf == folder.Name {
@@ -777,7 +826,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	for _, folder := range folderList.Folders {
-		folder.Name = restoreReservedChars(folder.Name)
+		folder.Name = f.opt.Enc.ToStandardName(folder.Name)
 		// fs.Debugf(nil, "Folder: %s (%s)", folder.Name, folder.FolderID)
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
@@ -788,7 +837,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 	}
 
 	for _, file := range folderList.Files {
-		file.Name = restoreReservedChars(file.Name)
+		file.Name = f.opt.Enc.ToStandardName(file.Name)
 		// fs.Debugf(nil, "File: %s (%s)", file.Name, file.FileID)
 		remote := path.Join(dir, file.Name)
 		o, err := f.newObjectWithInfo(ctx, remote, &file)
@@ -851,7 +900,11 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 		NoResponse: true,
 		Path:       "/file/filesettings.json",
 	}
-	update := modTimeFile{SessionID: o.fs.session.SessionID, FileID: o.id, FileModificationTime: strconv.FormatInt(modTime.Unix(), 10)}
+	update := modTimeFile{
+		SessionID:            o.fs.session.SessionID,
+		FileID:               o.id,
+		FileModificationTime: strconv.FormatInt(modTime.Unix(), 10),
+	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, nil)
 		return o.fs.shouldRetry(resp, err)
@@ -929,15 +982,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// resp.Body.Close()
 	// fs.Debugf(nil, "PostOpen: %#v", openResponse)
 
-	// 10 MB chunks size
-	chunkSize := int64(1024 * 1024 * 10)
-	buf := make([]byte, int(chunkSize))
+	buf := make([]byte, o.fs.opt.ChunkSize)
 	chunkOffset := int64(0)
 	remainingBytes := size
 	chunkCounter := 0
 
 	for remainingBytes > 0 {
-		currentChunkSize := chunkSize
+		currentChunkSize := int64(o.fs.opt.ChunkSize)
 		if currentChunkSize > remainingBytes {
 			currentChunkSize = remainingBytes
 		}
@@ -1038,7 +1089,8 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		opts := rest.Opts{
 			Method: "GET",
-			Path:   "/folder/itembyname.json/" + o.fs.session.SessionID + "/" + directoryID + "?name=" + url.QueryEscape(replaceReservedChars(leaf)),
+			Path: fmt.Sprintf("/folder/itembyname.json/%s/%s?name=%s",
+				o.fs.session.SessionID, directoryID, url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf))),
 		}
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &folderList)
 		return o.fs.shouldRetry(resp, err)

backend/opendrive/replace.go

@@ -1,78 +0,0 @@
-/*
-Translate file names for OpenDrive
-
-OpenDrive reserved characters
-
-The following characters are OpenDrive reserved characters, and can't
-be used in OpenDrive folder and file names.
-
-\ / : * ? " < > |
-
-OpenDrive files and folders can't have leading or trailing spaces also.
-
-*/
-
-package opendrive
-
-import (
-	"regexp"
-	"strings"
-)
-
-// charMap holds replacements for characters
-//
-// OpenDrive has a restricted set of characters compared to other cloud
-// storage systems, so we to map these to the FULLWIDTH unicode
-// equivalents
-//
-// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
-var (
-	charMap = map[rune]rune{
-		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
-		':':  '：', // FULLWIDTH COLON
-		'*':  '＊', // FULLWIDTH ASTERISK
-		'?':  '？', // FULLWIDTH QUESTION MARK
-		'"':  '＂', // FULLWIDTH QUOTATION MARK
-		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
-		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
-		'|':  '｜', // FULLWIDTH VERTICAL LINE
-		' ':  '␠', // SYMBOL FOR SPACE
-	}
-	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
-	fixEndingWithSpace   = regexp.MustCompile(` (/|$)`)
-	invCharMap           map[rune]rune
-)
-
-func init() {
-	// Create inverse charMap
-	invCharMap = make(map[rune]rune, len(charMap))
-	for k, v := range charMap {
-		invCharMap[v] = k
-	}
-}
-
-// replaceReservedChars takes a path and substitutes any reserved
-// characters in it
-func replaceReservedChars(in string) string {
-	// Filenames can't start with space
-	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
-	// Filenames can't end with space
-	in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := charMap[c]; ok && c != ' ' {
-			return replacement
-		}
-		return c
-	}, in)
-}
-
-// restoreReservedChars takes a path and undoes any substitutions
-// made by replaceReservedChars
-func restoreReservedChars(in string) string {
-	return strings.Map(func(c rune) rune {
-		if replacement, ok := invCharMap[c]; ok {
-			return replacement
-		}
-		return c
-	}, in)
-}

backend/opendrive/replace_test.go

@@ -1,28 +0,0 @@
-package opendrive
-
-import "testing"
-
-func TestReplace(t *testing.T) {
-	for _, test := range []struct {
-		in  string
-		out string
-	}{
-		{"", ""},
-		{"abc 123", "abc 123"},
-		{`\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~`},
-		{`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~/＼＊＜＞？：｜#%＂.~`},
-		{" leading space", "␠leading space"},
-		{" path/  leading spaces", "␠path/␠ leading spaces"},
-		{"trailing space ", "trailing space␠"},
-		{"trailing spaces  /path ", "trailing spaces ␠/path␠"},
-	} {
-		got := replaceReservedChars(test.in)
-		if got != test.out {
-			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
-		}
-		got2 := restoreReservedChars(got)
-		if got2 != test.in {
-			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
-		}
-	}
-}

backend/pcloud/pcloud.go

@@ -29,6 +29,7 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/rest"
@@ -78,12 +79,23 @@ func init() {
 		}, {
 			Name: config.ConfigClientSecret,
 			Help: "Pcloud App Client Secret\nLeave blank normally.",
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			//
+			// TODO: Investigate Unicode simplification (＼ gets converted to \ server-side)
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
+	Enc encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote pcloud
@@ -175,21 +187,6 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return doRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// substitute reserved characters for pcloud
-//
-// Generally all characters are allowed in filenames, except the NULL
-// byte, forward and backslash (/,\ and \0)
-func replaceReservedChars(x string) string {
-	// Backslash for FULLWIDTH REVERSE SOLIDUS
-	return strings.Replace(x, "\\", "＼", -1)
-}
-
-// restore reserved characters for pcloud
-func restoreReservedChars(x string) string {
-	// FULLWIDTH REVERSE SOLIDUS for Backslash
-	return strings.Replace(x, "＼", "\\", -1)
-}
-
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
@@ -354,7 +351,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Path:       "/createfolder",
 		Parameters: url.Values{},
 	}
-	opts.Parameters.Set("name", replaceReservedChars(leaf))
+	opts.Parameters.Set("name", f.opt.Enc.FromStandardName(leaf))
 	opts.Parameters.Set("folderid", dirIDtoNumber(pathID))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
@@ -430,7 +427,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 				continue
 			}
 		}
-		item.Name = restoreReservedChars(item.Name)
+		item.Name = f.opt.Enc.ToStandardName(item.Name)
 		if fn(item) {
 			found = true
 			break
@@ -622,7 +619,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Parameters: url.Values{},
 	}
 	opts.Parameters.Set("fileid", fileIDtoNumber(srcObj.id))
-	opts.Parameters.Set("toname", replaceReservedChars(leaf))
+	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(leaf))
 	opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID))
 	opts.Parameters.Set("mtime", fmt.Sprintf("%d", srcObj.modTime.Unix()))
 	var resp *http.Response
@@ -701,7 +698,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		Parameters: url.Values{},
 	}
 	opts.Parameters.Set("fileid", fileIDtoNumber(srcObj.id))
-	opts.Parameters.Set("toname", replaceReservedChars(leaf))
+	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(leaf))
 	opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID))
 	var resp *http.Response
 	var result api.ItemResult
@@ -798,7 +795,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		Parameters: url.Values{},
 	}
 	opts.Parameters.Set("folderid", dirIDtoNumber(srcID))
-	opts.Parameters.Set("toname", replaceReservedChars(leaf))
+	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(leaf))
 	opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID))
 	var resp *http.Response
 	var result api.ItemResult
@@ -1078,7 +1075,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Parameters:       url.Values{},
 		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
 	}
-	leaf = replaceReservedChars(leaf)
+	leaf = o.fs.opt.Enc.FromStandardName(leaf)
 	opts.Parameters.Set("filename", leaf)
 	opts.Parameters.Set("folderid", dirIDtoNumber(directoryID))
 	opts.Parameters.Set("nopartial", "1")

backend/premiumizeme/premiumizeme.go

@@ -2,9 +2,7 @@
 // object storage system.
 package premiumizeme
 
-/* FIXME
-escaping needs fixing
-
+/*
 Run of rclone info
 stringNeedsEscaping = []rune{
 	0x00, 0x0A, 0x0D, 0x22, 0x2F, 0x5C, 0xBF, 0xFE
@@ -33,6 +31,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/premiumizeme/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -40,6 +39,7 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/random"
@@ -92,13 +92,23 @@ This is not normally used - use oauth instead.
 `,
 			Hide:    fs.OptionHideBoth,
 			Default: "",
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeDoubleQuote |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	APIKey string `config:"api_key"`
+	APIKey string               `config:"api_key"`
+	Enc    encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote cloud storage system
@@ -170,24 +180,6 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// substitute reserved characters
-func replaceReservedChars(x string) string {
-	// Backslash for FULLWIDTH REVERSE SOLIDUS
-	x = strings.Replace(x, "\\", "＼", -1)
-	// Double quote for FULLWIDTH QUOTATION MARK
-	x = strings.Replace(x, `"`, "＂", -1)
-	return x
-}
-
-// restore reserved characters
-func restoreReservedChars(x string) string {
-	// FULLWIDTH QUOTATION MARK for Double quote
-	x = strings.Replace(x, "＂", `"`, -1)
-	// FULLWIDTH REVERSE SOLIDUS for Backslash
-	x = strings.Replace(x, "＼", "\\", -1)
-	return x
-}
-
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
@@ -323,14 +315,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
-// rootSlash returns root with a slash on if it is empty, otherwise empty string
-func (f *Fs) rootSlash() string {
-	if f.root == "" {
-		return f.root
-	}
-	return f.root + "/"
-}
-
 // Return an Object from a path
 //
 // If it can't be found it returns the error fs.ErrorObjectNotFound.
@@ -381,7 +365,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 		Path:       "/folder/create",
 		Parameters: f.baseParams(),
 		MultipartParams: url.Values{
-			"name":      {replaceReservedChars(leaf)},
+			"name":      {f.opt.Enc.FromStandardName(leaf)},
 			"parent_id": {pathID},
 		},
 	}
@@ -446,7 +430,7 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 			fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)
 			continue
 		}
-		item.Name = restoreReservedChars(item.Name)
+		item.Name = f.opt.Enc.ToStandardName(item.Name)
 		if fn(item) {
 			found = true
 			break
@@ -654,8 +638,8 @@ func (f *Fs) Purge(ctx context.Context) error {
 // between directories and a separate one to rename them.  We try to
 // call the minimum number of API calls.
 func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDirectoryID, newDirectoryID string) (err error) {
-	newLeaf = replaceReservedChars(newLeaf)
-	oldLeaf = replaceReservedChars(oldLeaf)
+	newLeaf = f.opt.Enc.FromStandardName(newLeaf)
+	oldLeaf = f.opt.Enc.FromStandardName(oldLeaf)
 	doRenameLeaf := oldLeaf != newLeaf
 	doMove := oldDirectoryID != newDirectoryID
 
@@ -686,7 +670,7 @@ func (f *Fs) move(ctx context.Context, isFile bool, id, oldLeaf, newLeaf, oldDir
 		} else {
 			opts.MultipartParams.Set("folders[]", id)
 		}
-		//replacedLeaf := replaceReservedChars(leaf)
+		//replacedLeaf := enc.FromStandardName(leaf)
 		var resp *http.Response
 		var result api.Response
 		err = f.pacer.Call(func() (bool, error) {
@@ -906,11 +890,6 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// srvPath returns a path for use in server
-func (o *Object) srvPath() string {
-	return replaceReservedChars(o.fs.rootSlash() + o.remote)
-}
-
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
 	return "", hash.ErrUnsupported
@@ -1001,14 +980,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return resp.Body, err
 }
 
-// metaHash returns a rough hash of metadata to detect if object has been updated
-func (o *Object) metaHash() string {
-	if !o.hasMetaData {
-		return ""
-	}
-	return fmt.Sprintf("remote=%q, size=%d, modTime=%v, id=%q, mimeType=%q", o.remote, o.size, o.modTime, o.id, o.mimeType)
-}
-
 // Update the object with the contents of the io.Reader, modTime and size
 //
 // If existing is set then it updates the object rather than creating a new one
@@ -1023,7 +994,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return err
 	}
-	leaf = replaceReservedChars(leaf)
+	leaf = o.fs.opt.Enc.FromStandardName(leaf)
 
 	var resp *http.Response
 	var info api.FolderUploadinfoResponse

backend/putio/error.go

@@ -0,0 +1,43 @@
+package putio
+
+import (
+	"fmt"
+	"net/http"
+
+	"github.com/putdotio/go-putio/putio"
+	"github.com/rclone/rclone/fs/fserrors"
+)
+
+func checkStatusCode(resp *http.Response, expected int) error {
+	if resp.StatusCode != expected {
+		return &statusCodeError{response: resp}
+	}
+	return nil
+}
+
+type statusCodeError struct {
+	response *http.Response
+}
+
+func (e *statusCodeError) Error() string {
+	return fmt.Sprintf("unexpected status code (%d) response while doing %s to %s", e.response.StatusCode, e.response.Request.Method, e.response.Request.URL.String())
+}
+
+func (e *statusCodeError) Temporary() bool {
+	return e.response.StatusCode == 429 || e.response.StatusCode >= 500
+}
+
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(err error) (bool, error) {
+	if err == nil {
+		return false, nil
+	}
+	if perr, ok := err.(*putio.ErrorResponse); ok {
+		err = &statusCodeError{response: perr.Response}
+	}
+	if fserrors.ShouldRetry(err) {
+		return true, err
+	}
+	return false, err
+}

backend/putio/fs.go

@@ -17,7 +17,7 @@ import (
 	"github.com/putdotio/go-putio/putio"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
-	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/oauthutil"
@@ -30,6 +30,7 @@ type Fs struct {
 	name        string             // name of this remote
 	root        string             // the path we are working on
 	features    *fs.Features       // optional features
+	opt         Options            // options for this Fs
 	client      *putio.Client      // client for making API calls to Put.io
 	pacer       *fs.Pacer          // To pace the API calls
 	dirCache    *dircache.DirCache // Map of directory path to directory id
@@ -58,26 +59,15 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry returns a boolean as to whether this err deserves to be
-// retried.  It returns the err as a convenience
-func shouldRetry(err error) (bool, error) {
-	if err == nil {
-		return false, nil
-	}
-	if fserrors.ShouldRetry(err) {
-		return true, err
-	}
-	if perr, ok := err.(*putio.ErrorResponse); ok {
-		if perr.Response.StatusCode == 429 || perr.Response.StatusCode >= 500 {
-			return true, err
-		}
-	}
-	return false, err
-}
-
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) {
 	// defer log.Trace(name, "root=%v", root)("f=%+v, err=%v", &f, &err)
+	// Parse config into Options struct
+	opt := new(Options)
+	err = configstruct.Set(m, opt)
+	if err != nil {
+		return nil, err
+	}
 	oAuthClient, _, err := oauthutil.NewClient(name, m, putioConfig)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure putio")
@@ -85,6 +75,7 @@ func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) {
 	p := &Fs{
 		name:        name,
 		root:        root,
+		opt:         *opt,
 		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		client:      putio.NewClient(oAuthClient),
 		oAuthClient: oAuthClient,
@@ -145,7 +136,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	var entry putio.File
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "creating folder. part: %s, parentID: %d", leaf, parentID)
-		entry, err = f.client.Files.CreateFolder(ctx, leaf, parentID)
+		entry, err = f.client.Files.CreateFolder(ctx, f.opt.Enc.FromStandardName(leaf), parentID)
 		return shouldRetry(err)
 	})
 	return itoa(entry.ID), err
@@ -172,11 +163,11 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 		return
 	}
 	for _, child := range children {
-		if child.Name == leaf {
+		if f.opt.Enc.ToStandardName(child.Name) == leaf {
 			found = true
 			pathIDOut = itoa(child.ID)
 			if !child.IsDir() {
-				err = fs.ErrorNotAFile
+				err = fs.ErrorIsFile
 			}
 			return
 		}
@@ -214,7 +205,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return
 	}
 	for _, child := range children {
-		remote := path.Join(dir, child.Name)
+		remote := path.Join(dir, f.opt.Enc.ToStandardName(child.Name))
 		// fs.Debugf(f, "child: %s", remote)
 		if child.IsDir() {
 			f.dirCache.Put(remote, itoa(child.ID))
@@ -292,7 +283,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
 		req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
 		req.Header.Set("tus-resumable", "1.0.0")
 		req.Header.Set("upload-length", strconv.FormatInt(size, 10))
-		b64name := base64.StdEncoding.EncodeToString([]byte(name))
+		b64name := base64.StdEncoding.EncodeToString([]byte(f.opt.Enc.FromStandardName(name)))
 		b64true := base64.StdEncoding.EncodeToString([]byte("true"))
 		b64parentID := base64.StdEncoding.EncodeToString([]byte(parentID))
 		b64modifiedAt := base64.StdEncoding.EncodeToString([]byte(modTime.Format(time.RFC3339)))
@@ -318,66 +309,125 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
 }
 
 func (f *Fs) sendUpload(ctx context.Context, location string, size int64, in io.Reader) (fileID int64, err error) {
-	// defer log.Trace(f, "location=%v, size=%v", location, size)("fileID=%v, err=%v", fileID, &err)
+	// defer log.Trace(f, "location=%v, size=%v", location, size)("fileID=%v, err=%v", &fileID, &err)
 	if size == 0 {
 		err = f.pacer.Call(func() (bool, error) {
 			fs.Debugf(f, "Sending zero length chunk")
-			fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
+			_, fileID, err = f.transferChunk(ctx, location, 0, bytes.NewReader([]byte{}), 0)
 			return shouldRetry(err)
 		})
 		return
 	}
-	var start int64
+	var clientOffset int64
+	var offsetMismatch bool
 	buf := make([]byte, defaultChunkSize)
-	for start < size {
-		reqSize := size - start
-		if reqSize >= int64(defaultChunkSize) {
-			reqSize = int64(defaultChunkSize)
+	for clientOffset < size {
+		chunkSize := size - clientOffset
+		if chunkSize >= int64(defaultChunkSize) {
+			chunkSize = int64(defaultChunkSize)
 		}
-		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, reqSize)
+		chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
+		chunkStart := clientOffset
+		reqSize := chunkSize
+		transferOffset := clientOffset
+		fs.Debugf(f, "chunkStart: %d, reqSize: %d", chunkStart, reqSize)
 
 		// Transfer the chunk
 		err = f.pacer.Call(func() (bool, error) {
-			fs.Debugf(f, "Sending chunk. start: %d length: %d", start, reqSize)
-			// TODO get file offset and seek to the position
-			fileID, err = f.transferChunk(ctx, location, start, chunk, reqSize)
+			if offsetMismatch {
+				// Get file offset and seek to the position
+				offset, err := f.getServerOffset(ctx, location)
+				if err != nil {
+					return shouldRetry(err)
+				}
+				sentBytes := offset - chunkStart
+				fs.Debugf(f, "sentBytes: %d", sentBytes)
+				_, err = chunk.Seek(sentBytes, io.SeekStart)
+				if err != nil {
+					return shouldRetry(err)
+				}
+				transferOffset = offset
+				reqSize = chunkSize - sentBytes
+				offsetMismatch = false
+			}
+			fs.Debugf(f, "Sending chunk. transferOffset: %d length: %d", transferOffset, reqSize)
+			var serverOffset int64
+			serverOffset, fileID, err = f.transferChunk(ctx, location, transferOffset, chunk, reqSize)
+			if cerr, ok := err.(*statusCodeError); ok && cerr.response.StatusCode == 409 {
+				offsetMismatch = true
+				return true, err
+			}
+			if serverOffset != (transferOffset + reqSize) {
+				offsetMismatch = true
+				return true, errors.New("connection broken")
+			}
 			return shouldRetry(err)
 		})
 		if err != nil {
 			return
 		}
 
-		start += reqSize
+		clientOffset += chunkSize
 	}
 	return
 }
 
-func (f *Fs) transferChunk(ctx context.Context, location string, start int64, chunk io.ReadSeeker, chunkSize int64) (fileID int64, err error) {
-	// defer log.Trace(f, "location=%v, start=%v, chunkSize=%v", location, start, chunkSize)("fileID=%v, err=%v", fileID, &err)
-	_, _ = chunk.Seek(0, io.SeekStart)
+func (f *Fs) getServerOffset(ctx context.Context, location string) (offset int64, err error) {
+	// defer log.Trace(f, "location=%v", location)("offset=%v, err=%v", &offset, &err)
+	req, err := f.makeUploadHeadRequest(ctx, location)
+	if err != nil {
+		return 0, err
+	}
+	resp, err := f.oAuthClient.Do(req)
+	if err != nil {
+		return 0, err
+	}
+	err = checkStatusCode(resp, 200)
+	if err != nil {
+		return 0, err
+	}
+	return strconv.ParseInt(resp.Header.Get("upload-offset"), 10, 64)
+}
+
+func (f *Fs) transferChunk(ctx context.Context, location string, start int64, chunk io.ReadSeeker, chunkSize int64) (serverOffset, fileID int64, err error) {
+	// defer log.Trace(f, "location=%v, start=%v, chunkSize=%v", location, start, chunkSize)("fileID=%v, err=%v", &fileID, &err)
 	req, err := f.makeUploadPatchRequest(ctx, location, chunk, start, chunkSize)
 	if err != nil {
-		return 0, err
+		return
 	}
-	req = req.WithContext(ctx)
-	res, err := f.oAuthClient.Do(req)
+	resp, err := f.oAuthClient.Do(req)
 	if err != nil {
-		return 0, err
+		return
 	}
 	defer func() {
-		_ = res.Body.Close()
+		_ = resp.Body.Close()
 	}()
-	if res.StatusCode != 204 {
-		return 0, fmt.Errorf("unexpected status code while transferring chunk: %d", res.StatusCode)
+	err = checkStatusCode(resp, 204)
+	if err != nil {
+		return
 	}
-	sfid := res.Header.Get("putio-file-id")
+	serverOffset, err = strconv.ParseInt(resp.Header.Get("upload-offset"), 10, 64)
+	if err != nil {
+		return
+	}
+	sfid := resp.Header.Get("putio-file-id")
 	if sfid != "" {
 		fileID, err = strconv.ParseInt(sfid, 10, 64)
 		if err != nil {
-			return 0, err
+			return
 		}
 	}
-	return fileID, nil
+	return
+}
+
+func (f *Fs) makeUploadHeadRequest(ctx context.Context, location string) (*http.Request, error) {
+	req, err := http.NewRequest("HEAD", location, nil)
+	if err != nil {
+		return nil, err
+	}
+	req = req.WithContext(ctx) // go1.13 can use NewRequestWithContext
+	req.Header.Set("tus-resumable", "1.0.0")
+	return req, nil
 }
 
 func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.Reader, offset, length int64) (*http.Request, error) {
@@ -505,7 +555,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		params := url.Values{}
 		params.Set("file_id", strconv.FormatInt(srcObj.file.ID, 10))
 		params.Set("parent_id", directoryID)
-		params.Set("name", leaf)
+		params.Set("name", f.opt.Enc.FromStandardName(leaf))
 		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/copy", strings.NewReader(params.Encode()))
 		if err != nil {
 			return false, err
@@ -544,7 +594,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 		params := url.Values{}
 		params.Set("file_id", strconv.FormatInt(srcObj.file.ID, 10))
 		params.Set("parent_id", directoryID)
-		params.Set("name", leaf)
+		params.Set("name", f.opt.Enc.FromStandardName(leaf))
 		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
 		if err != nil {
 			return false, err
@@ -633,7 +683,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		params := url.Values{}
 		params.Set("file_id", srcID)
 		params.Set("parent_id", dstDirectoryID)
-		params.Set("name", leaf)
+		params.Set("name", f.opt.Enc.FromStandardName(leaf))
 		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
 		if err != nil {
 			return false, err

backend/putio/object.go

@@ -137,7 +137,7 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		// fs.Debugf(o, "requesting child. directoryID: %s, name: %s", directoryID, leaf)
-		req, err := o.fs.client.NewRequest(ctx, "GET", "/v2/files/"+directoryID+"/child?name="+url.PathEscape(leaf), nil)
+		req, err := o.fs.client.NewRequest(ctx, "GET", "/v2/files/"+directoryID+"/child?name="+url.QueryEscape(o.fs.opt.Enc.FromStandardName(leaf)), nil)
 		if err != nil {
 			return false, err
 		}
@@ -147,6 +147,12 @@ func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
 		}
 		return shouldRetry(err)
 	})
+	if err != nil {
+		return nil, err
+	}
+	if resp.File.IsDir() {
+		return nil, fs.ErrorNotAFile
+	}
 	return &resp.File, err
 }
 

backend/putio/putio.go

@@ -6,13 +6,27 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/lib/dircache"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"golang.org/x/oauth2"
 )
 
+/*
+// TestPutio
+stringNeedsEscaping = []rune{
+	'/', '\x00'
+}
+maxFileLength = 255
+canWriteUnnormalized = true
+canReadUnnormalized   = true
+canReadRenormalized   = true
+canStream = false
+*/
+
 // Constants
 const (
 	rcloneClientID             = "4131"
@@ -51,9 +65,25 @@ func init() {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
 		},
+		Options: []fs.Option{{
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Note that \ is renamed to -
+			//
+			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
+			Default: (encoder.Display |
+				encoder.EncodeBackSlash |
+				encoder.EncodeInvalidUtf8),
+		}},
 	})
 }
 
+// Options defines the configuration for this backend
+type Options struct {
+	Enc encoder.MultiEncoder `config:"encoding"`
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/qingstor/qingstor.go

@@ -18,12 +18,14 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	qsConfig "github.com/yunify/qingstor-sdk-go/v3/config"
 	qsErr "github.com/yunify/qingstor-sdk-go/v3/request/errors"
 	qs "github.com/yunify/qingstor-sdk-go/v3/service"
@@ -110,6 +112,13 @@ and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
 			Default:  1,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			Default: (encoder.EncodeInvalidUtf8 |
+				encoder.EncodeCtl |
+				encoder.EncodeSlash),
 		}},
 	})
 }
@@ -133,15 +142,16 @@ func timestampToTime(tp int64) time.Time {
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth           bool          `config:"env_auth"`
-	AccessKeyID       string        `config:"access_key_id"`
-	SecretAccessKey   string        `config:"secret_access_key"`
-	Endpoint          string        `config:"endpoint"`
-	Zone              string        `config:"zone"`
-	ConnectionRetries int           `config:"connection_retries"`
-	UploadCutoff      fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize         fs.SizeSuffix `config:"chunk_size"`
-	UploadConcurrency int           `config:"upload_concurrency"`
+	EnvAuth           bool                 `config:"env_auth"`
+	AccessKeyID       string               `config:"access_key_id"`
+	SecretAccessKey   string               `config:"secret_access_key"`
+	Endpoint          string               `config:"endpoint"`
+	Zone              string               `config:"zone"`
+	ConnectionRetries int                  `config:"connection_retries"`
+	UploadCutoff      fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize         fs.SizeSuffix        `config:"chunk_size"`
+	UploadConcurrency int                  `config:"upload_concurrency"`
+	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote qingstor server
@@ -184,7 +194,8 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
 // split returns bucket and bucketPath from the object
@@ -265,7 +276,7 @@ func qsServiceConnection(opt *Options) (*qs.Service, error) {
 	cf.Protocol = protocol
 	cf.Host = host
 	cf.Port = port
-	cf.ConnectionRetries = opt.ConnectionRetries
+	// unsupported in v3.1: cf.ConnectionRetries = opt.ConnectionRetries
 	cf.Connection = fshttp.NewClient(fs.Config)
 
 	return qs.Init(cf)
@@ -353,7 +364,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, err
 		}
-		_, err = bucketInit.HeadObject(f.rootDirectory, &qs.HeadObjectInput{})
+		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
+		_, err = bucketInit.HeadObject(encodedDirectory, &qs.HeadObjectInput{})
 		if err == nil {
 			newRoot := path.Dir(f.root)
 			if newRoot == "." {
@@ -380,7 +392,7 @@ func (f *Fs) Root() string {
 // String converts this Fs to a string
 func (f *Fs) String() string {
 	if f.rootBucket == "" {
-		return fmt.Sprintf("QingStor root")
+		return "QingStor root"
 	}
 	if f.rootDirectory == "" {
 		return fmt.Sprintf("QingStor bucket %s", f.rootBucket)
@@ -550,6 +562,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 					continue
 				}
 				remote := *commonPrefix
+				remote = f.opt.Enc.ToStandardPath(remote)
 				if !strings.HasPrefix(remote, prefix) {
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
@@ -569,12 +582,13 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		}
 
 		for _, object := range resp.Keys {
-			key := qs.StringValue(object.Key)
-			if !strings.HasPrefix(key, prefix) {
-				fs.Logf(f, "Odd name received %q", key)
+			remote := qs.StringValue(object.Key)
+			remote = f.opt.Enc.ToStandardPath(remote)
+			if !strings.HasPrefix(remote, prefix) {
+				fs.Logf(f, "Odd name received %q", remote)
 				continue
 			}
-			remote := key[len(prefix):]
+			remote = remote[len(prefix):]
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
@@ -646,7 +660,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 	}
 
 	for _, bucket := range resp.Buckets {
-		d := fs.NewDir(qs.StringValue(bucket.Name), qs.TimeValue(bucket.Created))
+		d := fs.NewDir(f.opt.Enc.ToStandardName(qs.StringValue(bucket.Name)), qs.TimeValue(bucket.Created))
 		entries = append(entries, d)
 	}
 	return entries, nil

backend/qingstor/upload.go

@@ -297,21 +297,6 @@ func (mu *multiUploader) send(c chunk) error {
 	return err
 }
 
-// list list the ObjectParts of an multipart upload
-func (mu *multiUploader) list() error {
-	bucketInit, _ := mu.bucketInit()
-
-	req := qs.ListMultipartInput{
-		UploadID: mu.uploadID,
-	}
-	fs.Debugf(mu, "Reading multi-part details")
-	rsp, err := bucketInit.ListMultipart(mu.cfg.key, &req)
-	if err == nil {
-		mu.objectParts = rsp.ObjectParts
-	}
-	return err
-}
-
 // complete complete an multipart upload
 func (mu *multiUploader) complete() error {
 	var err error

backend/s3/s3.go

@@ -14,15 +14,22 @@ What happens if you CTRL-C a multipart upload
 */
 
 import (
+	"bytes"
 	"context"
+	"crypto/md5"
 	"encoding/base64"
 	"encoding/hex"
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"regexp"
+	"sort"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -30,15 +37,16 @@ import (
 	"github.com/aws/aws-sdk-go/aws/corehandlers"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
+	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/ncw/swift"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -46,8 +54,12 @@ import (
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
 )
 
 // Register with Fs
@@ -83,6 +95,9 @@ func init() {
 			}, {
 				Value: "Netease",
 				Help:  "Netease Object Storage (NOS)",
+			}, {
+				Value: "StackPath",
+				Help:  "StackPath Object Storage",
 			}, {
 				Value: "Wasabi",
 				Help:  "Wasabi Object Storage",
@@ -153,6 +168,9 @@ func init() {
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)\nNeeds location constraint ap-south-1.",
+			}, {
+				Value: "ap-east-1",
+				Help:  "Asia Patific (Hong Kong) Region\nNeeds location constraint ap-east-1.",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
@@ -342,10 +360,24 @@ func init() {
 				Value: "oss-me-east-1.aliyuncs.com",
 				Help:  "Middle East 1 (Dubai)",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for StackPath Object Storage.",
+			Provider: "StackPath",
+			Examples: []fs.OptionExample{{
+				Value: "s3.us-east-2.stackpathstorage.com",
+				Help:  "US East Endpoint",
+			}, {
+				Value: "s3.us-west-1.stackpathstorage.com",
+				Help:  "US West Endpoint",
+			}, {
+				Value: "s3.eu-central-1.stackpathstorage.com",
+				Help:  "EU Endpoint",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,Alibaba",
+			Provider: "!AWS,IBMCOS,Alibaba,StackPath",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -421,6 +453,9 @@ func init() {
 			}, {
 				Value: "ap-south-1",
 				Help:  "Asia Pacific (Mumbai)",
+			}, {
+				Value: "ap-east-1",
+				Help:  "Asia Pacific (Hong Kong)",
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region.",
@@ -529,7 +564,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba",
+			Provider: "!AWS,IBMCOS,Alibaba,StackPath",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -687,16 +722,37 @@ The minimum is 0 and the maximum is 5GB.`,
 			Name: "chunk_size",
 			Help: `Chunk size to use for uploading.
 
-When uploading files larger than upload_cutoff they will be uploaded
-as multipart uploads using this chunk size.
+When uploading files larger than upload_cutoff or files with unknown
+size (eg from "rclone rcat" or uploaded with "rclone mount" or google
+photos or google docs) they will be uploaded as multipart uploads
+using this chunk size.
 
 Note that "--s3-upload-concurrency" chunks of this size are buffered
 in memory per transfer.
 
 If you are transferring large files over high speed links and you have
-enough memory, then increasing this will speed up the transfers.`,
+enough memory, then increasing this will speed up the transfers.
+
+Rclone will automatically increase the chunk size when uploading a
+large file of known size to stay below the 10,000 chunks limit.
+
+Files of unknown size are uploaded with the configured
+chunk_size. Since the default chunk size is 5MB and there can be at
+most 10,000 chunks, this means that by default the maximum size of
+file you can stream upload is 48GB.  If you wish to stream upload
+larger files then you will need to increase chunk_size.`,
 			Default:  minChunkSize,
 			Advanced: true,
+		}, {
+			Name: "copy_cutoff",
+			Help: `Cutoff for switching to multipart copy
+
+Any files larger than this that need to be server side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 5GB.`,
+			Default:  fs.SizeSuffix(maxSizeForCopy),
+			Advanced: true,
 		}, {
 			Name:     "disable_checksum",
 			Help:     "Don't store MD5 checksum with object metadata",
@@ -727,7 +783,9 @@ if false then rclone will use virtual path style. See [the AWS S3
 docs](https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html#access-bucket-intro)
 for more info.
 
-Some providers (eg Aliyun OSS or Netease COS) require this set to false.`,
+Some providers (eg AWS, Aliyun OSS or Netease COS) require this set to
+false - rclone will do this automatically based on the provider
+setting.`,
 			Default:  true,
 			Advanced: true,
 		}, {
@@ -748,61 +806,121 @@ Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
 See: [AWS S3 Transfer acceleration](https://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration-examples.html)`,
 			Default:  false,
 			Advanced: true,
-		}},
-	})
+		}, {
+			Name:     "leave_parts_on_error",
+			Provider: "AWS",
+			Help: `If true avoid calling abort upload on a failure, leaving all successfully uploaded parts on S3 for manual recovery.
+
+It should be set to true for resuming uploads across different sessions.
+
+WARNING: Storing parts of an incomplete multipart upload counts towards space usage on S3 and will add additional costs if not cleaned up.
+`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name: "list_chunk",
+			Help: `Size of listing chunk (response list for each ListObject S3 request).
+
+This option is also known as "MaxKeys", "max-items", or "page-size" from the AWS S3 specification.
+Most services truncate the response list to 1000 objects even if requested more than that.
+In AWS S3 this is a global maximum and cannot be changed, see [AWS S3](https://docs.aws.amazon.com/cli/latest/reference/s3/ls.html).
+In Ceph, this can be increased with the "rgw list buckets max chunk" option.
+`,
+			Default:  1000,
+			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Any UTF-8 character is valid in a key, however it can't handle
+			// invalid UTF-8 and / have a special meaning.
+			//
+			// The SDK can't seem to handle uploading files called '.'
+			//
+			// FIXME would be nice to add
+			// - initial / encoding
+			// - doubled / encoding
+			// - trailing / encoding
+			// so that AWS keys are always valid file names
+			Default: encoder.EncodeInvalidUtf8 |
+				encoder.EncodeSlash |
+				encoder.EncodeDot,
+		}, {
+			Name:     "memory_pool_flush_time",
+			Default:  memoryPoolFlushTime,
+			Advanced: true,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
+		}, {
+			Name:     "memory_pool_use_mmap",
+			Default:  memoryPoolUseMmap,
+			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
+		},
+		}})
 }
 
 // Constants
 const (
-	metaMtime           = "Mtime"                       // the meta key to store mtime in - eg X-Amz-Meta-Mtime
-	metaMD5Hash         = "Md5chksum"                   // the meta key to store md5hash in
-	listChunkSize       = 1000                          // number of items to read at once
-	maxRetries          = 10                            // number of retries to make of operations
-	maxSizeForCopy      = 5 * 1024 * 1024 * 1024        // The maximum size of object we can COPY
-	maxFileSize         = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
-	minChunkSize        = fs.SizeSuffix(s3manager.MinUploadPartSize)
+	metaMtime           = "Mtime"                // the meta key to store mtime in - eg X-Amz-Meta-Mtime
+	metaMD5Hash         = "Md5chksum"            // the meta key to store md5hash in
+	maxSizeForCopy      = 5 * 1024 * 1024 * 1024 // The maximum size of object we can COPY
+	maxUploadParts      = 10000                  // maximum allowed number of parts in a multi-part upload
+	minChunkSize        = fs.SizeSuffix(1024 * 1024 * 5)
 	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
 	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
 	minSleep            = 10 * time.Millisecond // In case of error, start at 10ms sleep.
+
+	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap   = false
 )
 
 // Options defines the configuration for this backend
 type Options struct {
-	Provider              string        `config:"provider"`
-	EnvAuth               bool          `config:"env_auth"`
-	AccessKeyID           string        `config:"access_key_id"`
-	SecretAccessKey       string        `config:"secret_access_key"`
-	Region                string        `config:"region"`
-	Endpoint              string        `config:"endpoint"`
-	LocationConstraint    string        `config:"location_constraint"`
-	ACL                   string        `config:"acl"`
-	BucketACL             string        `config:"bucket_acl"`
-	ServerSideEncryption  string        `config:"server_side_encryption"`
-	SSEKMSKeyID           string        `config:"sse_kms_key_id"`
-	StorageClass          string        `config:"storage_class"`
-	UploadCutoff          fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize             fs.SizeSuffix `config:"chunk_size"`
-	DisableChecksum       bool          `config:"disable_checksum"`
-	SessionToken          string        `config:"session_token"`
-	UploadConcurrency     int           `config:"upload_concurrency"`
-	ForcePathStyle        bool          `config:"force_path_style"`
-	V2Auth                bool          `config:"v2_auth"`
-	UseAccelerateEndpoint bool          `config:"use_accelerate_endpoint"`
+	Provider              string               `config:"provider"`
+	EnvAuth               bool                 `config:"env_auth"`
+	AccessKeyID           string               `config:"access_key_id"`
+	SecretAccessKey       string               `config:"secret_access_key"`
+	Region                string               `config:"region"`
+	Endpoint              string               `config:"endpoint"`
+	LocationConstraint    string               `config:"location_constraint"`
+	ACL                   string               `config:"acl"`
+	BucketACL             string               `config:"bucket_acl"`
+	ServerSideEncryption  string               `config:"server_side_encryption"`
+	SSEKMSKeyID           string               `config:"sse_kms_key_id"`
+	StorageClass          string               `config:"storage_class"`
+	UploadCutoff          fs.SizeSuffix        `config:"upload_cutoff"`
+	CopyCutoff            fs.SizeSuffix        `config:"copy_cutoff"`
+	ChunkSize             fs.SizeSuffix        `config:"chunk_size"`
+	DisableChecksum       bool                 `config:"disable_checksum"`
+	SessionToken          string               `config:"session_token"`
+	UploadConcurrency     int                  `config:"upload_concurrency"`
+	ForcePathStyle        bool                 `config:"force_path_style"`
+	V2Auth                bool                 `config:"v2_auth"`
+	UseAccelerateEndpoint bool                 `config:"use_accelerate_endpoint"`
+	LeavePartsOnError     bool                 `config:"leave_parts_on_error"`
+	ListChunk             int64                `config:"list_chunk"`
+	Enc                   encoder.MultiEncoder `config:"encoding"`
+	MemoryPoolFlushTime   fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap     bool                 `config:"memory_pool_use_mmap"`
 }
 
 // Fs represents a remote s3 server
 type Fs struct {
-	name          string           // the name of the remote
-	root          string           // root of the bucket - ignore all objects above this
-	opt           Options          // parsed options
-	features      *fs.Features     // optional features
-	c             *s3.S3           // the connection to the s3 server
-	ses           *session.Session // the s3 session
-	rootBucket    string           // bucket part of root (if any)
-	rootDirectory string           // directory part of root (if any)
-	cache         *bucket.Cache    // cache for bucket creation status
-	pacer         *fs.Pacer        // To pace the API calls
-	srv           *http.Client     // a plain http client
+	name          string               // the name of the remote
+	root          string               // root of the bucket - ignore all objects above this
+	opt           Options              // parsed options
+	features      *fs.Features         // optional features
+	c             *s3.S3               // the connection to the s3 server
+	ses           *session.Session     // the s3 session
+	rootBucket    string               // bucket part of root (if any)
+	rootDirectory string               // directory part of root (if any)
+	cache         *bucket.Cache        // cache for bucket creation status
+	pacer         *fs.Pacer            // To pace the API calls
+	srv           *http.Client         // a plain http client
+	poolMu        sync.Mutex           // mutex protecting memory pools map
+	pools         map[int64]*pool.Pool // memory pools
 }
 
 // Object describes a s3 object
@@ -852,7 +970,7 @@ func (f *Fs) Features() *fs.Features {
 // retryErrorCodes is a slice of error codes that we will retry
 // See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 var retryErrorCodes = []int{
-	// 409, // Conflict - various states that could be resolved on a retry
+	500, // Internal Server Error - "We encountered an internal error. Please try again."
 	503, // Service Unavailable/Slow Down - "Reduce your request rate"
 }
 
@@ -899,7 +1017,8 @@ func parsePath(path string) (root string) {
 // split returns bucket and bucketPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (bucketName, bucketPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	bucketName, bucketPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(bucketName), f.opt.Enc.FromStandardPath(bucketPath)
 }
 
 // split returns bucket and bucketPath from the object
@@ -942,7 +1061,12 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 			Client: ec2metadata.New(session.New(), &aws.Config{
 				HTTPClient: lowTimeoutClient,
 			}),
-			ExpiryWindow: 3,
+			ExpiryWindow: 3 * time.Minute,
+		},
+
+		// Pick up IAM role if we are in EKS
+		&stscreds.WebIdentityRoleProvider{
+			ExpiryWindow: 3 * time.Minute,
 		},
 	}
 	cred := credentials.NewChainCredentials(providers)
@@ -965,11 +1089,11 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
+	if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
 	awsConfig := aws.NewConfig().
-		WithMaxRetries(maxRetries).
+		WithMaxRetries(fs.Config.LowLevelRetries).
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle).
@@ -1075,15 +1199,23 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	pc := fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep)))
+	// Set pacer retries to 0 because we are relying on SDK retry mechanism.
+	// Setting it to 1 because in context of pacer it means 1 attempt.
+	pc.SetRetries(1)
+
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
 		c:     c,
 		ses:   ses,
-		pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
+		pacer: pc,
 		cache: bucket.NewCache(),
 		srv:   fshttp.NewClient(fs.Config),
+		pools: make(map[int64]*pool.Pool),
 	}
+
 	f.setRoot(root)
 	f.features = (&fs.Features{
 		ReadMimeType:      true,
@@ -1095,9 +1227,10 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	if f.rootBucket != "" && f.rootDirectory != "" {
 		// Check to see if the object exists
+		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		req := s3.HeadObjectInput{
 			Bucket: &f.rootBucket,
-			Key:    &f.rootDirectory,
+			Key:    &encodedDirectory,
 		}
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.c.HeadObject(&req)
@@ -1212,25 +1345,56 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 	if directory != "" {
 		directory += "/"
 	}
-	maxKeys := int64(listChunkSize)
 	delimiter := ""
 	if !recurse {
 		delimiter = "/"
 	}
 	var marker *string
+	// URL encode the listings so we can use control characters in object names
+	// See: https://github.com/aws/aws-sdk-go/issues/1914
+	//
+	// However this doesn't work perfectly under Ceph (and hence DigitalOcean/Dreamhost) because
+	// it doesn't encode CommonPrefixes.
+	// See: https://tracker.ceph.com/issues/41870
+	//
+	// This does not work under IBM COS also: See https://github.com/rclone/rclone/issues/3345
+	// though maybe it does on some versions.
+	//
+	// This does work with minio but was only added relatively recently
+	// https://github.com/minio/minio/pull/7265
+	//
+	// So we enable only on providers we know supports it properly, all others can retry when a
+	// XML Syntax error is detected.
+	var urlEncodeListings = (f.opt.Provider == "AWS" || f.opt.Provider == "Wasabi" || f.opt.Provider == "Alibaba" || f.opt.Provider == "Minio")
 	for {
 		// FIXME need to implement ALL loop
 		req := s3.ListObjectsInput{
 			Bucket:    &bucket,
 			Delimiter: &delimiter,
 			Prefix:    &directory,
-			MaxKeys:   &maxKeys,
+			MaxKeys:   &f.opt.ListChunk,
 			Marker:    marker,
 		}
+		if urlEncodeListings {
+			req.EncodingType = aws.String(s3.EncodingTypeUrl)
+		}
 		var resp *s3.ListObjectsOutput
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.c.ListObjectsWithContext(ctx, &req)
+			if err != nil && !urlEncodeListings {
+				if awsErr, ok := err.(awserr.RequestFailure); ok {
+					if origErr := awsErr.OrigErr(); origErr != nil {
+						if _, ok := origErr.(*xml.SyntaxError); ok {
+							// Retry the listing with URL encoding as there were characters that XML can't encode
+							urlEncodeListings = true
+							req.EncodingType = aws.String(s3.EncodingTypeUrl)
+							fs.Debugf(f, "Retrying listing because of characters which can't be XML encoded")
+							return true, err
+						}
+					}
+				}
+			}
 			return f.shouldRetry(err)
 		})
 		if err != nil {
@@ -1259,6 +1423,14 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 					continue
 				}
 				remote := *commonPrefix.Prefix
+				if urlEncodeListings {
+					remote, err = url.QueryUnescape(remote)
+					if err != nil {
+						fs.Logf(f, "failed to URL decode %q in listing common prefix: %v", *commonPrefix.Prefix, err)
+						continue
+					}
+				}
+				remote = f.opt.Enc.ToStandardPath(remote)
 				if !strings.HasPrefix(remote, prefix) {
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
@@ -1278,6 +1450,14 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		}
 		for _, object := range resp.Contents {
 			remote := aws.StringValue(object.Key)
+			if urlEncodeListings {
+				remote, err = url.QueryUnescape(remote)
+				if err != nil {
+					fs.Logf(f, "failed to URL decode %q in listing: %v", aws.StringValue(object.Key), err)
+					continue
+				}
+			}
+			remote = f.opt.Enc.ToStandardPath(remote)
 			if !strings.HasPrefix(remote, prefix) {
 				fs.Logf(f, "Odd name received %q", remote)
 				continue
@@ -1308,6 +1488,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 		} else {
 			marker = resp.NextMarker
 		}
+		if urlEncodeListings {
+			*marker, err = url.QueryUnescape(*marker)
+			if err != nil {
+				return errors.Wrapf(err, "failed to URL decode NextMarker %q", *marker)
+			}
+		}
 	}
 	return nil
 }
@@ -1362,7 +1548,7 @@ func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error)
 		return nil, err
 	}
 	for _, bucket := range resp.Buckets {
-		bucketName := aws.StringValue(bucket.Name)
+		bucketName := f.opt.Enc.ToStandardName(aws.StringValue(bucket.Name))
 		f.cache.MarkOK(bucketName)
 		d := fs.NewDir(bucketName, aws.TimeValue(bucket.CreationDate))
 		entries = append(entries, d)
@@ -1558,7 +1744,7 @@ func pathEscape(s string) string {
 //
 // It adds the boiler plate to the req passed in and calls the s3
 // method
-func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string) error {
+func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, srcSize int64) error {
 	req.Bucket = &dstBucket
 	req.ACL = &f.opt.ACL
 	req.Key = &dstPath
@@ -1573,12 +1759,114 @@ func (f *Fs) copy(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPa
 	if req.StorageClass == nil && f.opt.StorageClass != "" {
 		req.StorageClass = &f.opt.StorageClass
 	}
+
+	if srcSize >= int64(f.opt.CopyCutoff) {
+		return f.copyMultipart(ctx, req, dstBucket, dstPath, srcBucket, srcPath, srcSize)
+	}
 	return f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CopyObjectWithContext(ctx, req)
 		return f.shouldRetry(err)
 	})
 }
 
+func calculateRange(partSize, partIndex, numParts, totalSize int64) string {
+	start := partIndex * partSize
+	var ends string
+	if partIndex == numParts-1 {
+		if totalSize >= 1 {
+			ends = strconv.FormatInt(totalSize-1, 10)
+		}
+	} else {
+		ends = strconv.FormatInt(start+partSize-1, 10)
+	}
+	return fmt.Sprintf("bytes=%v-%v", start, ends)
+}
+
+func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBucket, dstPath, srcBucket, srcPath string, srcSize int64) (err error) {
+	var cout *s3.CreateMultipartUploadOutput
+	if err := f.pacer.Call(func() (bool, error) {
+		var err error
+		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
+			Bucket: &dstBucket,
+			Key:    &dstPath,
+		})
+		return f.shouldRetry(err)
+	}); err != nil {
+		return err
+	}
+	uid := cout.UploadId
+
+	defer func() {
+		if err != nil {
+			// We can try to abort the upload, but ignore the error.
+			fs.Debugf(nil, "Cancelling multipart copy")
+			_ = f.pacer.Call(func() (bool, error) {
+				_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
+					Bucket:       &dstBucket,
+					Key:          &dstPath,
+					UploadId:     uid,
+					RequestPayer: req.RequestPayer,
+				})
+				return f.shouldRetry(err)
+			})
+		}
+	}()
+
+	partSize := int64(f.opt.CopyCutoff)
+	numParts := (srcSize-1)/partSize + 1
+
+	var parts []*s3.CompletedPart
+	for partNum := int64(1); partNum <= numParts; partNum++ {
+		if err := f.pacer.Call(func() (bool, error) {
+			partNum := partNum
+			uploadPartReq := &s3.UploadPartCopyInput{
+				Bucket:          &dstBucket,
+				Key:             &dstPath,
+				PartNumber:      &partNum,
+				UploadId:        uid,
+				CopySourceRange: aws.String(calculateRange(partSize, partNum-1, numParts, srcSize)),
+				// Args copy from req
+				CopySource:                     req.CopySource,
+				CopySourceIfMatch:              req.CopySourceIfMatch,
+				CopySourceIfModifiedSince:      req.CopySourceIfModifiedSince,
+				CopySourceIfNoneMatch:          req.CopySourceIfNoneMatch,
+				CopySourceIfUnmodifiedSince:    req.CopySourceIfUnmodifiedSince,
+				CopySourceSSECustomerAlgorithm: req.CopySourceSSECustomerAlgorithm,
+				CopySourceSSECustomerKey:       req.CopySourceSSECustomerKey,
+				CopySourceSSECustomerKeyMD5:    req.CopySourceSSECustomerKeyMD5,
+				RequestPayer:                   req.RequestPayer,
+				SSECustomerAlgorithm:           req.SSECustomerAlgorithm,
+				SSECustomerKey:                 req.SSECustomerKey,
+				SSECustomerKeyMD5:              req.SSECustomerKeyMD5,
+			}
+			uout, err := f.c.UploadPartCopyWithContext(ctx, uploadPartReq)
+			if err != nil {
+				return f.shouldRetry(err)
+			}
+			parts = append(parts, &s3.CompletedPart{
+				PartNumber: &partNum,
+				ETag:       uout.CopyPartResult.ETag,
+			})
+			return false, nil
+		}); err != nil {
+			return err
+		}
+	}
+
+	return f.pacer.Call(func() (bool, error) {
+		_, err := f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{
+			Bucket: &dstBucket,
+			Key:    &dstPath,
+			MultipartUpload: &s3.CompletedMultipartUpload{
+				Parts: parts,
+			},
+			RequestPayer: req.RequestPayer,
+			UploadId:     uid,
+		})
+		return f.shouldRetry(err)
+	})
+}
+
 // Copy src to this remote using server side copy operations.
 //
 // This is stored with the remote path given
@@ -1603,7 +1891,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	req := s3.CopyObjectInput{
 		MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
 	}
-	err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath)
+	err = f.copy(ctx, &req, dstBucket, dstPath, srcBucket, srcPath, srcObj.Size())
 	if err != nil {
 		return nil, err
 	}
@@ -1615,6 +1903,22 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
 }
 
+func (f *Fs) getMemoryPool(size int64) *pool.Pool {
+	f.poolMu.Lock()
+	defer f.poolMu.Unlock()
+
+	_, ok := f.pools[size]
+	if !ok {
+		f.pools[size] = pool.New(
+			time.Duration(f.opt.MemoryPoolFlushTime),
+			int(f.opt.ChunkSize),
+			f.opt.UploadConcurrency*fs.Config.Transfers,
+			f.opt.MemoryPoolUseMmap,
+		)
+	}
+	return f.pools[size]
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1703,6 +2007,9 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	o.etag = aws.StringValue(resp.ETag)
 	o.bytes = size
 	o.meta = resp.Metadata
+	if o.meta == nil {
+		o.meta = map[string]*string{}
+	}
 	o.storageClass = aws.StringValue(resp.StorageClass)
 	if resp.LastModified == nil {
 		fs.Logf(o, "Failed to read last modified from HEAD: %v", err)
@@ -1749,11 +2056,6 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	}
 	o.meta[metaMtime] = aws.String(swift.TimeToFloatString(modTime))
 
-	if o.bytes >= maxSizeForCopy {
-		fs.Debugf(o, "SetModTime is unsupported for objects bigger than %v bytes", fs.SizeSuffix(maxSizeForCopy))
-		return nil
-	}
-
 	// Can't update metadata here, so return this error to force a recopy
 	if o.storageClass == "GLACIER" || o.storageClass == "DEEP_ARCHIVE" {
 		return fs.ErrorCantSetModTime
@@ -1766,7 +2068,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 		Metadata:          o.meta,
 		MetadataDirective: aws.String(s3.MetadataDirectiveReplace), // replace metadata with that passed in
 	}
-	return o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath)
+	return o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath, o.bytes)
 }
 
 // Storable raturns a boolean indicating if this object is storable
@@ -1810,6 +2112,193 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	return resp.Body, nil
 }
 
+var warnStreamUpload sync.Once
+
+func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (err error) {
+	f := o.fs
+
+	// make concurrency machinery
+	concurrency := f.opt.UploadConcurrency
+	if concurrency < 1 {
+		concurrency = 1
+	}
+	tokens := pacer.NewTokenDispenser(concurrency)
+
+	// calculate size of parts
+	partSize := int(f.opt.ChunkSize)
+
+	// size can be -1 here meaning we don't know the size of the incoming file.  We use ChunkSize
+	// buffers here (default 5MB). With a maximum number of parts (10,000) this will be a file of
+	// 48GB which seems like a not too unreasonable limit.
+	if size == -1 {
+		warnStreamUpload.Do(func() {
+			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				f.opt.ChunkSize, fs.SizeSuffix(partSize*maxUploadParts))
+		})
+	} else {
+		// Adjust partSize until the number of parts is small enough.
+		if size/int64(partSize) >= maxUploadParts {
+			// Calculate partition size rounded up to the nearest MB
+			partSize = int((((size / maxUploadParts) >> 20) + 1) << 20)
+		}
+	}
+
+	memPool := f.getMemoryPool(int64(partSize))
+
+	var cout *s3.CreateMultipartUploadOutput
+	err = f.pacer.Call(func() (bool, error) {
+		var err error
+		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
+			Bucket:               req.Bucket,
+			ACL:                  req.ACL,
+			Key:                  req.Key,
+			ContentType:          req.ContentType,
+			Metadata:             req.Metadata,
+			ServerSideEncryption: req.ServerSideEncryption,
+			SSEKMSKeyId:          req.SSEKMSKeyId,
+			StorageClass:         req.StorageClass,
+		})
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "multipart upload failed to initialise")
+	}
+	uid := cout.UploadId
+
+	defer func() {
+		if o.fs.opt.LeavePartsOnError {
+			return
+		}
+		if err != nil {
+			// We can try to abort the upload, but ignore the error.
+			fs.Debugf(o, "Cancelling multipart upload")
+			errCancel := f.pacer.Call(func() (bool, error) {
+				_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
+					Bucket:       req.Bucket,
+					Key:          req.Key,
+					UploadId:     uid,
+					RequestPayer: req.RequestPayer,
+				})
+				return f.shouldRetry(err)
+			})
+			if errCancel != nil {
+				fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
+			}
+		}
+	}()
+
+	var (
+		g, gCtx  = errgroup.WithContext(ctx)
+		finished = false
+		partsMu  sync.Mutex // to protect parts
+		parts    []*s3.CompletedPart
+		off      int64
+	)
+
+	for partNum := int64(1); !finished; partNum++ {
+		// Get a block of memory from the pool and token which limits concurrency.
+		tokens.Get()
+		buf := memPool.Get()
+
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in uploading all the other parts.
+		if gCtx.Err() != nil {
+			break
+		}
+
+		// Read the chunk
+		var n int
+		n, err = readers.ReadFill(in, buf) // this can never return 0, nil
+		if err == io.EOF {
+			if n == 0 && partNum != 1 { // end if no data and if not first chunk
+				break
+			}
+			finished = true
+		} else if err != nil {
+			return errors.Wrap(err, "multipart upload failed to read source")
+		}
+		buf = buf[:n]
+
+		partNum := partNum
+		fs.Debugf(o, "multipart upload starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(off), fs.SizeSuffix(size))
+		off += int64(n)
+		g.Go(func() (err error) {
+			partLength := int64(len(buf))
+
+			// create checksum of buffer for integrity checking
+			md5sumBinary := md5.Sum(buf)
+			md5sum := base64.StdEncoding.EncodeToString(md5sumBinary[:])
+
+			err = f.pacer.Call(func() (bool, error) {
+				uploadPartReq := &s3.UploadPartInput{
+					Body:                 bytes.NewReader(buf),
+					Bucket:               req.Bucket,
+					Key:                  req.Key,
+					PartNumber:           &partNum,
+					UploadId:             uid,
+					ContentMD5:           &md5sum,
+					ContentLength:        &partLength,
+					RequestPayer:         req.RequestPayer,
+					SSECustomerAlgorithm: req.SSECustomerAlgorithm,
+					SSECustomerKey:       req.SSECustomerKey,
+					SSECustomerKeyMD5:    req.SSECustomerKeyMD5,
+				}
+				uout, err := f.c.UploadPartWithContext(gCtx, uploadPartReq)
+				if err != nil {
+					if partNum <= int64(concurrency) {
+						return f.shouldRetry(err)
+					}
+					// retry all chunks once have done the first batch
+					return true, err
+				}
+				partsMu.Lock()
+				parts = append(parts, &s3.CompletedPart{
+					PartNumber: &partNum,
+					ETag:       uout.ETag,
+				})
+				partsMu.Unlock()
+
+				return false, nil
+			})
+
+			// return the memory and token
+			memPool.Put(buf[:partSize])
+			tokens.Put()
+
+			if err != nil {
+				return errors.Wrap(err, "multipart upload failed to upload part")
+			}
+			return nil
+		})
+	}
+	err = g.Wait()
+	if err != nil {
+		return err
+	}
+
+	// sort the completed parts by part number
+	sort.Slice(parts, func(i, j int) bool {
+		return *parts[i].PartNumber < *parts[j].PartNumber
+	})
+
+	err = f.pacer.Call(func() (bool, error) {
+		_, err := f.c.CompleteMultipartUploadWithContext(ctx, &s3.CompleteMultipartUploadInput{
+			Bucket: req.Bucket,
+			Key:    req.Key,
+			MultipartUpload: &s3.CompletedMultipartUpload{
+				Parts: parts,
+			},
+			RequestPayer: req.RequestPayer,
+			UploadId:     uid,
+		})
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "multipart upload failed to finalise")
+	}
+	return nil
+}
+
 // Update the Object from in with modTime and size
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
 	bucket, bucketPath := o.split()
@@ -1821,35 +2310,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	size := src.Size()
 
 	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
-	var uploader *s3manager.Uploader
-	if multipart {
-		uploader = s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
-			u.Concurrency = o.fs.opt.UploadConcurrency
-			u.LeavePartsOnError = false
-			u.S3 = o.fs.c
-			u.PartSize = int64(o.fs.opt.ChunkSize)
-
-			if size == -1 {
-				// Make parts as small as possible while still being able to upload to the
-				// S3 file size limit. Rounded up to nearest MB.
-				u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
-				return
-			}
-			// Adjust PartSize until the number of parts is small enough.
-			if size/u.PartSize >= s3manager.MaxUploadParts {
-				// Calculate partition size rounded up to the nearest MB
-				u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20
-			}
-		})
-	}
 
 	// Set the mtime in the meta data
 	metadata := map[string]*string{
 		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
 	}
 
-	// read the md5sum if available for non multpart and if
-	// disable checksum isn't present.
+	// read the md5sum if available
+	// - for non multpart
+	//    - so we can add a ContentMD5
+	// - for multipart provided checksums aren't disabled
+	//    - so we can add the md5sum in the metadata as metaMD5Hash
 	var md5sum string
 	if !multipart || !o.fs.opt.DisableChecksum {
 		hash, err := src.Hash(ctx, hash.MD5)
@@ -1866,52 +2337,32 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Guess the content type
 	mimeType := fs.MimeType(ctx, src)
+	req := s3.PutObjectInput{
+		Bucket:      &bucket,
+		ACL:         &o.fs.opt.ACL,
+		Key:         &bucketPath,
+		ContentType: &mimeType,
+		Metadata:    metadata,
+	}
+	if md5sum != "" {
+		req.ContentMD5 = &md5sum
+	}
+	if o.fs.opt.ServerSideEncryption != "" {
+		req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
+	}
+	if o.fs.opt.SSEKMSKeyID != "" {
+		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
+	}
+	if o.fs.opt.StorageClass != "" {
+		req.StorageClass = &o.fs.opt.StorageClass
+	}
+
 	if multipart {
-		req := s3manager.UploadInput{
-			Bucket:      &bucket,
-			ACL:         &o.fs.opt.ACL,
-			Key:         &bucketPath,
-			Body:        in,
-			ContentType: &mimeType,
-			Metadata:    metadata,
-			//ContentLength: &size,
-		}
-		if o.fs.opt.ServerSideEncryption != "" {
-			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-		}
-		if o.fs.opt.SSEKMSKeyID != "" {
-			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-		}
-		if o.fs.opt.StorageClass != "" {
-			req.StorageClass = &o.fs.opt.StorageClass
-		}
-		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			_, err = uploader.UploadWithContext(ctx, &req)
-			return o.fs.shouldRetry(err)
-		})
+		err = o.uploadMultipart(ctx, &req, size, in)
 		if err != nil {
 			return err
 		}
 	} else {
-		req := s3.PutObjectInput{
-			Bucket:      &bucket,
-			ACL:         &o.fs.opt.ACL,
-			Key:         &bucketPath,
-			ContentType: &mimeType,
-			Metadata:    metadata,
-		}
-		if md5sum != "" {
-			req.ContentMD5 = &md5sum
-		}
-		if o.fs.opt.ServerSideEncryption != "" {
-			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-		}
-		if o.fs.opt.SSEKMSKeyID != "" {
-			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-		}
-		if o.fs.opt.StorageClass != "" {
-			req.StorageClass = &o.fs.opt.StorageClass
-		}
 
 		// Create the request
 		putObj, _ := o.fs.c.PutObjectRequest(&req)
@@ -2004,7 +2455,7 @@ func (o *Object) SetTier(tier string) (err error) {
 		MetadataDirective: aws.String(s3.MetadataDirectiveCopy),
 		StorageClass:      aws.String(tier),
 	}
-	err = o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath)
+	err = o.fs.copy(ctx, &req, bucket, bucketPath, bucket, bucketPath, o.bytes)
 	if err != nil {
 		return err
 	}

backend/sftp/sftp.go

@@ -29,15 +29,17 @@ import (
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/env"
+	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 	sshagent "github.com/xanzy/ssh-agent"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/time/rate"
 )
 
 const (
-	connectionsPerSecond    = 10 // don't make more than this many ssh connections/s
 	hashCommandNotSupported = "none"
+	minSleep                = 100 * time.Millisecond
+	maxSleep                = 2 * time.Second
+	decayConstant           = 2 // bigger for slower decay, exponential
 )
 
 var (
@@ -86,8 +88,19 @@ requested from the ssh-agent. This allows to avoid ` + "`Too many authentication
 when the ssh-agent contains many keys.`,
 			Default: false,
 		}, {
-			Name:    "use_insecure_cipher",
-			Help:    "Enable the use of the aes128-cbc cipher and diffie-hellman-group-exchange-sha256, diffie-hellman-group-exchange-sha1 key exchange. Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.",
+			Name: "use_insecure_cipher",
+			Help: `Enable the use of insecure ciphers and key exchange methods. 
+
+This enables the use of the the following insecure ciphers and key exchange methods:
+
+- aes128-cbc
+- aes192-cbc
+- aes256-cbc
+- 3des-cbc
+- diffie-hellman-group-exchange-sha256
+- diffie-hellman-group-exchange-sha1
+
+Those algorithms are insecure and may allow plaintext data to be recovered by an attacker.`,
 			Default: false,
 			Examples: []fs.OptionExample{
 				{
@@ -143,6 +156,11 @@ Home directory can be found in a shared folder called "home"
 			Default:  "",
 			Help:     "The command used to read sha1 hashes. Leave blank for autodetect.",
 			Advanced: true,
+		}, {
+			Name:     "skip_links",
+			Default:  false,
+			Help:     "Set to skip any symlinks and any other non regular files.",
+			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -164,6 +182,7 @@ type Options struct {
 	SetModTime        bool   `config:"set_modtime"`
 	Md5sumCommand     string `config:"md5sum_command"`
 	Sha1sumCommand    string `config:"sha1sum_command"`
+	SkipLinks         bool   `config:"skip_links"`
 }
 
 // Fs stores the interface to the remote SFTP files
@@ -179,7 +198,7 @@ type Fs struct {
 	cachedHashes *hash.Set
 	poolMu       sync.Mutex
 	pool         []*conn
-	connLimit    *rate.Limiter // for limiting number of connections per second
+	pacer        *fs.Pacer // pacer for operations
 }
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
@@ -259,10 +278,6 @@ func (c *conn) closed() error {
 // Open a new connection to the SFTP server.
 func (f *Fs) sftpConnection() (c *conn, err error) {
 	// Rate limit rate of new connections
-	err = f.connLimit.Wait(context.Background())
-	if err != nil {
-		return nil, errors.Wrap(err, "limiter failed in connect")
-	}
 	c = &conn{
 		err: make(chan error, 1),
 	}
@@ -296,7 +311,14 @@ func (f *Fs) getSftpConnection() (c *conn, err error) {
 	if c != nil {
 		return c, nil
 	}
-	return f.sftpConnection()
+	err = f.pacer.Call(func() (bool, error) {
+		c, err = f.sftpConnection()
+		if err != nil {
+			return true, err
+		}
+		return false, nil
+	})
+	return c, err
 }
 
 // Return an SFTP connection to the pool
@@ -363,7 +385,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	if opt.UseInsecureCipher {
 		sshConfig.Config.SetDefaults()
-		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
+		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
 		sshConfig.Config.KeyExchanges = append(sshConfig.Config.KeyExchanges, "diffie-hellman-group-exchange-sha1", "diffie-hellman-group-exchange-sha256")
 	}
 
@@ -417,7 +439,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 				return nil, err
 			}
 		}
-		signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
+		var signer ssh.Signer
+		if clearpass == "" {
+			signer, err = ssh.ParsePrivateKey(key)
+		} else {
+			signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
+		}
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse private key file")
 		}
@@ -454,7 +481,7 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
 		config:    sshConfig,
 		url:       "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root,
 		mkdirLock: newStringLock(),
-		connLimit: rate.NewLimiter(rate.Limit(connectionsPerSecond), 1),
+		pacer:     fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
@@ -584,12 +611,16 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		remote := path.Join(dir, info.Name())
 		// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
 		// pick up the size and type of the destination, instead of the size and type of the symlink.
-		if !info.Mode().IsRegular() {
+		if !info.Mode().IsRegular() && !info.IsDir() {
+			if f.opt.SkipLinks {
+				// skip non regular file if SkipLinks is set
+				continue
+			}
 			oldInfo := info
 			info, err = f.stat(remote)
 			if err != nil {
 				if !os.IsNotExist(err) {
-					fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
+					fs.Errorf(remote, "stat of non-regular file failed: %v", err)
 				}
 				info = oldInfo
 			}
@@ -950,6 +981,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		escapedPath = shellEscape(path.Join(o.fs.opt.PathOverride, o.remote))
 	}
 	err = session.Run(hashCmd + " " + escapedPath)
+	fs.Debugf(nil, "sftp cmd = %s", escapedPath)
 	if err != nil {
 		_ = session.Close()
 		fs.Debugf(o, "Failed to calculate %v hash: %v (%s)", r, err, bytes.TrimSpace(stderr.Bytes()))
@@ -957,7 +989,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	}
 
 	_ = session.Close()
-	str := parseHash(stdout.Bytes())
+	b := stdout.Bytes()
+	fs.Debugf(nil, "sftp output = %q", b)
+	str := parseHash(b)
+	fs.Debugf(nil, "sftp hash = %q", str)
 	if r == hash.MD5 {
 		o.md5sum = &str
 	} else if r == hash.SHA1 {
@@ -966,7 +1001,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	return str, nil
 }
 
-var shellEscapeRegex = regexp.MustCompile(`[^A-Za-z0-9_.,:/@\n-]`)
+var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\n-]")
 
 // Escape a string s.t. it cannot cause unintended behavior
 // when sending it to a shell.
@@ -979,7 +1014,9 @@ func shellEscape(str string) string {
 // an invocation of md5sum/sha1sum to a hash string
 // as expected by the rest of this application
 func parseHash(bytes []byte) string {
-	return strings.Split(string(bytes), " ")[0] // Split at hash / filename separator
+	// For strings with backslash *sum writes a leading \
+	// https://unix.stackexchange.com/q/313733/94054
+	return strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0] // Split at hash / filename separator
 }
 
 // Parses the byte array output from the SSH session
@@ -1178,7 +1215,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if err != nil {
 		return errors.Wrap(err, "Update")
 	}
-	file, err := c.sftpClient.Create(o.path())
+	file, err := c.sftpClient.OpenFile(o.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
 	o.fs.putSftpConnection(&c, err)
 	if err != nil {
 		return errors.Wrap(err, "Update Create failed")

backend/sftp/sftp_test.go

@@ -8,13 +8,24 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/sftp"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestSftp:",
+		RemoteName: "TestSFTPOpenssh:",
+		NilObject:  (*sftp.Object)(nil),
+	})
+}
+
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSFTPRclone:",
 		NilObject:  (*sftp.Object)(nil),
 	})
 }

backend/sharefile/api/types.go

@@ -0,0 +1,152 @@
+// Package api contains definitions for using the premiumize.me API
+package api
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/pkg/errors"
+)
+
+// ListRequestSelect should be used in $select for Items/Children
+const ListRequestSelect = "odata.count,FileCount,Name,FileName,CreationDate,IsHidden,FileSizeBytes,odata.type,Id,Hash,ClientModifiedDate"
+
+// ListResponse is returned from the Items/Children call
+type ListResponse struct {
+	OdataCount int    `json:"odata.count"`
+	Value      []Item `json:"value"`
+}
+
+// Item Types
+const (
+	ItemTypeFolder = "ShareFile.Api.Models.Folder"
+	ItemTypeFile   = "ShareFile.Api.Models.File"
+)
+
+// Item refers to a file or folder
+type Item struct {
+	FileCount  int32     `json:"FileCount,omitempty"`
+	Name       string    `json:"Name,omitempty"`
+	FileName   string    `json:"FileName,omitempty"`
+	CreatedAt  time.Time `json:"CreationDate,omitempty"`
+	ModifiedAt time.Time `json:"ClientModifiedDate,omitempty"`
+	IsHidden   bool      `json:"IsHidden,omitempty"`
+	Size       int64     `json:"FileSizeBytes,omitempty"`
+	Type       string    `json:"odata.type,omitempty"`
+	ID         string    `json:"Id,omitempty"`
+	Hash       string    `json:"Hash,omitempty"`
+}
+
+// Error is an odata error return
+type Error struct {
+	Code    string `json:"code"`
+	Message struct {
+		Lang  string `json:"lang"`
+		Value string `json:"value"`
+	} `json:"message"`
+	Reason string `json:"reason"`
+}
+
+// Satisfy error interface
+func (e *Error) Error() string {
+	return fmt.Sprintf("%s: %s: %s", e.Message.Value, e.Code, e.Reason)
+}
+
+// Check Error satisfies error interface
+var _ error = &Error{}
+
+// DownloadSpecification is the response to /Items/Download
+type DownloadSpecification struct {
+	Token    string `json:"DownloadToken"`
+	URL      string `json:"DownloadUrl"`
+	Metadata string `json:"odata.metadata"`
+	Type     string `json:"odata.type"`
+}
+
+// UploadRequest is set to /Items/Upload2 to receive an UploadSpecification
+type UploadRequest struct {
+	Method         string    `json:"method"`                   // Upload method: one of: standard, streamed or threaded
+	Raw            bool      `json:"raw"`                      // Raw post if true or MIME upload if false
+	Filename       string    `json:"fileName"`                 // Uploaded item file name.
+	Filesize       *int64    `json:"fileSize,omitempty"`       // Uploaded item file size.
+	Overwrite      bool      `json:"overwrite"`                // Indicates whether items with the same name will be overwritten or not.
+	CreatedDate    time.Time `json:"ClientCreatedDate"`        // Created Date of this Item.
+	ModifiedDate   time.Time `json:"ClientModifiedDate"`       // Modified Date of this Item.
+	BatchID        string    `json:"batchId,omitempty"`        // Indicates part of a batch. Batched uploads do not send notification until the whole batch is completed.
+	BatchLast      *bool     `json:"batchLast,omitempty"`      // Indicates is the last in a batch. Upload notifications for the whole batch are sent after this upload.
+	CanResume      *bool     `json:"canResume,omitempty"`      // Indicates uploader supports resume.
+	StartOver      *bool     `json:"startOver,omitempty"`      // Indicates uploader wants to restart the file - i.e., ignore previous failed upload attempts.
+	Tool           string    `json:"tool,omitempty"`           // Identifies the uploader tool.
+	Title          string    `json:"title,omitempty"`          // Item Title
+	Details        string    `json:"details,omitempty"`        // Item description
+	IsSend         *bool     `json:"isSend,omitempty"`         // Indicates that this upload is part of a Send operation
+	SendGUID       string    `json:"sendGuid,omitempty"`       // Used if IsSend is true. Specifies which Send operation this upload is part of.
+	OpID           string    `json:"opid,omitempty"`           // Used for Asynchronous copy/move operations - called by Zones to push files to other Zones
+	ThreadCount    *int      `json:"threadCount,omitempty"`    // Specifies the number of threads the threaded uploader will use. Only used is method is threaded, ignored otherwise
+	Notify         *bool     `json:"notify,omitempty"`         // Indicates whether users will be notified of this upload - based on folder preferences
+	ExpirationDays *int      `json:"expirationDays,omitempty"` // File expiration days
+	BaseFileID     string    `json:"baseFileId,omitempty"`     // Used to check conflict in file during File Upload.
+}
+
+// UploadSpecification is returned from /Items/Upload
+type UploadSpecification struct {
+	Method             string `json:"Method"`             // The Upload method that must be used for this upload
+	PrepareURI         string `json:"PrepareUri"`         // If provided, clients must issue a request to this Uri before uploading any data.
+	ChunkURI           string `json:"ChunkUri"`           // Specifies the URI the client must send the file data to
+	FinishURI          string `json:"FinishUri"`          // If provided, specifies the final call the client must perform to finish the upload process
+	ProgressData       string `json:"ProgressData"`       // Allows the client to check progress of standard uploads
+	IsResume           bool   `json:"IsResume"`           // Specifies a Resumable upload is supproted.
+	ResumeIndex        int64  `json:"ResumeIndex"`        // Specifies the initial index for resuming, if IsResume is true.
+	ResumeOffset       int64  `json:"ResumeOffset"`       // Specifies the initial file offset by bytes, if IsResume is true
+	ResumeFileHash     string `json:"ResumeFileHash"`     // Specifies the MD5 hash of the first ResumeOffset bytes of the partial file found at the server
+	MaxNumberOfThreads int    `json:"MaxNumberOfThreads"` // Specifies the max number of chunks that can be sent simultaneously for threaded uploads
+}
+
+// UploadFinishResponse is returnes from calling UploadSpecification.FinishURI
+type UploadFinishResponse struct {
+	Error        bool   `json:"error"`
+	ErrorMessage string `json:"errorMessage"`
+	ErrorCode    int    `json:"errorCode"`
+	Value        []struct {
+		UploadID    string `json:"uploadid"`
+		ParentID    string `json:"parentid"`
+		ID          string `json:"id"`
+		StreamID    string `json:"streamid"`
+		FileName    string `json:"filename"`
+		DisplayName string `json:"displayname"`
+		Size        int    `json:"size"`
+		Md5         string `json:"md5"`
+	} `json:"value"`
+}
+
+// ID returns the ID of the first response if available
+func (finish *UploadFinishResponse) ID() (string, error) {
+	if finish.Error {
+		return "", errors.Errorf("upload failed: %s (%d)", finish.ErrorMessage, finish.ErrorCode)
+	}
+	if len(finish.Value) == 0 {
+		return "", errors.New("upload failed: no results returned")
+	}
+	return finish.Value[0].ID, nil
+}
+
+// Parent is the ID of the parent folder
+type Parent struct {
+	ID string `json:"Id,omitempty"`
+}
+
+// Zone is where the data is stored
+type Zone struct {
+	ID string `json:"Id,omitempty"`
+}
+
+// UpdateItemRequest is sent to PATCH /v3/Items(id)
+type UpdateItemRequest struct {
+	Name           string     `json:"Name,omitempty"`
+	FileName       string     `json:"FileName,omitempty"`
+	Description    string     `json:"Description,omitempty"`
+	ExpirationDate *time.Time `json:"ExpirationDate,omitempty"`
+	Parent         *Parent    `json:"Parent,omitempty"`
+	Zone           *Zone      `json:"Zone,omitempty"`
+	ModifiedAt     *time.Time `json:"ClientModifiedDate,omitempty"`
+}

backend/sharefile/generate_tzdata.go

@@ -0,0 +1,22 @@
+// +build ignore
+
+package main
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/shurcooL/vfsgen"
+)
+
+func main() {
+	var AssetDir http.FileSystem = http.Dir("./tzdata")
+	err := vfsgen.Generate(AssetDir, vfsgen.Options{
+		PackageName:  "sharefile",
+		BuildTags:    "!dev",
+		VariableName: "tzdata",
+	})
+	if err != nil {
+		log.Fatalln(err)
+	}
+}

backend/sharefile/sharefile_test.go

@@ -0,0 +1,34 @@
+// Test filesystem interface
+package sharefile
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSharefile:",
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize:  minChunkSize,
+			CeilChunkSize: fstests.NextPowerOfTwo,
+		},
+	})
+}
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadChunkSize(cs)
+}
+
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadCutoff(cs)
+}
+
+var (
+	_ fstests.SetUploadChunkSizer = (*Fs)(nil)
+	_ fstests.SetUploadCutoffer   = (*Fs)(nil)
+)

backend/sharefile/tzdata_vfsdata.go

@@ -0,0 +1,193 @@
+// Code generated by vfsgen; DO NOT EDIT.
+
+// +build !dev
+
+package sharefile
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	pathpkg "path"
+	"time"
+)
+
+// tzdata statically implements the virtual filesystem provided to vfsgen.
+var tzdata = func() http.FileSystem {
+	fs := vfsgen۰FS{
+		"/": &vfsgen۰DirInfo{
+			name:    "/",
+			modTime: time.Date(2019, 9, 12, 14, 55, 27, 600751842, time.UTC),
+		},
+		"/America": &vfsgen۰DirInfo{
+			name:    "America",
+			modTime: time.Date(2019, 9, 12, 14, 55, 27, 600751842, time.UTC),
+		},
+		"/America/New_York": &vfsgen۰CompressedFileInfo{
+			name:             "New_York",
+			modTime:          time.Date(2019, 7, 2, 0, 44, 57, 0, time.UTC),
+			uncompressedSize: 3536,
+
+			compressedContent: []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\xd6\x7f\x54\xd5\xf5\x1d\xc7\xf1\xaf\x4a\x6a\x28\xa1\x2b\x9a\xa6\xb1\xa6\xdb\x08\x13\xf0\x47\xe4\x2f\xb6\x68\xc9\x18\xda\x0d\xd3\x4b\x22\x39\xfc\xd4\x20\x0e\xea\xc6\x67\xfd\x20\xdc\x0c\xdb\xb4\x18\x9e\xd3\xdc\x4e\x1e\xd8\x56\xe7\x36\x53\x94\x5c\x44\x0a\x38\xc9\x76\xfd\xc5\xc8\xe3\x8e\xab\xb8\x66\xd8\x8f\xeb\x5b\xa7\x77\x65\x92\x1f\xa4\x8c\xc9\xba\x77\xe7\xf9\x9e\xff\xed\x9f\xf9\xdf\xfe\x48\x4f\x3e\xfc\x9c\x13\xf0\x8f\xf7\xf5\x7c\xfb\x8b\xca\x1f\x9c\xe6\xfd\xd7\xaf\xab\x2e\xff\xc7\xaf\x73\x97\xff\x7e\xdd\x13\x9e\xe7\x05\xb6\x26\xdb\xe7\x5f\xfd\xd8\xfc\xe1\x29\xcf\x6e\xfa\xfd\x11\xf3\x42\xe9\x29\xbb\x79\xed\x47\xb2\x65\xf9\xcb\xb6\x21\x73\x9b\xd9\xba\xe8\xb0\xdb\x96\x54\x6b\x1a\xa7\xbf\xe4\x1a\xa3\x0d\xb2\xfd\xda\x5f\xb9\xed\xe1\x1a\xf9\x63\x9f\x75\x2f\x05\xcb\xa5\x29\xb4\xd0\xbd\x1c\x98\x2f\xcd\x2d\xb7\xba\x57\xaa\xd3\x64\xc7\x73\xf7\xd8\x9d\x65\xf3\x4c\xcb\xea\xe9\xb6\x35\x77\xb2\x69\x5b\x9a\x64\x77\xa5\x5c\x63\xfe\x34\xe7\x73\xbb\x7b\xa8\x33\xed\xe3\x8e\xdb\xf6\x48\x97\xd9\x13\xf7\x99\xdb\xd3\xd9\x6a\x5e\x3b\xfd\x8e\xfb\x73\xf3\x9b\x12\xec\x68\x77\x7b\x37\xec\x94\x7d\x5b\x9e\x75\xfb\x2b\x36\xca\x81\x75\x8f\xbb\x83\xf9\x95\xd2\x51\xb2\xcc\xfd\x25\xa3\x50\x3a\x7d\xab\xed\xeb\x89\xb3\xe5\x50\x5a\xb1\x3d\xd4\xbf\xd8\x1c\x4e\xc8\xb6\x87\xbb\x67\x99\xbf\xfe\xd2\xd9\xae\x89\x9f\xda\x2e\x33\x20\xa1\x47\x4f\xbb\xa3\xd9\x1f\xc8\xdb\x05\x9d\xee\xd8\x4d\x7b\xe5\x9d\xcc\x46\xd7\xed\x6d\x92\xe3\x49\xeb\xdd\x71\x59\x2b\xef\x46\xb7\xd9\xf7\xf6\x95\xca\xfb\xe1\x5a\xfb\xc1\x8b\xbf\x30\xe1\xe0\x0a\x7b\xa2\xb6\xc4\x48\xc0\x67\x4f\x96\x7f\xcf\x9c\xaa\xce\xb0\x7f\xcf\xbb\xd9\x9c\x2e\x1e\x6d\xcf\x2c\x4e\x97\x48\x6e\x9a\xfb\xc7\x8c\x51\xf2\x61\x4a\xa2\xfb\xe8\xfa\x0b\x72\x76\x68\xaf\x3b\x7b\xf1\xa8\x7c\x1c\x09\xb9\x73\xc7\x76\x49\x4f\x67\x9b\xfb\x64\x6f\xc8\x9c\x6f\xee\xb2\xee\xf9\x36\xd3\xbb\xa1\xd5\x5e\x58\x53\x6f\xfa\x2a\xea\xec\xa7\xcb\x56\x99\xcf\xf2\xab\xec\xc5\xdb\xef\x33\x9f\x67\x14\xd9\xfe\x9b\x1f\x93\x7f\x26\x66\xd9\x4b\xc3\x97\xc8\xa5\xfe\x42\x37\xf0\xe1\x1c\xf9\x57\xf7\x6c\xf7\xc5\xa1\x1b\x25\xba\x7b\xbc\x8b\x6d\x8b\x89\x57\x1f\x75\x83\x6a\x4e\xca\xe0\xc7\xc4\x0d\xb1\x51\x13\x67\xbe\xb0\x57\x2d\x10\x33\x34\xfb\x84\x1d\x36\xe5\x80\x19\xf6\xf4\x58\x7b\xf5\xa8\x2d\xe6\xea\xa6\x8d\x2e\xde\x3d\x65\x46\xc8\x93\x76\xe4\xf1\x17\x24\x61\x5f\x99\xbd\xa6\x7d\x9d\x24\xbe\xb8\xd6\x8d\xfa\xdd\x83\x32\xba\xb6\xd4\x7d\x65\xd5\xf7\xe5\xda\xf2\x5c\x77\xdd\x92\x49\x92\x94\x97\xea\xae\x9f\x35\x52\xbe\x9a\x3a\xc2\x8d\x99\x90\x6a\xc6\x0e\xef\x71\x37\x0c\x1e\x61\x6e\xb8\x10\x6f\xc7\x9d\xec\x31\xe3\xdf\x3a\x67\x6f\xdc\xff\x86\x49\xde\xf1\x37\xfb\xb5\x4d\x3b\xcc\x4d\x95\x9e\xfb\xfa\xcf\x9f\x31\x13\x0a\x4e\xb9\x89\xcb\x9b\xe5\x1b\x99\x1d\xee\x9b\xf3\x7e\x23\xdf\x4a\xda\xea\x52\x26\x3d\x2c\x29\xd1\x83\x36\x35\xbe\x40\x52\xc3\x0d\x76\xd2\xd9\x19\x72\x4b\xb0\xc6\x4e\x0e\xf9\x4d\x5a\xa0\xdc\xa6\xb7\xdc\x66\x32\xaa\xe7\xdb\x29\xcf\x8c\x31\x53\xcb\xca\xdc\xb4\x87\x2e\x99\xe9\xb9\x79\xee\xd6\x85\xef\x9b\xcc\x94\x5b\xdc\x6d\xd3\x82\x66\xc6\xb0\x04\x37\x73\xdc\xbb\x32\x33\x72\xde\xcd\x1a\x78\x4d\x66\x77\xbe\xe5\xe6\xbc\x17\x90\xac\xe6\x4f\xec\xb7\xf7\x3c\x21\xdf\xd9\xf0\xa6\xbd\xfd\xd9\x07\x24\xbb\x62\xa7\xbd\x63\xdd\x1a\xf3\xdd\xfc\x8d\xf6\xce\x92\xfb\xcd\xdc\x8c\x4a\x9b\x33\xf7\x4e\x93\xd3\xd7\xe8\x72\xd3\x96\x49\x6e\x68\xbd\xcb\x4b\xb8\x43\xf2\x5a\x56\xba\x79\x3d\x13\x65\xfe\x73\xb5\xf6\xae\x63\xd9\xc6\xb7\x7a\x85\xbd\x7b\xd7\x04\x93\xbf\xd4\x67\x17\xd4\xc5\x99\x7b\xb2\x32\xec\xc2\x47\x23\x66\xd1\xf8\xd1\xd6\x5f\x70\xc8\xf8\x07\xfa\xec\xbd\x99\xdb\xcd\xbd\x67\x12\x5d\x61\x72\xa7\x14\x76\xf4\xba\x25\xd1\x46\x29\xda\x12\x72\xf7\x85\xd7\xcb\xd2\x75\x6d\xee\x07\xc1\x95\x52\x5c\x52\xef\x96\x05\xee\x16\xe3\x6b\xb5\xf7\xd7\xac\x30\x0f\xa4\xd5\xd9\x1f\x96\xf9\x4c\x49\x42\x95\x2d\xcd\xcd\x30\xa5\x3d\x45\xb6\x2c\x65\xb4\x29\x3b\x92\x65\xcb\x87\xf6\x99\xf2\xa6\x64\xbb\x3c\xf2\xb6\x59\x51\x37\xdb\xad\x7c\xa3\x57\x7e\x54\x39\xde\xfd\xb8\x39\x24\x15\x05\x51\x67\x37\xb4\xc9\x4f\x32\xc5\x3d\x54\x51\x2f\x0f\x27\x1d\x70\x8f\xe4\xaf\x92\x47\xa2\x27\x6c\xe5\xcc\x3a\x53\x19\xde\x6f\xab\xc6\x54\x99\xaa\xe0\x66\xbb\xaa\xbf\xc8\xfc\x34\xf0\xa4\xfd\x59\x77\x96\x59\x5d\x5d\x66\x1f\xdf\x9d\x6c\xaa\x8b\xf3\xec\x9a\xdf\x7a\x66\xf0\xa0\x2b\xfc\x3d\x24\xee\x8a\xbf\xe4\xff\xe5\x77\x2c\xf6\x6a\xc0\xf3\x62\xb1\xd7\xf7\x0d\x8a\x8b\xc5\xda\x5f\xf1\x86\xeb\xdf\x47\xea\x9f\xa3\xee\xf2\xf9\xbd\x9c\xb9\x7e\x2f\x67\x91\xdf\xcb\x59\xec\xf7\x72\x16\xf8\x75\xda\x06\xe9\x1f\x57\xb2\x81\xb1\x58\x2c\x56\x3c\xc4\xfd\x1a\xd9\x42\x64\x0f\x91\x4d\x44\x76\x11\xd9\x46\x64\x1f\x91\x8d\x44\x76\x12\xd9\x4a\x64\x2f\x91\xcd\x54\xa3\x0d\xfa\xff\xb3\x9d\x6a\xb8\x46\xdf\x6c\x28\xb2\xa3\xc8\x96\x22\x7b\x8a\x6c\x2a\xb2\xab\xc8\xb6\x22\xfb\x8a\x6c\x2c\xb2\xb3\xc8\xd6\x22\x7b\x8b\x6c\x2e\xb2\xbb\xc8\xf6\xaa\x91\x2e\x7d\xb3\xc1\x6a\x67\xab\xbe\xd9\x62\x64\x8f\x91\x4d\x46\x76\x19\xd9\x66\x64\x9f\x91\x8d\x46\x76\x1a\xd9\x6a\x64\xaf\x91\xcd\x46\x76\x1b\xd9\x6e\xb5\x7f\xb1\xfe\x3c\x36\x5c\xed\x9e\xa5\x6f\xb6\x1c\xd9\x73\xd5\x0c\xe8\xd7\xb1\xeb\Line truncated
+		},
+	}
+	fs["/"].(*vfsgen۰DirInfo).entries = []os.FileInfo{
+		fs["/America"].(os.FileInfo),
+	}
+	fs["/America"].(*vfsgen۰DirInfo).entries = []os.FileInfo{
+		fs["/America/New_York"].(os.FileInfo),
+	}
+
+	return fs
+}()
+
+type vfsgen۰FS map[string]interface{}
+
+func (fs vfsgen۰FS) Open(path string) (http.File, error) {
+	path = pathpkg.Clean("/" + path)
+	f, ok := fs[path]
+	if !ok {
+		return nil, &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist}
+	}
+
+	switch f := f.(type) {
+	case *vfsgen۰CompressedFileInfo:
+		gr, err := gzip.NewReader(bytes.NewReader(f.compressedContent))
+		if err != nil {
+			// This should never happen because we generate the gzip bytes such that they are always valid.
+			panic("unexpected error reading own gzip compressed bytes: " + err.Error())
+		}
+		return &vfsgen۰CompressedFile{
+			vfsgen۰CompressedFileInfo: f,
+			gr:                        gr,
+		}, nil
+	case *vfsgen۰DirInfo:
+		return &vfsgen۰Dir{
+			vfsgen۰DirInfo: f,
+		}, nil
+	default:
+		// This should never happen because we generate only the above types.
+		panic(fmt.Sprintf("unexpected type %T", f))
+	}
+}
+
+// vfsgen۰CompressedFileInfo is a static definition of a gzip compressed file.
+type vfsgen۰CompressedFileInfo struct {
+	name              string
+	modTime           time.Time
+	compressedContent []byte
+	uncompressedSize  int64
+}
+
+func (f *vfsgen۰CompressedFileInfo) Readdir(count int) ([]os.FileInfo, error) {
+	return nil, fmt.Errorf("cannot Readdir from file %s", f.name)
+}
+func (f *vfsgen۰CompressedFileInfo) Stat() (os.FileInfo, error) { return f, nil }
+
+func (f *vfsgen۰CompressedFileInfo) GzipBytes() []byte {
+	return f.compressedContent
+}
+
+func (f *vfsgen۰CompressedFileInfo) Name() string       { return f.name }
+func (f *vfsgen۰CompressedFileInfo) Size() int64        { return f.uncompressedSize }
+func (f *vfsgen۰CompressedFileInfo) Mode() os.FileMode  { return 0444 }
+func (f *vfsgen۰CompressedFileInfo) ModTime() time.Time { return f.modTime }
+func (f *vfsgen۰CompressedFileInfo) IsDir() bool        { return false }
+func (f *vfsgen۰CompressedFileInfo) Sys() interface{}   { return nil }
+
+// vfsgen۰CompressedFile is an opened compressedFile instance.
+type vfsgen۰CompressedFile struct {
+	*vfsgen۰CompressedFileInfo
+	gr      *gzip.Reader
+	grPos   int64 // Actual gr uncompressed position.
+	seekPos int64 // Seek uncompressed position.
+}
+
+func (f *vfsgen۰CompressedFile) Read(p []byte) (n int, err error) {
+	if f.grPos > f.seekPos {
+		// Rewind to beginning.
+		err = f.gr.Reset(bytes.NewReader(f.compressedContent))
+		if err != nil {
+			return 0, err
+		}
+		f.grPos = 0
+	}
+	if f.grPos < f.seekPos {
+		// Fast-forward.
+		_, err = io.CopyN(ioutil.Discard, f.gr, f.seekPos-f.grPos)
+		if err != nil {
+			return 0, err
+		}
+		f.grPos = f.seekPos
+	}
+	n, err = f.gr.Read(p)
+	f.grPos += int64(n)
+	f.seekPos = f.grPos
+	return n, err
+}
+func (f *vfsgen۰CompressedFile) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		f.seekPos = 0 + offset
+	case io.SeekCurrent:
+		f.seekPos += offset
+	case io.SeekEnd:
+		f.seekPos = f.uncompressedSize + offset
+	default:
+		panic(fmt.Errorf("invalid whence value: %v", whence))
+	}
+	return f.seekPos, nil
+}
+func (f *vfsgen۰CompressedFile) Close() error {
+	return f.gr.Close()
+}
+
+// vfsgen۰DirInfo is a static definition of a directory.
+type vfsgen۰DirInfo struct {
+	name    string
+	modTime time.Time
+	entries []os.FileInfo
+}
+
+func (d *vfsgen۰DirInfo) Read([]byte) (int, error) {
+	return 0, fmt.Errorf("cannot Read from directory %s", d.name)
+}
+func (d *vfsgen۰DirInfo) Close() error               { return nil }
+func (d *vfsgen۰DirInfo) Stat() (os.FileInfo, error) { return d, nil }
+
+func (d *vfsgen۰DirInfo) Name() string       { return d.name }
+func (d *vfsgen۰DirInfo) Size() int64        { return 0 }
+func (d *vfsgen۰DirInfo) Mode() os.FileMode  { return 0755 | os.ModeDir }
+func (d *vfsgen۰DirInfo) ModTime() time.Time { return d.modTime }
+func (d *vfsgen۰DirInfo) IsDir() bool        { return true }
+func (d *vfsgen۰DirInfo) Sys() interface{}   { return nil }
+
+// vfsgen۰Dir is an opened dir instance.
+type vfsgen۰Dir struct {
+	*vfsgen۰DirInfo
+	pos int // Position within entries for Seek and Readdir.
+}
+
+func (d *vfsgen۰Dir) Seek(offset int64, whence int) (int64, error) {
+	if offset == 0 && whence == io.SeekStart {
+		d.pos = 0
+		return 0, nil
+	}
+	return 0, fmt.Errorf("unsupported Seek in directory %s", d.name)
+}
+
+func (d *vfsgen۰Dir) Readdir(count int) ([]os.FileInfo, error) {
+	if d.pos >= len(d.entries) && count > 0 {
+		return nil, io.EOF
+	}
+	if count <= 0 || count > len(d.entries)-d.pos {
+		count = len(d.entries) - d.pos
+	}
+	e := d.entries[d.pos : d.pos+count]
+	d.pos += count
+	return e, nil
+}

backend/sharefile/update-timezone.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+# Extract just the America/New_York timezone from
+tzinfo=$(go env GOROOT)/lib/time/zoneinfo.zip
+
+rm -rf tzdata
+mkdir tzdata
+cd tzdata
+unzip ${tzinfo} America/New_York
+
+cd ..
+# Make the embedded assets
+go run generate_tzdata.go
+
+# tidy up
+rm -rf tzdata

backend/sharefile/upload.go

@@ -0,0 +1,261 @@
+// Upload large files for sharefile
+//
+// Docs - https://api.sharefile.com/rest/docs/resource.aspx?name=Items#Upload_File
+
+package sharefile
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/backend/sharefile/api"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/readers"
+	"github.com/rclone/rclone/lib/rest"
+)
+
+// largeUpload is used to control the upload of large files which need chunking
+type largeUpload struct {
+	ctx      context.Context
+	f        *Fs                      // parent Fs
+	o        *Object                  // object being uploaded
+	in       io.Reader                // read the data from here
+	wrap     accounting.WrapFn        // account parts being transferred
+	size     int64                    // total size
+	parts    int64                    // calculated number of parts, if known
+	info     *api.UploadSpecification // where to post chunks etc
+	threads  int                      // number of threads to use in upload
+	streamed bool                     // set if using streamed upload
+}
+
+// newLargeUpload starts an upload of object o from in with metadata in src
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, info *api.UploadSpecification) (up *largeUpload, err error) {
+	size := src.Size()
+	parts := int64(-1)
+	if size >= 0 {
+		parts = size / int64(o.fs.opt.ChunkSize)
+		if size%int64(o.fs.opt.ChunkSize) != 0 {
+			parts++
+		}
+	}
+
+	var streamed bool
+	switch strings.ToLower(info.Method) {
+	case "streamed":
+		streamed = true
+	case "threaded":
+		streamed = false
+	default:
+		return nil, errors.Errorf("can't use method %q with newLargeUpload", info.Method)
+	}
+
+	threads := fs.Config.Transfers
+	if threads > info.MaxNumberOfThreads {
+		threads = info.MaxNumberOfThreads
+	}
+
+	// unwrap the accounting from the input, we use wrap to put it
+	// back on after the buffering
+	in, wrap := accounting.UnWrap(in)
+	up = &largeUpload{
+		ctx:      ctx,
+		f:        f,
+		o:        o,
+		in:       in,
+		wrap:     wrap,
+		size:     size,
+		threads:  threads,
+		info:     info,
+		parts:    parts,
+		streamed: streamed,
+	}
+	return up, nil
+}
+
+// parse the api.UploadFinishResponse in respBody
+func (up *largeUpload) parseUploadFinishResponse(respBody []byte) (err error) {
+	var finish api.UploadFinishResponse
+	err = json.Unmarshal(respBody, &finish)
+	if err != nil {
+		// Sometimes the unmarshal fails in which case return the body
+		return errors.Errorf("upload: bad response: %q", bytes.TrimSpace(respBody))
+	}
+	return up.o.checkUploadResponse(up.ctx, &finish)
+}
+
+// Transfer a chunk
+func (up *largeUpload) transferChunk(ctx context.Context, part int64, offset int64, body []byte, fileHash string) error {
+	md5sumRaw := md5.Sum(body)
+	md5sum := hex.EncodeToString(md5sumRaw[:])
+	size := int64(len(body))
+
+	// Add some more parameters to the ChunkURI
+	u := up.info.ChunkURI
+	u += fmt.Sprintf("&index=%d&byteOffset=%d&hash=%s&fmt=json",
+		part, offset, md5sum,
+	)
+	if fileHash != "" {
+		u += fmt.Sprintf("&finish=true&fileSize=%d&fileHash=%s",
+			offset+int64(len(body)),
+			fileHash,
+		)
+	}
+	opts := rest.Opts{
+		Method:        "POST",
+		RootURL:       u,
+		ContentLength: &size,
+	}
+	var respBody []byte
+	err := up.f.pacer.Call(func() (bool, error) {
+		fs.Debugf(up.o, "Sending chunk %d length %d", part, len(body))
+		opts.Body = up.wrap(bytes.NewReader(body))
+		resp, err := up.f.srv.Call(ctx, &opts)
+		if err != nil {
+			fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
+		} else {
+			respBody, err = rest.ReadBody(resp)
+		}
+		// retry all errors now that the multipart upload has started
+		return err != nil, err
+	})
+	if err != nil {
+		fs.Debugf(up.o, "Error sending chunk %d: %v", part, err)
+		return err
+	}
+	// If last chunk and using "streamed" transfer, get the response back now
+	if up.streamed && fileHash != "" {
+		return up.parseUploadFinishResponse(respBody)
+	}
+	fs.Debugf(up.o, "Done sending chunk %d", part)
+	return nil
+}
+
+// finish closes off the large upload and reads the metadata
+func (up *largeUpload) finish(ctx context.Context) error {
+	fs.Debugf(up.o, "Finishing large file upload")
+	// For a streamed transfer we will already have read the info
+	if up.streamed {
+		return nil
+	}
+
+	opts := rest.Opts{
+		Method:  "POST",
+		RootURL: up.info.FinishURI,
+	}
+	var respBody []byte
+	err := up.f.pacer.Call(func() (bool, error) {
+		resp, err := up.f.srv.Call(ctx, &opts)
+		if err != nil {
+			return shouldRetry(resp, err)
+		}
+		respBody, err = rest.ReadBody(resp)
+		// retry all errors now that the multipart upload has started
+		return err != nil, err
+	})
+	if err != nil {
+		return err
+	}
+	return up.parseUploadFinishResponse(respBody)
+}
+
+// Upload uploads the chunks from the input
+func (up *largeUpload) Upload(ctx context.Context) error {
+	if up.parts >= 0 {
+		fs.Debugf(up.o, "Starting upload of large file in %d chunks", up.parts)
+	} else {
+		fs.Debugf(up.o, "Starting streaming upload of large file")
+	}
+	var (
+		offset        int64
+		errs          = make(chan error, 1)
+		wg            sync.WaitGroup
+		err           error
+		wholeFileHash = md5.New()
+		eof           = false
+	)
+outer:
+	for part := int64(0); !eof; part++ {
+		// Check any errors
+		select {
+		case err = <-errs:
+			break outer
+		default:
+		}
+
+		// Get a block of memory
+		buf := up.f.getUploadBlock()
+
+		// Read the chunk
+		var n int
+		n, err = readers.ReadFill(up.in, buf)
+		if err == io.EOF {
+			eof = true
+			buf = buf[:n]
+			err = nil
+		} else if err != nil {
+			up.f.putUploadBlock(buf)
+			break outer
+		}
+
+		// Hash it
+		_, _ = io.Copy(wholeFileHash, bytes.NewBuffer(buf))
+
+		// Get file hash if was last chunk
+		fileHash := ""
+		if eof {
+			fileHash = hex.EncodeToString(wholeFileHash.Sum(nil))
+		}
+
+		// Transfer the chunk
+		wg.Add(1)
+		transferChunk := func(part, offset int64, buf []byte, fileHash string) {
+			defer wg.Done()
+			defer up.f.putUploadBlock(buf)
+			err := up.transferChunk(ctx, part, offset, buf, fileHash)
+			if err != nil {
+				select {
+				case errs <- err:
+				default:
+				}
+			}
+		}
+		if up.streamed {
+			transferChunk(part, offset, buf, fileHash) // streamed
+		} else {
+			go transferChunk(part, offset, buf, fileHash) // multithreaded
+		}
+
+		offset += int64(n)
+	}
+	wg.Wait()
+
+	// check size read is correct
+	if eof && err == nil && up.size >= 0 && up.size != offset {
+		err = errors.Errorf("upload: short read: read %d bytes expected %d", up.size, offset)
+	}
+
+	// read any errors
+	if err == nil {
+		select {
+		case err = <-errs:
+		default:
+		}
+	}
+
+	// finish regardless of errors
+	finishErr := up.finish(ctx)
+	if err == nil {
+		err = finishErr
+	}
+
+	return err
+}

backend/sugarsync/api/types.go

@@ -0,0 +1,160 @@
+// Package api has type definitions for sugarsync
+//
+// Converted from the API docs with help from https://www.onlinetool.io/xmltogo/
+package api
+
+import (
+	"encoding/xml"
+	"time"
+)
+
+// AppAuthorization is used to request a refresh token
+//
+// The token is returned in the Location: field
+type AppAuthorization struct {
+	XMLName          xml.Name `xml:"appAuthorization"`
+	Username         string   `xml:"username"`
+	Password         string   `xml:"password"`
+	Application      string   `xml:"application"`
+	AccessKeyID      string   `xml:"accessKeyId"`
+	PrivateAccessKey string   `xml:"privateAccessKey"`
+}
+
+// TokenAuthRequest is the request to get Authorization
+type TokenAuthRequest struct {
+	XMLName          xml.Name `xml:"tokenAuthRequest"`
+	AccessKeyID      string   `xml:"accessKeyId"`
+	PrivateAccessKey string   `xml:"privateAccessKey"`
+	RefreshToken     string   `xml:"refreshToken"`
+}
+
+// Authorization is returned from the TokenAuthRequest
+type Authorization struct {
+	XMLName    xml.Name  `xml:"authorization"`
+	Expiration time.Time `xml:"expiration"`
+	User       string    `xml:"user"`
+}
+
+// File represents a single file
+type File struct {
+	Name            string    `xml:"displayName"`
+	Ref             string    `xml:"ref"`
+	DsID            string    `xml:"dsid"`
+	TimeCreated     time.Time `xml:"timeCreated"`
+	Parent          string    `xml:"parent"`
+	Size            int64     `xml:"size"`
+	LastModified    time.Time `xml:"lastModified"`
+	MediaType       string    `xml:"mediaType"`
+	PresentOnServer bool      `xml:"presentOnServer"`
+	FileData        string    `xml:"fileData"`
+	Versions        string    `xml:"versions"`
+	PublicLink      PublicLink
+}
+
+// Collection represents
+// - Workspace Collection
+// - Sync Folders collection
+// - Folder
+type Collection struct {
+	Type        string    `xml:"type,attr"`
+	Name        string    `xml:"displayName"`
+	Ref         string    `xml:"ref"` // only for Folder
+	DsID        string    `xml:"dsid"`
+	TimeCreated time.Time `xml:"timeCreated"`
+	Parent      string    `xml:"parent"`
+	Collections string    `xml:"collections"`
+	Files       string    `xml:"files"`
+	Contents    string    `xml:"contents"`
+	// Sharing     bool      `xml:"sharing>enabled,attr"`
+}
+
+// CollectionContents is the result of a list call
+type CollectionContents struct {
+	//XMLName     xml.Name     `xml:"collectionContents"`
+	Start       int          `xml:"start,attr"`
+	HasMore     bool         `xml:"hasMore,attr"`
+	End         int          `xml:"end,attr"`
+	Collections []Collection `xml:"collection"`
+	Files       []File       `xml:"file"`
+}
+
+// User is returned from the /user call
+type User struct {
+	XMLName  xml.Name `xml:"user"`
+	Username string   `xml:"username"`
+	Nickname string   `xml:"nickname"`
+	Quota    struct {
+		Limit int64 `xml:"limit"`
+		Usage int64 `xml:"usage"`
+	} `xml:"quota"`
+	Workspaces            string `xml:"workspaces"`
+	SyncFolders           string `xml:"syncfolders"`
+	Deleted               string `xml:"deleted"`
+	MagicBriefcase        string `xml:"magicBriefcase"`
+	WebArchive            string `xml:"webArchive"`
+	MobilePhotos          string `xml:"mobilePhotos"`
+	Albums                string `xml:"albums"`
+	RecentActivities      string `xml:"recentActivities"`
+	ReceivedShares        string `xml:"receivedShares"`
+	PublicLinks           string `xml:"publicLinks"`
+	MaximumPublicLinkSize int    `xml:"maximumPublicLinkSize"`
+}
+
+// CreateFolder is posted to a folder URL to create a folder
+type CreateFolder struct {
+	XMLName xml.Name `xml:"folder"`
+	Name    string   `xml:"displayName"`
+}
+
+// MoveFolder is posted to a folder URL to move a folder
+type MoveFolder struct {
+	XMLName xml.Name `xml:"folder"`
+	Name    string   `xml:"displayName"`
+	Parent  string   `xml:"parent"`
+}
+
+// CreateSyncFolder is posted to the root folder URL to create a sync folder
+type CreateSyncFolder struct {
+	XMLName xml.Name `xml:"syncFolder"`
+	Name    string   `xml:"displayName"`
+}
+
+// CreateFile is posted to a folder URL to create a file
+type CreateFile struct {
+	XMLName   xml.Name `xml:"file"`
+	Name      string   `xml:"displayName"`
+	MediaType string   `xml:"mediaType"`
+}
+
+// MoveFile is posted to a file URL to create a file
+type MoveFile struct {
+	XMLName xml.Name `xml:"file"`
+	Name    string   `xml:"displayName"`
+	Parent  string   `xml:"parent"`
+}
+
+// CopyFile copies a file from source
+type CopyFile struct {
+	XMLName xml.Name `xml:"fileCopy"`
+	Source  string   `xml:"source,attr"`
+	Name    string   `xml:"displayName"`
+}
+
+// PublicLink is the URL and enabled flag for a public link
+type PublicLink struct {
+	XMLName xml.Name `xml:"publicLink"`
+	URL     string   `xml:",chardata"`
+	Enabled bool     `xml:"enabled,attr"`
+}
+
+// SetPublicLink can be used to enable the file for sharing
+type SetPublicLink struct {
+	XMLName    xml.Name `xml:"file"`
+	PublicLink PublicLink
+}
+
+// SetLastModified sets the modified time for a file
+type SetLastModified struct {
+	XMLName      xml.Name  `xml:"file"`
+	LastModified time.Time `xml:"lastModified"`
+}

backend/sugarsync/sugarsync_internal_test.go

@@ -0,0 +1,59 @@
+package sugarsync
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestErrorHandler(t *testing.T) {
+	for _, test := range []struct {
+		name   string
+		body   string
+		code   int
+		status string
+		want   string
+	}{
+		{
+			name:   "empty",
+			body:   "",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error) returned body: ""`,
+		},
+		{
+			name:   "unknown",
+			body:   "<h1>unknown</h1>",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error) returned body: "<h1>unknown</h1>"`,
+		},
+		{
+			name:   "blank",
+			body:   "Nothing here <h3></h3>",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error) returned body: "Nothing here <h3></h3>"`,
+		},
+		{
+			name:   "real",
+			body:   "<h1>an error</h1>\n<h3>Can not move sync folder.</h3>\n<p>more stuff</p>",
+			code:   500,
+			status: "internal error",
+			want:   `HTTP error 500 (internal error): Can not move sync folder.`,
+		},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			resp := http.Response{
+				Body:       ioutil.NopCloser(bytes.NewBufferString(test.body)),
+				StatusCode: test.code,
+				Status:     test.status,
+			}
+			got := errorHandler(&resp)
+			assert.Equal(t, test.want, got.Error())
+		})
+	}
+}

backend/sugarsync/sugarsync_test.go

@@ -0,0 +1,17 @@
+// Test Sugarsync filesystem interface
+package sugarsync_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/sugarsync"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSugarSync:Test",
+		NilObject:  (*sugarsync.Object)(nil),
+	})
+}

backend/swift/swift.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/url"
 	"path"
 	"strconv"
 	"strings"
@@ -15,6 +16,7 @@ import (
 	"github.com/ncw/swift"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fserrors"
@@ -23,6 +25,7 @@ import (
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 )
@@ -58,6 +61,12 @@ Rclone will still chunk files bigger than chunk_size when doing normal
 copy operations.`,
 	Default:  false,
 	Advanced: true,
+}, {
+	Name:     config.ConfigEncoding,
+	Help:     config.ConfigEncodingHelp,
+	Advanced: true,
+	Default: (encoder.EncodeInvalidUtf8 |
+		encoder.EncodeSlash),
 }}
 
 // Register with Fs
@@ -105,7 +114,7 @@ func init() {
 				Value: "https://auth.storage.memset.com/v2.0",
 			}, {
 				Help:  "OVH",
-				Value: "https://auth.cloud.ovh.net/v2.0",
+				Value: "https://auth.cloud.ovh.net/v3",
 			}},
 		}, {
 			Name: "user_id",
@@ -183,26 +192,27 @@ provider.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth                     bool          `config:"env_auth"`
-	User                        string        `config:"user"`
-	Key                         string        `config:"key"`
-	Auth                        string        `config:"auth"`
-	UserID                      string        `config:"user_id"`
-	Domain                      string        `config:"domain"`
-	Tenant                      string        `config:"tenant"`
-	TenantID                    string        `config:"tenant_id"`
-	TenantDomain                string        `config:"tenant_domain"`
-	Region                      string        `config:"region"`
-	StorageURL                  string        `config:"storage_url"`
-	AuthToken                   string        `config:"auth_token"`
-	AuthVersion                 int           `config:"auth_version"`
-	ApplicationCredentialID     string        `config:"application_credential_id"`
-	ApplicationCredentialName   string        `config:"application_credential_name"`
-	ApplicationCredentialSecret string        `config:"application_credential_secret"`
-	StoragePolicy               string        `config:"storage_policy"`
-	EndpointType                string        `config:"endpoint_type"`
-	ChunkSize                   fs.SizeSuffix `config:"chunk_size"`
-	NoChunk                     bool          `config:"no_chunk"`
+	EnvAuth                     bool                 `config:"env_auth"`
+	User                        string               `config:"user"`
+	Key                         string               `config:"key"`
+	Auth                        string               `config:"auth"`
+	UserID                      string               `config:"user_id"`
+	Domain                      string               `config:"domain"`
+	Tenant                      string               `config:"tenant"`
+	TenantID                    string               `config:"tenant_id"`
+	TenantDomain                string               `config:"tenant_domain"`
+	Region                      string               `config:"region"`
+	StorageURL                  string               `config:"storage_url"`
+	AuthToken                   string               `config:"auth_token"`
+	AuthVersion                 int                  `config:"auth_version"`
+	ApplicationCredentialID     string               `config:"application_credential_id"`
+	ApplicationCredentialName   string               `config:"application_credential_name"`
+	ApplicationCredentialSecret string               `config:"application_credential_secret"`
+	StoragePolicy               string               `config:"storage_policy"`
+	EndpointType                string               `config:"endpoint_type"`
+	ChunkSize                   fs.SizeSuffix        `config:"chunk_size"`
+	NoChunk                     bool                 `config:"no_chunk"`
+	Enc                         encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote swift server
@@ -320,7 +330,8 @@ func parsePath(path string) (root string) {
 // split returns container and containerPath from the rootRelativePath
 // relative to f.root
 func (f *Fs) split(rootRelativePath string) (container, containerPath string) {
-	return bucket.Split(path.Join(f.root, rootRelativePath))
+	container, containerPath = bucket.Split(path.Join(f.root, rootRelativePath))
+	return f.opt.Enc.FromStandardName(container), f.opt.Enc.FromStandardPath(containerPath)
 }
 
 // split returns container and containerPath from the object
@@ -441,9 +452,10 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		// Check to see if the object exists - ignoring directory markers
 		var info swift.Object
 		var err error
+		encodedDirectory := f.opt.Enc.FromStandardPath(f.rootDirectory)
 		err = f.pacer.Call(func() (bool, error) {
 			var rxHeaders swift.Headers
-			info, rxHeaders, err = f.c.Object(f.rootContainer, f.rootDirectory)
+			info, rxHeaders, err = f.c.Object(f.rootContainer, encodedDirectory)
 			return shouldRetryHeaders(rxHeaders, err)
 		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
@@ -525,10 +537,10 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error
 //
 // Set recurse to read sub directories
 func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, fn listFn) error {
-	if prefix != "" {
+	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
-	if directory != "" {
+	if directory != "" && !strings.HasSuffix(directory, "/") {
 		directory += "/"
 	}
 	// Options for ObjectsWalk
@@ -553,17 +565,18 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
 				if !recurse {
 					isDirectory = strings.HasSuffix(object.Name, "/")
 				}
-				if !strings.HasPrefix(object.Name, prefix) {
-					fs.Logf(f, "Odd name received %q", object.Name)
+				remote := f.opt.Enc.ToStandardPath(object.Name)
+				if !strings.HasPrefix(remote, prefix) {
+					fs.Logf(f, "Odd name received %q", remote)
 					continue
 				}
-				if object.Name == prefix {
+				if remote == prefix {
 					// If we have zero length directory markers ending in / then swift
 					// will return them in the listing for the directory which causes
 					// duplicate directories.  Ignore them here.
 					continue
 				}
-				remote := object.Name[len(prefix):]
+				remote = remote[len(prefix):]
 				if addContainer {
 					remote = path.Join(container, remote)
 				}
@@ -635,7 +648,7 @@ func (f *Fs) listContainers(ctx context.Context) (entries fs.DirEntries, err err
 	}
 	for _, container := range containers {
 		f.cache.MarkOK(container.Name)
-		d := fs.NewDir(container.Name, time.Time{}).SetSize(container.Bytes).SetItems(container.Count)
+		d := fs.NewDir(f.opt.Enc.ToStandardName(container.Name), time.Time{}).SetSize(container.Bytes).SetItems(container.Count)
 		entries = append(entries, d)
 	}
 	return entries, nil
@@ -946,6 +959,18 @@ func (o *Object) isStaticLargeObject() (bool, error) {
 	return o.hasHeader("X-Static-Large-Object")
 }
 
+func (o *Object) isInContainerVersioning(container string) (bool, error) {
+	_, headers, err := o.fs.c.Container(container)
+	if err != nil {
+		return false, err
+	}
+	xHistoryLocation := headers["X-History-Location"]
+	if len(xHistoryLocation) > 0 {
+		return true, nil
+	}
+	return false, nil
+}
+
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
 	return o.size
@@ -1077,9 +1102,8 @@ func min(x, y int64) int64 {
 //
 // if except is passed in then segments with that prefix won't be deleted
 func (o *Object) removeSegments(except string) error {
-	container, containerPath := o.split()
-	segmentsContainer := container + "_segments"
-	err := o.fs.listContainerRoot(segmentsContainer, containerPath, "", false, true, func(remote string, object *swift.Object, isDirectory bool) error {
+	segmentsContainer, prefix, err := o.getSegmentsDlo()
+	err = o.fs.listContainerRoot(segmentsContainer, prefix, "", false, true, func(remote string, object *swift.Object, isDirectory bool) error {
 		if isDirectory {
 			return nil
 		}
@@ -1108,6 +1132,23 @@ func (o *Object) removeSegments(except string) error {
 	return nil
 }
 
+func (o *Object) getSegmentsDlo() (segmentsContainer string, prefix string, err error) {
+	if err = o.readMetaData(); err != nil {
+		return
+	}
+	dirManifest := o.headers["X-Object-Manifest"]
+	dirManifest, err = url.PathUnescape(dirManifest)
+	if err != nil {
+		return
+	}
+	delimiter := strings.Index(dirManifest, "/")
+	if len(dirManifest) == 0 || delimiter < 0 {
+		err = errors.New("Missing or wrong structure of manifest of Dynamic large object")
+		return
+	}
+	return dirManifest[:delimiter], dirManifest[delimiter+1:], nil
+}
+
 // urlEncode encodes a string so that it is a valid URL
 //
 // We don't use any of Go's standard methods as we need `/` not
@@ -1294,12 +1335,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 // Remove an object
-func (o *Object) Remove(ctx context.Context) error {
+func (o *Object) Remove(ctx context.Context) (err error) {
 	container, containerPath := o.split()
-	isDynamicLargeObject, err := o.isDynamicLargeObject()
-	if err != nil {
-		return err
-	}
+
 	// Remove file/manifest first
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.c.ObjectDelete(container, containerPath)
@@ -1308,12 +1346,22 @@ func (o *Object) Remove(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
+	isDynamicLargeObject, err := o.isDynamicLargeObject()
+	if err != nil {
+		return err
+	}
 	// ...then segments if required
 	if isDynamicLargeObject {
-		err = o.removeSegments("")
+		isInContainerVersioning, err := o.isInContainerVersioning(container)
 		if err != nil {
 			return err
 		}
+		if !isInContainerVersioning {
+			err = o.removeSegments("")
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }

backend/swift/swift_test.go

@@ -20,7 +20,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestSwift:",
+		RemoteName: "TestSwiftAIO:",
 		NilObject:  (*Object)(nil),
 	})
 }

backend/webdav/odrvcookie/fetch.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/xml"
+	"fmt"
 	"html/template"
 	"net/http"
 	"net/http/cookiejar"
@@ -32,13 +33,13 @@ type CookieResponse struct {
 	FedAuth http.Cookie
 }
 
-// SuccessResponse hold a response from the sharepoint webdav
-type SuccessResponse struct {
+// SharepointSuccessResponse holds a response from a successful microsoft login
+type SharepointSuccessResponse struct {
 	XMLName xml.Name            `xml:"Envelope"`
-	Succ    SuccessResponseBody `xml:"Body"`
+	Body    SuccessResponseBody `xml:"Body"`
 }
 
-// SuccessResponseBody is the body of a success response, it holds the token
+// SuccessResponseBody is the body of a successful response, it holds the token
 type SuccessResponseBody struct {
 	XMLName xml.Name
 	Type    string    `xml:"RequestSecurityTokenResponse>TokenType"`
@@ -47,6 +48,24 @@ type SuccessResponseBody struct {
 	Token   string    `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"`
 }
 
+// SharepointError holds a error response microsoft login
+type SharepointError struct {
+	XMLName xml.Name          `xml:"Envelope"`
+	Body    ErrorResponseBody `xml:"Body"`
+}
+
+func (e *SharepointError) Error() string {
+	return fmt.Sprintf("%s: %s (%s)", e.Body.FaultCode, e.Body.Reason, e.Body.Detail)
+}
+
+// ErrorResponseBody contains the body of a erroneous repsonse
+type ErrorResponseBody struct {
+	XMLName   xml.Name
+	FaultCode string `xml:"Fault>Code>Subcode>Value"`
+	Reason    string `xml:"Fault>Reason>Text"`
+	Detail    string `xml:"Fault>Detail>error>internalerror>text"`
+}
+
 // reqString is a template that gets populated with the user data in order to retrieve a "BinarySecurityToken"
 const reqString = `<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
 xmlns:a="http://www.w3.org/2005/08/addressing"
@@ -100,7 +119,7 @@ func (ca *CookieAuth) Cookies(ctx context.Context) (*CookieResponse, error) {
 	return ca.getSPCookie(tokenResp)
 }
 
-func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
+func (ca *CookieAuth) getSPCookie(conf *SharepointSuccessResponse) (*CookieResponse, error) {
 	spRoot, err := url.Parse(ca.endpoint)
 	if err != nil {
 		return nil, errors.Wrap(err, "Error while constructing endpoint URL")
@@ -123,7 +142,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
 	}
 
 	// Send the previously acquired Token as a Post parameter
-	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
+	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Body.Token)); err != nil {
 		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
 	}
 
@@ -141,7 +160,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
 	return &cookieResponse, nil
 }
 
-func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SuccessResponse, err error) {
+func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SharepointSuccessResponse, err error) {
 	reqData := map[string]interface{}{
 		"Username": ca.user,
 		"Password": ca.pass,
@@ -177,12 +196,21 @@ func (ca *CookieAuth) getSPToken(ctx context.Context) (conf *SuccessResponse, er
 	}
 	s := respBuf.Bytes()
 
-	conf = &SuccessResponse{}
+	conf = &SharepointSuccessResponse{}
 	err = xml.Unmarshal(s, conf)
-	if err != nil {
-		// FIXME: Try to parse with FailedResponse struct (check for server error code)
-		return nil, errors.Wrap(err, "Error while reading endpoint response")
+	if conf.Body.Token == "" {
+		// xml Unmarshal won't fail if the response doesn't contain a token
+		// However, the token will be empty
+		sErr := &SharepointError{}
+
+		errSErr := xml.Unmarshal(s, sErr)
+		if errSErr == nil {
+			return nil, sErr
+		}
 	}
 
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while reading endpoint response")
+	}
 	return
 }

backend/webdav/webdav.go

@@ -113,7 +113,8 @@ type Fs struct {
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
-	hasChecksums       bool          // set if can use owncloud style checksums
+	hasMD5             bool          // set if can use owncloud style checksums for MD5
+	hasSHA1            bool          // set if can use owncloud style checksums for SHA1
 }
 
 // Object describes a webdav object
@@ -215,7 +216,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, depth string)
 		},
 		NoRedirect: true,
 	}
-	if f.hasChecksums {
+	if f.hasMD5 || f.hasSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
 	}
 	var result api.Multistatus
@@ -357,6 +358,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	f.srv.SetHeader("Referer", u.String())
 
 	if root != "" && !rootIsDir {
 		// Check to see if the root actually an existing file
@@ -383,7 +385,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 // sets the BearerToken up
 func (f *Fs) setBearerToken(token string) {
 	f.opt.BearerToken = token
-	f.srv.SetHeader("Authorization", "BEARER "+token)
+	f.srv.SetHeader("Authorization", "Bearer "+token)
 }
 
 // fetch the bearer token using the command
@@ -430,11 +432,12 @@ func (f *Fs) setQuirks(ctx context.Context, vendor string) error {
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasChecksums = true
+		f.hasMD5 = true
+		f.hasSHA1 = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
-		f.hasChecksums = true
+		f.hasSHA1 = true
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -536,7 +539,7 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
 			"Depth": depth,
 		},
 	}
-	if f.hasChecksums {
+	if f.hasMD5 || f.hasSHA1 {
 		opts.Body = bytes.NewBuffer(owncloudProps)
 	}
 	var result api.Multistatus
@@ -835,7 +838,7 @@ func (f *Fs) copyOrMove(ctx context.Context, src fs.Object, remote string, metho
 		},
 	}
 	if f.useOCMtime {
-		opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime(ctx).UnixNano())/1e9)
+		opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
@@ -945,10 +948,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
-	if f.hasChecksums {
-		return hash.NewHashSet(hash.MD5, hash.SHA1)
+	hashes := hash.Set(hash.None)
+	if f.hasMD5 {
+		hashes.Add(hash.MD5)
 	}
-	return hash.Set(hash.None)
+	if f.hasSHA1 {
+		hashes.Add(hash.SHA1)
+	}
+	return hashes
 }
 
 // About gets quota information
@@ -1015,13 +1022,11 @@ func (o *Object) Remote() string {
 
 // Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if o.fs.hasChecksums {
-		switch t {
-		case hash.SHA1:
-			return o.sha1, nil
-		case hash.MD5:
-			return o.md5, nil
-		}
+	if t == hash.MD5 && o.fs.hasMD5 {
+		return o.md5, nil
+	}
+	if t == hash.SHA1 && o.fs.hasSHA1 {
+		return o.sha1, nil
 	}
 	return "", hash.ErrUnsupported
 }
@@ -1042,10 +1047,14 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
-	if o.fs.hasChecksums {
+	if o.fs.hasMD5 || o.fs.hasSHA1 {
 		hashes := info.Hashes()
-		o.sha1 = hashes[hash.SHA1]
-		o.md5 = hashes[hash.MD5]
+		if o.fs.hasSHA1 {
+			o.sha1 = hashes[hash.SHA1]
+		}
+		if o.fs.hasMD5 {
+			o.md5 = hashes[hash.MD5]
+		}
 	}
 	return nil
 }
@@ -1126,19 +1135,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
 		ContentType:   fs.MimeType(ctx, src),
 	}
-	if o.fs.useOCMtime || o.fs.hasChecksums {
+	if o.fs.useOCMtime || o.fs.hasMD5 || o.fs.hasSHA1 {
 		opts.ExtraHeaders = map[string]string{}
 		if o.fs.useOCMtime {
-			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime(ctx).UnixNano())/1e9)
+			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%d", src.ModTime(ctx).Unix())
 		}
-		if o.fs.hasChecksums {
-			// Set an upload checksum - prefer SHA1
-			//
-			// This is used as an upload integrity test. If we set
-			// only SHA1 here, owncloud will calculate the MD5 too.
+		// Set one upload checksum
+		// Owncloud uses one checksum only to check the upload and stores its own SHA1 and MD5
+		// Nextcloud stores the checksum you supply (SHA1 or MD5) but only stores one
+		if o.fs.hasSHA1 {
 			if sha1, _ := src.Hash(ctx, hash.SHA1); sha1 != "" {
 				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
-			} else if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
+			}
+		}
+		if o.fs.hasMD5 && opts.ExtraHeaders["OC-Checksum"] == "" {
+			if md5, _ := src.Hash(ctx, hash.MD5); md5 != "" {
 				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
 			}
 		}

backend/webdav/webdav_test.go

@@ -5,13 +5,36 @@ import (
 	"testing"
 
 	"github.com/rclone/rclone/backend/webdav"
+	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
 )
 
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestWebdav:",
+		RemoteName: "TestWebdavNexcloud:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration2(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavOwncloud:",
+		NilObject:  (*webdav.Object)(nil),
+	})
+}
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration3(t *testing.T) {
+	if *fstest.RemoteName != "" {
+		t.Skip("skipping as -remote is set")
+	}
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestWebdavRclone:",
 		NilObject:  (*webdav.Object)(nil),
 	})
 }

backend/yandex/yandex.go

@@ -22,6 +22,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -77,14 +78,23 @@ func init() {
 			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     config.ConfigEncoding,
+			Help:     config.ConfigEncodingHelp,
+			Advanced: true,
+			// Of the control characters \t \n \r are allowed
+			// it doesn't seem worth making an exception for this
+			Default: (encoder.Display |
+				encoder.EncodeInvalidUtf8),
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Token  string `config:"token"`
-	Unlink bool   `config:"unlink"`
+	Token  string               `config:"token"`
+	Unlink bool                 `config:"unlink"`
+	Enc    encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote yandex
@@ -207,7 +217,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set("path", path)
+	opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path))
 
 	if options.SortMode != nil {
 		opts.Parameters.Set("sort", options.SortMode.String())
@@ -234,6 +244,7 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string, options *api.
 		return nil, err
 	}
 
+	info.Name = f.opt.Enc.ToStandardName(info.Name)
 	return &info, nil
 }
 
@@ -360,6 +371,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		if info.ResourceType == "dir" {
 			//list all subdirs
 			for _, element := range info.Embedded.Items {
+				element.Name = f.opt.Enc.ToStandardName(element.Name)
 				remote := path.Join(dir, element.Name)
 				entry, err := f.itemToDirEntry(ctx, remote, &element)
 				if err != nil {
@@ -458,14 +470,18 @@ func (f *Fs) CreateDir(ctx context.Context, path string) (err error) {
 		NoResponse: true,
 	}
 
-	opts.Parameters.Set("path", path)
+	// If creating a directory with a : use (undocumented) disk: prefix
+	if strings.IndexRune(path, ':') >= 0 {
+		path = "disk:" + path
+	}
+	opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path))
 
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
-		//fmt.Printf("CreateDir Error: %s\n", err.Error())
+		// fmt.Printf("CreateDir %q Error: %s\n", path, err.Error())
 		return err
 	}
 	// fmt.Printf("...Id %q\n", *info.Id)
@@ -572,7 +588,7 @@ func (f *Fs) delete(ctx context.Context, path string, hardDelete bool) (err erro
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set("path", path)
+	opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(path))
 	opts.Parameters.Set("permanently", strconv.FormatBool(hardDelete))
 
 	var resp *http.Response
@@ -644,8 +660,8 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dst string, overwrite
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set("from", src)
-	opts.Parameters.Set("path", dst)
+	opts.Parameters.Set("from", f.opt.Enc.FromStandardPath(src))
+	opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(dst))
 	opts.Parameters.Set("overwrite", strconv.FormatBool(overwrite))
 
 	var resp *http.Response
@@ -794,12 +810,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 	}
 	opts := rest.Opts{
 		Method:     "PUT",
-		Path:       path,
+		Path:       f.opt.Enc.FromStandardPath(path),
 		Parameters: url.Values{},
 		NoResponse: true,
 	}
 
-	opts.Parameters.Set("path", f.filePath(remote))
+	opts.Parameters.Set("path", f.opt.Enc.FromStandardPath(f.filePath(remote)))
 
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -985,7 +1001,7 @@ func (o *Object) setCustomProperty(ctx context.Context, property string, value s
 		NoResponse: true,
 	}
 
-	opts.Parameters.Set("path", o.filePath())
+	opts.Parameters.Set("path", o.fs.opt.Enc.FromStandardPath(o.filePath()))
 	rcm := map[string]interface{}{
 		property: value,
 	}
@@ -1022,7 +1038,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set("path", o.filePath())
+	opts.Parameters.Set("path", o.fs.opt.Enc.FromStandardPath(o.filePath()))
 
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, nil, &dl)
@@ -1059,7 +1075,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, overwrite bool, mimeT
 		Parameters: url.Values{},
 	}
 
-	opts.Parameters.Set("path", o.filePath())
+	opts.Parameters.Set("path", o.fs.opt.Enc.FromStandardPath(o.filePath()))
 	opts.Parameters.Set("overwrite", strconv.FormatBool(overwrite))
 
 	err = o.fs.pacer.Call(func() (bool, error) {

bin/get-github-release.go

@@ -9,6 +9,7 @@ package main
 
 import (
 	"archive/tar"
+	"compress/bzip2"
 	"compress/gzip"
 	"encoding/json"
 	"flag"
@@ -349,6 +350,8 @@ func untar(srcFile, fileName, extractDir string) {
 			log.Fatalf("Couldn't open gzip: %v", err)
 		}
 		in = gzf
+	} else if srcExt == ".bz2" {
+		in = bzip2.NewReader(f)
 	}
 
 	tarReader := tar.NewReader(in)