fs: Debug the names of env vars we read values from as part of the config

See: https://forum.rclone.org/t/how-can-i-check-if-environment-variables-are-being-used/17490

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+github: [ncw]
+patreon: njcw
+liberapay: ncw
+custom: ["https://rclone.org/donate/"]

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Rclone Forum Community Support
+    url: https://forum.rclone.org/
+    about: Please ask and answer questions here.

.github/workflows/build.yml

@@ -197,8 +197,9 @@ jobs:
       - name: Deploy built binaries
         shell: bash
         run: |
-          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep ; fi
-          make travis_beta
+          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
+          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
+          make ci_beta
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
@@ -227,7 +228,7 @@ jobs:
       - name: Cross-compile rclone
         run: |
           docker pull billziss/xgo-cgofuse
-          go get -v github.com/karalabe/xgo
+          GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
           xgo \
               -image=billziss/xgo-cgofuse \
               -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
@@ -247,7 +248,7 @@ jobs:
 
       - name: Upload artifacts
         run: |
-          make circleci_upload
+          make ci_upload
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # Upload artifacts if not a PR && not a fork

.github/workflows/build_publish_docker_image.yml

@@ -0,0 +1,25 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                fetch-depth: 0
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              with:
+                tag: beta
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -0,0 +1,33 @@
+name: Docker release build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                fetch-depth: 0
+            - name: Get actual patch version
+              id: actual_patch_version
+              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
+            - name: Get actual minor version
+              id: actual_minor_version
+              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
+            - name: Get actual major version
+              id: actual_major_version
+              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              with:
+                tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

CONTRIBUTING.md

@@ -79,8 +79,15 @@ request](https://help.github.com/articles/creating-a-pull-request/).
 
 You patch will get reviewed and you might get asked to fix some stuff.
 
-If so, then make the changes in the same branch, squash the commits,
-rebase it to master then push it to GitHub with `--force`.
+If so, then make the changes in the same branch, squash the commits (make multiple commits one commit) by running:
+```
+git log # See how many commits you want to squash
+git reset --soft HEAD~2 # This squashes the 2 latest commits together.
+git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
+git commit # Add a new commit message.
+git push --force # Push the squashed commit to your GitHub repo.
+# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also reccommends wizardzines.com
+```
 
 ## CI for your fork ##
 
@@ -158,7 +165,7 @@ with modules beneath.
     * fserrors - rclone specific error handling
     * fshttp - http handling for rclone
     * fspath - path handling for rclone
-    * hash - defines rclones hash types and functions
+    * hash - defines rclone's hash types and functions
     * list - list a remote
     * log - logging facilities
     * march - iterates directories in lock step
@@ -295,7 +302,7 @@ If you need to update a dependency then run
     GO111MODULE=on go get -u github.com/pkg/errors
     GO111MODULE=on go mod vendor
 
-Check in in a single commit as above.
+Check in a single commit as above.
 
 ## Updating all the dependencies ##
 
@@ -383,7 +390,7 @@ alphabetical order of full name of remote (eg `drive` is ordered as
     * update them with `make backenddocs` - revert any changes in other backends
   * `docs/content/overview.md` - overview docs
   * `docs/content/docs.md` - list of remotes in config section
-  * `docs/content/about.md` - front page of rclone.org
+  * `docs/content/_index.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
 

Dockerfile

@@ -3,16 +3,16 @@ FROM golang AS builder
 COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN make quicktest
 RUN \
-  CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+  CGO_ENABLED=0 \
   make
 RUN ./rclone version
 
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse
+RUN apk --no-cache add ca-certificates fuse tzdata && \
+  echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 

MAINTAINERS.md

@@ -2,17 +2,20 @@
 
 Current active maintainers of rclone are:
 
-| Name             | GitHub ID   | Specific Responsibilities    |
-| :--------------- | :---------- | :--------------------------  |
-| Nick Craig-Wood  | @ncw        | overall project health       |
-| Stefan Breunig   | @breunigs   |                              |
-| Ishuah Kariuki   | @ishuah     |                              |
-| Remus Bunduc     | @remusb     | cache backend                |
-| Fabian Möller    | @B4dM4n     |                              |
-| Alex Chen        | @Cnly       | onedrive backend             |
-| Sandeep Ummadi   | @sandeepkru | azureblob backend            |
-| Sebastian Bünger | @buengese   | jottacloud & yandex backends |
-| Ivan Andreev     | @ivandeex   | chunker & mailru backends    |
+| Name             | GitHub ID         | Specific Responsibilities    |
+| :--------------- | :---------------- | :--------------------------  |
+| Nick Craig-Wood  | @ncw              | overall project health       |
+| Stefan Breunig   | @breunigs         |                              |
+| Ishuah Kariuki   | @ishuah           |                              |
+| Remus Bunduc     | @remusb           | cache backend                |
+| Fabian Möller    | @B4dM4n           |                              |
+| Alex Chen        | @Cnly             | onedrive backend             |
+| Sandeep Ummadi   | @sandeepkru       | azureblob backend            |
+| Sebastian Bünger | @buengese         | jottacloud & yandex backends |
+| Ivan Andreev     | @ivandeex         | chunker & mailru backends    |
+| Max Sum          | @Max-Sum          | union backend                |
+| Fred             | @creativeprojects | seafile backend              |
+| Caleb Case       | @calebcase        | tardigrade backend           |
 
 **This is a work in progress Draft**
 
@@ -30,7 +33,7 @@ Rclone uses the labels like this:
 * `duplicate` - normally close these and ask the user to subscribe to the original
 * `enhancement: new remote` - a new rclone backend
 * `enhancement` - a new feature
-* `FUSE` - do do with `rclone mount` command
+* `FUSE` - to do with `rclone mount` command
 * `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project
 * `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project
 * `IMPORTANT` - note to maintainers not to forget to fix this for the release

Makefile

@@ -1,6 +1,6 @@
 SHELL = bash
 # Branch we are working on
-BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
+BRANCH := $(or $(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
 # Tag of the current commit, if any.  If this is not "" then we are building a release
 RELEASE_TAG := $(shell git tag -l --points-at HEAD)
 # Version of last release (may not be on this branch)
@@ -61,6 +61,10 @@ vars:
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
+btest:
+	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
+	@echo "Copied markdown of beta release to clip board"
+
 version:
 	@echo '$(TAG)'
 
@@ -86,11 +90,15 @@ check:	rclone
 build_dep:
 	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
-# Get the release dependencies
-release_dep:
+# Get the release dependencies we only install on linux
+release_dep_linux:
 	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
 	go run bin/get-github-release.go -extract github-release aktau/github-release 'linux-amd64-github-release.tar.bz2'
 
+# Get the release dependencies we only install on Windows
+release_dep_windows:
+	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
+
 # Update dependencies
 update:
 	GO111MODULE=on go get -u ./...
@@ -105,16 +113,16 @@ tidy:
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
-	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
+	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
 MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
+	pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
 
 MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
+	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
@@ -136,11 +144,19 @@ clean:
 	rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log
 
 website:
+	rm -rf docs/public
 	cd docs && hugo
+	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
 	rclone -v sync docs/public memstore:www-rclone-org
 
+upload_test_website:	website
+	rclone -P sync docs/public test-rclone-org:
+
+validate_website: website
+	find docs/public -type f -name "*.html" | xargs tidy --mute-id yes -errors --gnu-emacs yes --drop-empty-elements no --warn-proprietary-attributes no --mute MISMATCHED_ATTRIBUTE_WARN
+
 tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
@@ -175,14 +191,7 @@ log_since_last_release:
 compile_all:
 	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
 
-appveyor_upload:
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
-ifndef BRANCH_PATH
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
-endif
-	@echo Beta release ready at $(BETA_URL)
-
-circleci_upload:
+ci_upload:
 	sudo chown -R $$USER build
 	find build -type l -delete
 	gzip -r9v build
@@ -192,10 +201,7 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
-travis_beta:
-ifeq (linux,$(filter linux,$(subst Linux,linux,$(TRAVIS_OS_NAME) $(AGENT_OS))))
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
-endif
+ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -204,12 +210,12 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-# Fetch the binary builds from travis and appveyor
+# Fetch the binary builds from GitHub actions
 fetch_binaries:
 	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server -v -w
+	cd docs && hugo server -v -w --disableFastRender
 
 tag:	doc
 	@echo "Old tag is $(VERSION)"

README.md

@@ -59,9 +59,11 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
   * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
+  * Seafile [:page_facing_up:](https://rclone.org/seafile/)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
+  * Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)

RELEASE.md

@@ -31,7 +31,7 @@ This file describes how to make the various kinds of releases
   * make upload_website
   * make upload_github
   * make startdev
-  * # announce with forum post, twitter post, G+ post
+  * # announce with forum post, twitter post, patreon post
 
 Early in the next release cycle update the vendored dependencies
 
@@ -63,14 +63,14 @@ If rclone needs a point release due to some horrendous bug:
 First make the release branch.  If this is a second point release then
 this will be done already.
 
-  * BASE_TAG=v1.XX          # eg v1.49
-  * NEW_TAG=${BASE_TAG}.Y   # eg v1.49.1
-  * echo $BASE_TAG $NEW_TAG # v1.49 v1.49.1
-  * git branch ${BASE_TAG} ${BASE_TAG}-fixes
+  * BASE_TAG=v1.XX          # eg v1.52
+  * NEW_TAG=${BASE_TAG}.Y   # eg v1.52.1
+  * echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
+  * git branch ${BASE_TAG} ${BASE_TAG}-stable
 
 Now
 
-  * git co ${BASE_TAG}-fixes
+  * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
   * Test (see above)
   * make NEXT_VERSION=${NEW_TAG} tag
@@ -79,7 +79,7 @@ Now
   * git commit -a -v -m "Version ${NEW_TAG}"
   * git tag -d ${NEW_TAG}
   * git tag -s -m "Version ${NEW_TAG}" ${NEW_TAG}
-  * git push --tags -u origin ${BASE_TAG}-fixes
+  * git push --tags -u origin ${BASE_TAG}-stable
   * Wait for builds to complete
   * make BRANCH_PATH= TAG=${NEW_TAG} fetch_binaries
   * make TAG=${NEW_TAG} tarball
@@ -92,20 +92,21 @@ Now
   * git co master
   * make VERSION=${NEW_TAG} startdev
   * # cherry pick the changes to the changelog and VERSION
-  * git checkout ${BASE_TAG}-fixes VERSION docs/content/changelog.md
+  * git checkout ${BASE_TAG}-stable VERSION docs/content/changelog.md
   * git commit --amend
   * git push
   * Announce!
 
 ## Making a manual build of docker
 
-The rclone docker image should autobuild on docker hub.  If it doesn't
+The rclone docker image should autobuild on via GitHub actions.  If it doesn't
 or needs to be updated then rebuild like this.
 
 ```
-docker build -t rclone/rclone:1.49.1 -t rclone/rclone:1.49 -t rclone/rclone:1 -t rclone/rclone:latest .
-docker push rclone/rclone:1.49.1
-docker push rclone/rclone:1.49
+docker pull golang
+docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
+docker push rclone/rclone:1.52.0
+docker push rclone/rclone:1.52
 docker push rclone/rclone:1
 docker push rclone/rclone:latest
 ```

VERSION

@@ -1 +1 @@
-v1.51.0
+v1.52.2

backend/all/all.go

@@ -31,10 +31,12 @@ import (
 	_ "github.com/rclone/rclone/backend/putio"
 	_ "github.com/rclone/rclone/backend/qingstor"
 	_ "github.com/rclone/rclone/backend/s3"
+	_ "github.com/rclone/rclone/backend/seafile"
 	_ "github.com/rclone/rclone/backend/sftp"
 	_ "github.com/rclone/rclone/backend/sharefile"
 	_ "github.com/rclone/rclone/backend/sugarsync"
 	_ "github.com/rclone/rclone/backend/swift"
+	_ "github.com/rclone/rclone/backend/tardigrade"
 	_ "github.com/rclone/rclone/backend/union"
 	_ "github.com/rclone/rclone/backend/webdav"
 	_ "github.com/rclone/rclone/backend/yandex"

backend/amazonclouddrive/amazonclouddrive.go

@@ -71,7 +71,7 @@ func init() {
 		Description: "Amazon Drive",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("amazon cloud drive", name, m, acdConfig)
+			err := oauthutil.Config("amazon cloud drive", name, m, acdConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -169,7 +169,7 @@ type Fs struct {
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
-// Object describes a acd object
+// Object describes an acd object
 //
 // Will definitely have info but maybe not meta
 type Object struct {
@@ -229,7 +229,7 @@ func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 		}
 		// Work around receiving this error sporadically on authentication
 		//
-		// HTTP code 403: "403 Forbidden", reponse body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
+		// HTTP code 403: "403 Forbidden", response body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
 		if resp.StatusCode == 403 && strings.Contains(err.Error(), "Authorization header requires") {
 			fs.Debugf(f, "403 \"Authorization header requires...\" error received - retry")
 			return true, err
@@ -514,10 +514,6 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -665,7 +661,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return nil, err
 	}
 	// If not create it
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -696,13 +692,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -724,10 +714,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// create the destination directory if necessary
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return nil, err
-	}
 	srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
 	if err != nil {
 		return nil, err
@@ -797,54 +783,24 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return errors.New("can't move root directory")
 	}
 
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
 	// Find ID of dst parent, creating subdirs if necessary
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, findPath, true)
+	dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, dstRemote, true)
 	if err != nil {
 		return err
 	}
 
 	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
+	_, err = f.dirCache.FindDir(ctx, dstRemote, false)
+	if err == fs.ErrorDirNotFound {
+		// OK
+	} else if err != nil {
+		return err
+	} else {
+		return fs.ErrorDirExists
 	}
 
 	// Find ID of src parent
-	findPath = srcRemote
-	var srcDirectoryID string
-	if srcRemote == "" {
-		srcDirectoryID, err = srcFs.dirCache.RootParentID()
-	} else {
-		_, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
-	}
+	_, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcRemote, false)
 	if err != nil {
 		return err
 	}
@@ -890,10 +846,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1037,7 +989,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.info != nil {
 		return nil
 	}
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return fs.ErrorObjectNotFound

backend/azureblob/azureblob.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"crypto/md5"
 	"encoding/base64"
-	"encoding/binary"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -35,6 +34,9 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
+	"github.com/rclone/rclone/lib/readers"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -59,6 +61,8 @@ const (
 	emulatorAccount      = "devstoreaccount1"
 	emulatorAccountKey   = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
 	emulatorBlobEndpoint = "http://127.0.0.1:10000/devstoreaccount1"
+	memoryPoolFlushTime  = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap    = false
 )
 
 // Register with Fs
@@ -125,6 +129,28 @@ If blobs are in "archive tier" at remote, trying to perform data transfer
 operations from remote will not be allowed. User should first restore by
 tiering blob to "Hot" or "Cool".`,
 			Advanced: true,
+		}, {
+			Name: "disable_checksum",
+			Help: `Don't store MD5 checksum with object metadata.
+
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.`,
+			Default:  false,
+			Advanced: true,
+		}, {
+			Name:     "memory_pool_flush_time",
+			Default:  memoryPoolFlushTime,
+			Advanced: true,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
+		}, {
+			Name:     "memory_pool_use_mmap",
+			Default:  memoryPoolUseMmap,
+			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -141,16 +167,19 @@ tiering blob to "Hot" or "Cool".`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account       string               `config:"account"`
-	Key           string               `config:"key"`
-	Endpoint      string               `config:"endpoint"`
-	SASURL        string               `config:"sas_url"`
-	UploadCutoff  fs.SizeSuffix        `config:"upload_cutoff"`
-	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
-	ListChunkSize uint                 `config:"list_chunk"`
-	AccessTier    string               `config:"access_tier"`
-	UseEmulator   bool                 `config:"use_emulator"`
-	Enc           encoder.MultiEncoder `config:"encoding"`
+	Account             string               `config:"account"`
+	Key                 string               `config:"key"`
+	Endpoint            string               `config:"endpoint"`
+	SASURL              string               `config:"sas_url"`
+	UploadCutoff        fs.SizeSuffix        `config:"upload_cutoff"`
+	ChunkSize           fs.SizeSuffix        `config:"chunk_size"`
+	ListChunkSize       uint                 `config:"list_chunk"`
+	AccessTier          string               `config:"access_tier"`
+	UseEmulator         bool                 `config:"use_emulator"`
+	DisableCheckSum     bool                 `config:"disable_checksum"`
+	MemoryPoolFlushTime fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap   bool                 `config:"memory_pool_use_mmap"`
+	Enc                 encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote azure server
@@ -169,9 +198,10 @@ type Fs struct {
 	cache         *bucket.Cache                   // cache for container creation status
 	pacer         *fs.Pacer                       // To pace and retry the API calls
 	uploadToken   *pacer.TokenDispenser           // control concurrency
+	pool          *pool.Pool                      // memory pool
 }
 
-// Object describes a azure object
+// Object describes an azure object
 type Object struct {
 	fs         *Fs                   // what this object is part of
 	remote     string                // The remote path
@@ -257,6 +287,12 @@ var retryErrorCodes = []int{
 func (f *Fs) shouldRetry(err error) (bool, error) {
 	// FIXME interpret special errors - more to do here
 	if storageErr, ok := err.(azblob.StorageError); ok {
+		switch storageErr.ServiceCode() {
+		case "InvalidBlobOrBlock":
+			// These errors happen sometimes in multipart uploads
+			// because of block concurrency issues
+			return true, err
+		}
 		statusCode := storageErr.Response().StatusCode
 		for _, e := range retryErrorCodes {
 			if statusCode == e {
@@ -302,7 +338,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 }
 
 // httpClientFactory creates a Factory object that sends HTTP requests
-// to a rclone's http.Client.
+// to an rclone's http.Client.
 //
 // copied from azblob.newDefaultHTTPClientFactory
 func httpClientFactory(client *http.Client) pipeline.Factory {
@@ -382,6 +418,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		client:      fshttp.NewClient(fs.Config),
 		cache:       bucket.NewCache(),
 		cntURLcache: make(map[string]*azblob.ContainerURL, 1),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			fs.Config.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -816,6 +858,11 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	return fs, fs.Update(ctx, in, src, options...)
 }
 
+// PutStream uploads to the remote path with the modTime given of indeterminate size
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+	return f.Put(ctx, in, src, options...)
+}
+
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	container, _ := f.split(dir)
@@ -825,6 +872,10 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 // makeContainer creates the container if it doesn't exist
 func (f *Fs) makeContainer(ctx context.Context, container string) error {
 	return f.cache.Create(container, func() error {
+		// If this is a SAS URL limited to a container then assume it is already created
+		if f.isLimited {
+			return nil
+		}
 		// now try to create the container
 		return f.pacer.Call(func() (bool, error) {
 			_, err := f.cntURL(container).Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone)
@@ -978,6 +1029,19 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.NewObject(ctx, remote)
 }
 
+func (f *Fs) getMemoryPool(size int64) *pool.Pool {
+	if size == int64(f.opt.ChunkSize) {
+		return f.pool
+	}
+
+	return pool.New(
+		time.Duration(f.opt.MemoryPoolFlushTime),
+		int(size),
+		fs.Config.Transfers,
+		f.opt.MemoryPoolUseMmap,
+	)
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1221,103 +1285,116 @@ type readSeeker struct {
 	io.Seeker
 }
 
+// increment the slice passed in as LSB binary
+func increment(xs []byte) {
+	for i, digit := range xs {
+		newDigit := digit + 1
+		xs[i] = newDigit
+		if newDigit >= digit {
+			// exit if no carry
+			break
+		}
+	}
+}
+
+var warnStreamUpload sync.Once
+
 // uploadMultipart uploads a file using multipart upload
 //
 // Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.
-func (o *Object) uploadMultipart(in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) {
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, blob *azblob.BlobURL, httpHeaders *azblob.BlobHTTPHeaders) (err error) {
 	// Calculate correct chunkSize
 	chunkSize := int64(o.fs.opt.ChunkSize)
-	var totalParts int64
-	for {
-		// Calculate number of parts
-		var remainder int64
-		totalParts, remainder = size/chunkSize, size%chunkSize
-		if remainder != 0 {
-			totalParts++
+	totalParts := -1
+
+	// Note that the max size of file is 4.75 TB (100 MB X 50,000
+	// blocks) and this is bigger than the max uncommitted block
+	// size (9.52 TB) so we do not need to part commit block lists
+	// or garbage collect uncommitted blocks.
+	//
+	// See: https://docs.microsoft.com/en-gb/rest/api/storageservices/put-block
+
+	// size can be -1 here meaning we don't know the size of the incoming file.  We use ChunkSize
+	// buffers here (default 4MB). With a maximum number of parts (50,000) this will be a file of
+	// 195GB which seems like a not too unreasonable limit.
+	if size == -1 {
+		warnStreamUpload.Do(func() {
+			fs.Logf(o, "Streaming uploads using chunk size %v will have maximum file size of %v",
+				o.fs.opt.ChunkSize, fs.SizeSuffix(chunkSize*maxTotalParts))
+		})
+	} else {
+		// Adjust partSize until the number of parts is small enough.
+		if size/chunkSize >= maxTotalParts {
+			// Calculate partition size rounded up to the nearest MB
+			chunkSize = (((size / maxTotalParts) >> 20) + 1) << 20
 		}
-		if totalParts < maxTotalParts {
-			break
-		}
-		// Double chunk size if the number of parts is too big
-		chunkSize *= 2
 		if chunkSize > int64(maxChunkSize) {
 			return errors.Errorf("can't upload as it is too big %v - takes more than %d chunks of %v", fs.SizeSuffix(size), totalParts, fs.SizeSuffix(chunkSize/2))
 		}
+		totalParts = int(size / chunkSize)
+		if size%chunkSize != 0 {
+			totalParts++
+		}
 	}
+
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", totalParts, fs.SizeSuffix(chunkSize))
 
-	// https://godoc.org/github.com/Azure/azure-storage-blob-go/2017-07-29/azblob#example-BlockBlobURL
-	// Utilities are cloned from above example
-	// These helper functions convert a binary block ID to a base-64 string and vice versa
-	// NOTE: The blockID must be <= 64 bytes and ALL blockIDs for the block must be the same length
-	blockIDBinaryToBase64 := func(blockID []byte) string { return base64.StdEncoding.EncodeToString(blockID) }
-	// These helper functions convert an int block ID to a base-64 string and vice versa
-	blockIDIntToBase64 := func(blockID uint64) string {
-		binaryBlockID := (&[8]byte{})[:] // All block IDs are 8 bytes long
-		binary.LittleEndian.PutUint64(binaryBlockID, blockID)
-		return blockIDBinaryToBase64(binaryBlockID)
-	}
-
-	// block ID variables
-	var (
-		rawID   uint64
-		blockID = "" // id in base64 encoded form
-		blocks  []string
-	)
-
-	// increment the blockID
-	nextID := func() {
-		rawID++
-		blockID = blockIDIntToBase64(rawID)
-		blocks = append(blocks, blockID)
-	}
-
-	// Get BlockBlobURL, we will use default pipeline here
-	blockBlobURL := blob.ToBlockBlobURL()
-	ctx := context.Background()
-	ac := azblob.LeaseAccessConditions{} // Use default lease access conditions
-
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
 	in, wrap := accounting.UnWrap(in)
 
 	// Upload the chunks
-	remaining := size
-	position := int64(0)
-	errs := make(chan error, 1)
-	var wg sync.WaitGroup
-outer:
-	for part := 0; part < int(totalParts); part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+	var (
+		g, gCtx       = errgroup.WithContext(ctx)
+		remaining     = size                           // remaining size in file for logging only, -1 if size < 0
+		position      = int64(0)                       // position in file
+		memPool       = o.fs.getMemoryPool(chunkSize)  // pool to get memory from
+		finished      = false                          // set when we have read EOF
+		blocks        []string                         // list of blocks for finalize
+		blockBlobURL  = blob.ToBlockBlobURL()          // Get BlockBlobURL, we will use default pipeline here
+		ac            = azblob.LeaseAccessConditions{} // Use default lease access conditions
+		binaryBlockID = make([]byte, 8)                // block counter as LSB first 8 bytes
+	)
+	for part := 0; !finished; part++ {
+		// Get a block of memory from the pool and a token which limits concurrency
+		o.fs.uploadToken.Get()
+		buf := memPool.Get()
+
+		free := func() {
+			memPool.Put(buf)       // return the buf
+			o.fs.uploadToken.Put() // return the token
 		}
 
-		reqSize := remaining
-		if reqSize >= chunkSize {
-			reqSize = chunkSize
+		// Fail fast, in case an errgroup managed function returns an error
+		// gCtx is cancelled. There is no point in uploading all the other parts.
+		if gCtx.Err() != nil {
+			free()
+			break
 		}
 
-		// Make a block of memory
-		buf := make([]byte, reqSize)
-
 		// Read the chunk
-		_, err = io.ReadFull(in, buf)
-		if err != nil {
-			err = errors.Wrap(err, "multipart upload failed to read source")
-			break outer
+		n, err := readers.ReadFill(in, buf) // this can never return 0, nil
+		if err == io.EOF {
+			if n == 0 { // end if no data
+				free()
+				break
+			}
+			finished = true
+		} else if err != nil {
+			free()
+			return errors.Wrap(err, "multipart upload failed to read source")
 		}
+		buf = buf[:n]
+
+		// increment the blockID and save the blocks for finalize
+		increment(binaryBlockID)
+		blockID := base64.StdEncoding.EncodeToString(binaryBlockID)
+		blocks = append(blocks, blockID)
 
 		// Transfer the chunk
-		nextID()
-		wg.Add(1)
-		o.fs.uploadToken.Get()
-		go func(part int, position int64, blockID string) {
-			defer wg.Done()
-			defer o.fs.uploadToken.Put()
-			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
+		fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, totalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
+		g.Go(func() (err error) {
+			defer free()
 
 			// Upload the block, with MD5 for check
 			md5sum := md5.Sum(buf)
@@ -1329,28 +1406,19 @@ outer:
 				_, err = blockBlobURL.StageBlock(ctx, blockID, &rs, ac, transactionalMD5)
 				return o.fs.shouldRetry(err)
 			})
-
 			if err != nil {
-				err = errors.Wrap(err, "multipart upload failed to upload part")
-				select {
-				case errs <- err:
-				default:
-				}
-				return
+				return errors.Wrap(err, "multipart upload failed to upload part")
 			}
-		}(part, position, blockID)
+			return nil
+		})
 
 		// ready for next block
-		remaining -= chunkSize
+		if size >= 0 {
+			remaining -= chunkSize
+		}
 		position += chunkSize
 	}
-	wg.Wait()
-	if err == nil {
-		select {
-		case err = <-errs:
-		default:
-		}
-	}
+	err = g.Wait()
 	if err != nil {
 		return err
 	}
@@ -1387,14 +1455,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	httpHeaders.ContentType = fs.MimeType(ctx, o)
 	// Compute the Content-MD5 of the file, for multiparts uploads it
 	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
+	// Note: If multipart, an MD5 checksum will also be computed for each uploaded block
 	// 		 in order to validate its integrity during transport
-	if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
-		sourceMD5bytes, err := hex.DecodeString(sourceMD5)
-		if err == nil {
-			httpHeaders.ContentMD5 = sourceMD5bytes
-		} else {
-			fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+	if !o.fs.opt.DisableCheckSum {
+		if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {
+			sourceMD5bytes, err := hex.DecodeString(sourceMD5)
+			if err == nil {
+				httpHeaders.ContentMD5 = sourceMD5bytes
+			} else {
+				fs.Debugf(o, "Failed to decode %q as MD5: %v", sourceMD5, err)
+			}
 		}
 	}
 
@@ -1408,7 +1478,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	// is merged the SDK can't upload a single blob of exactly the chunk
 	// size, so upload with a multpart upload to work around.
 	// See: https://github.com/rclone/rclone/issues/2653
-	multipartUpload := size >= int64(o.fs.opt.UploadCutoff)
+	multipartUpload := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
 	if size == int64(o.fs.opt.ChunkSize) {
 		multipartUpload = true
 		fs.Debugf(o, "Setting multipart upload for file of chunk size (%d) to work around SDK bug", size)
@@ -1418,7 +1488,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		if multipartUpload {
 			// If a large file upload in chunks
-			err = o.uploadMultipart(in, size, &blob, &httpHeaders)
+			err = o.uploadMultipart(ctx, in, size, &blob, &httpHeaders)
 		} else {
 			// Write a small blob in one transaction
 			blockBlobURL := blob.ToBlockBlobURL()
@@ -1503,12 +1573,13 @@ func (o *Object) GetTier() string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs        = &Fs{}
-	_ fs.Copier    = &Fs{}
-	_ fs.Purger    = &Fs{}
-	_ fs.ListRer   = &Fs{}
-	_ fs.Object    = &Object{}
-	_ fs.MimeTyper = &Object{}
-	_ fs.GetTierer = &Object{}
-	_ fs.SetTierer = &Object{}
+	_ fs.Fs          = &Fs{}
+	_ fs.Copier      = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+	_ fs.Purger      = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.Object      = &Object{}
+	_ fs.MimeTyper   = &Object{}
+	_ fs.GetTierer   = &Object{}
+	_ fs.SetTierer   = &Object{}
 )

backend/azureblob/azureblob_internal_test.go

@@ -16,3 +16,20 @@ func (f *Fs) InternalTest(t *testing.T) {
 	enabled = f.Features().GetTier
 	assert.True(t, enabled)
 }
+
+func TestIncrement(t *testing.T) {
+	for _, test := range []struct {
+		in   []byte
+		want []byte
+	}{
+		{[]byte{0, 0, 0, 0}, []byte{1, 0, 0, 0}},
+		{[]byte{0xFE, 0, 0, 0}, []byte{0xFF, 0, 0, 0}},
+		{[]byte{0xFF, 0, 0, 0}, []byte{0, 1, 0, 0}},
+		{[]byte{0, 1, 0, 0}, []byte{1, 1, 0, 0}},
+		{[]byte{0xFF, 0xFF, 0xFF, 0xFE}, []byte{0, 0, 0, 0xFF}},
+		{[]byte{0xFF, 0xFF, 0xFF, 0xFF}, []byte{0, 0, 0, 0}},
+	} {
+		increment(test.in)
+		assert.Equal(t, test.want, test.in)
+	}
+}

backend/b2/api/types.go

@@ -337,3 +337,11 @@ type CopyFileRequest struct {
 	Info              map[string]string `json:"fileInfo,omitempty"`            // This field stores the metadata that will be stored with the file. (REPLACE only)
 	DestBucketID      string            `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
 }
+
+// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
+type CopyPartRequest struct {
+	SourceID    string `json:"sourceFileId"`    // The ID of the source file being copied.
+	LargeFileID string `json:"largeFileId"`     // The ID of the large file the part will belong to, as returned by b2_start_large_file.
+	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
+	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
+}

backend/b2/b2.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -54,6 +55,9 @@ const (
 	minChunkSize        = 5 * fs.MebiByte
 	defaultChunkSize    = 96 * fs.MebiByte
 	defaultUploadCutoff = 200 * fs.MebiByte
+	largeFileCopyCutoff = 4 * fs.GibiByte          // 5E9 is the max
+	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap   = false
 )
 
 // Globals
@@ -113,6 +117,16 @@ Files above this size will be uploaded in chunks of "--b2-chunk-size".
 This value should be set no larger than 4.657GiB (== 5GB).`,
 			Default:  defaultUploadCutoff,
 			Advanced: true,
+		}, {
+			Name: "copy_cutoff",
+			Help: `Cutoff for switching to multipart copy
+
+Any files larger than this that need to be server side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 4.6GB.`,
+			Default:  largeFileCopyCutoff,
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Upload chunk size. Must fit in memory.
@@ -124,8 +138,13 @@ minimum size.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
-			Name:     "disable_checksum",
-			Help:     `Disable checksums for large (> upload cutoff) files`,
+			Name: "disable_checksum",
+			Help: `Disable checksums for large (> upload cutoff) files
+
+Normally rclone will calculate the SHA1 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -145,6 +164,18 @@ The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
+		}, {
+			Name:     "memory_pool_flush_time",
+			Default:  memoryPoolFlushTime,
+			Advanced: true,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
+		}, {
+			Name:     "memory_pool_use_mmap",
+			Default:  memoryPoolUseMmap,
+			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -168,10 +199,13 @@ type Options struct {
 	Versions                      bool                 `config:"versions"`
 	HardDelete                    bool                 `config:"hard_delete"`
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
+	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
 	ChunkSize                     fs.SizeSuffix        `config:"chunk_size"`
 	DisableCheckSum               bool                 `config:"disable_checksum"`
 	DownloadURL                   string               `config:"download_url"`
 	DownloadAuthorizationDuration fs.Duration          `config:"download_auth_duration"`
+	MemoryPoolFlushTime           fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap             bool                 `config:"memory_pool_use_mmap"`
 	Enc                           encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -194,7 +228,8 @@ type Fs struct {
 	uploads         map[string][]*api.GetUploadURLResponse // Upload URLs by buckedID
 	authMu          sync.Mutex                             // lock for authorizing the account
 	pacer           *fs.Pacer                              // To pace and retry the API calls
-	bufferTokens    chan []byte                            // control concurrency of multipart uploads
+	uploadToken     *pacer.TokenDispenser                  // control concurrency
+	pool            *pool.Pool                             // memory pool
 }
 
 // Object describes a b2 object
@@ -330,7 +365,6 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	err = checkUploadChunkSize(cs)
 	if err == nil {
 		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-		f.fillBufferTokens() // reset the buffer tokens
 	}
 	return
 }
@@ -391,6 +425,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		_bucketType: make(map[string]string, 1),
 		uploads:     make(map[string][]*api.GetUploadURLResponse),
 		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			fs.Config.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -405,7 +446,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		f.srv.SetHeader(testModeHeader, testMode)
 		fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
 	}
-	f.fillBufferTokens()
 	err = f.authorizeAccount(ctx)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to authorize account")
@@ -528,32 +568,25 @@ func (f *Fs) clearUploadURL(bucketID string) {
 	f.uploadMu.Unlock()
 }
 
-// Fill up (or reset) the buffer tokens
-func (f *Fs) fillBufferTokens() {
-	f.bufferTokens = make(chan []byte, fs.Config.Transfers)
-	for i := 0; i < fs.Config.Transfers; i++ {
-		f.bufferTokens <- nil
+// getBuf gets a buffer of f.opt.ChunkSize and an upload token
+//
+// If noBuf is set then it just gets an upload token
+func (f *Fs) getBuf(noBuf bool) (buf []byte) {
+	f.uploadToken.Get()
+	if !noBuf {
+		buf = f.pool.Get()
 	}
-}
-
-// getUploadBlock gets a block from the pool of size chunkSize
-func (f *Fs) getUploadBlock() []byte {
-	buf := <-f.bufferTokens
-	if buf == nil {
-		buf = make([]byte, f.opt.ChunkSize)
-	}
-	// fs.Debugf(f, "Getting upload block %p", buf)
 	return buf
 }
 
-// putUploadBlock returns a block to the pool of size chunkSize
-func (f *Fs) putUploadBlock(buf []byte) {
-	buf = buf[:cap(buf)]
-	if len(buf) != int(f.opt.ChunkSize) {
-		panic("bad blocksize returned to pool")
+// putBuf returns a buffer to the memory pool and an upload token
+//
+// If noBuf is set then it just returns the upload token
+func (f *Fs) putBuf(buf []byte, noBuf bool) {
+	if !noBuf {
+		f.pool.Put(buf)
 	}
-	// fs.Debugf(f, "Returning upload block %p", buf)
-	f.bufferTokens <- buf
+	f.uploadToken.Put()
 }
 
 // Return an Object from a path
@@ -668,7 +701,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 			}
 			remote := file.Name[len(prefix):]
 			// Check for directory
-			isDirectory := strings.HasSuffix(remote, "/")
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if isDirectory {
 				remote = remote[:len(remote)-1]
 			}
@@ -1200,6 +1233,63 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return f.purge(ctx, f.rootBucket, f.rootDirectory, true)
 }
 
+// copy does a server side copy from dstObj <- srcObj
+//
+// If newInfo is nil then the metadata will be copied otherwise it
+// will be replaced with newInfo
+func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *api.File) (err error) {
+	if srcObj.size >= int64(f.opt.CopyCutoff) {
+		if newInfo == nil {
+			newInfo, err = srcObj.getMetaData(ctx)
+			if err != nil {
+				return err
+			}
+		}
+		up, err := f.newLargeUpload(ctx, dstObj, nil, srcObj, f.opt.CopyCutoff, true, newInfo)
+		if err != nil {
+			return err
+		}
+		return up.Upload(ctx)
+	}
+
+	dstBucket, dstPath := dstObj.split()
+	err = f.makeBucket(ctx, dstBucket)
+	if err != nil {
+		return err
+	}
+
+	destBucketID, err := f.getBucketID(ctx, dstBucket)
+	if err != nil {
+		return err
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_copy_file",
+	}
+	var request = api.CopyFileRequest{
+		SourceID:     srcObj.id,
+		Name:         f.opt.Enc.FromStandardPath(dstPath),
+		DestBucketID: destBucketID,
+	}
+	if newInfo == nil {
+		request.MetadataDirective = "COPY"
+	} else {
+		request.MetadataDirective = "REPLACE"
+		request.ContentType = newInfo.ContentType
+		request.Info = newInfo.Info
+	}
+	var response api.FileInfo
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	return dstObj.decodeMetaDataFileInfo(&response)
+}
+
 // Copy src to this remote using server side copy operations.
 //
 // This is stored with the remote path given
@@ -1210,47 +1300,21 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 //
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	dstBucket, dstPath := f.split(remote)
-	err := f.makeBucket(ctx, dstBucket)
-	if err != nil {
-		return nil, err
-	}
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	destBucketID, err := f.getBucketID(ctx, dstBucket)
-	if err != nil {
-		return nil, err
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          srcObj.id,
-		Name:              f.opt.Enc.FromStandardPath(dstPath),
-		MetadataDirective: "COPY",
-		DestBucketID:      destBucketID,
-	}
-	var response api.FileInfo
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	o := &Object{
+	// Temporary Object under construction
+	dstObj := &Object{
 		fs:     f,
 		remote: remote,
 	}
-	err = o.decodeMetaDataFileInfo(&response)
+	err := f.copy(ctx, dstObj, srcObj, nil)
 	if err != nil {
 		return nil, err
 	}
-	return o, nil
+	return dstObj, nil
 }
 
 // Hashes returns the supported hash sets.
@@ -1293,7 +1357,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
 }
 
 // PublicLink returns a link for downloading without account
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	bucket, bucketPath := f.split(remote)
 	var RootURL string
 	if f.opt.DownloadURL == "" {
@@ -1370,6 +1434,21 @@ func (o *Object) Size() int64 {
 	return o.size
 }
 
+// Clean the SHA1
+//
+// Make sure it is lower case
+//
+// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
+// Some tools (eg Cyberduck) use this
+func cleanSHA1(sha1 string) (out string) {
+	out = strings.ToLower(sha1)
+	const unverified = "unverified:"
+	if strings.HasPrefix(out, unverified) {
+		out = out[len(unverified):]
+	}
+	return out
+}
+
 // decodeMetaDataRaw sets the metadata from the data passed in
 //
 // Sets
@@ -1385,12 +1464,7 @@ func (o *Object) decodeMetaDataRaw(ID, SHA1 string, Size int64, UploadTimestamp
 	if o.sha1 == "" || o.sha1 == "none" {
 		o.sha1 = Info[sha1Key]
 	}
-	// Remove unverified prefix - see https://www.backblaze.com/b2/docs/uploading.html
-	// Some tools (eg Cyberduck) use this
-	const unverified = "unverified:"
-	if strings.HasPrefix(o.sha1, unverified) {
-		o.sha1 = o.sha1[len(unverified):]
-	}
+	o.sha1 = cleanSHA1(o.sha1)
 	o.size = Size
 	// Use the UploadTimestamp if can't get file info
 	o.modTime = time.Time(UploadTimestamp)
@@ -1511,28 +1585,10 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if err != nil {
 		return err
 	}
-	_, bucketPath := o.split()
 	info.Info[timeKey] = timeString(modTime)
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          o.id,
-		Name:              o.fs.opt.Enc.FromStandardPath(bucketPath), // copy to same name
-		MetadataDirective: "REPLACE",
-		ContentType:       info.ContentType,
-		Info:              info.Info,
-	}
-	var response api.FileInfo
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return o.fs.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return err
-	}
-	return o.decodeMetaDataFileInfo(&response)
+
+	// Copy to the same name, overwriting the metadata only
+	return o.fs.copy(ctx, o, o, info)
 }
 
 // Storable returns if this object is storable
@@ -1648,6 +1704,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			o.sha1 = resp.Header.Get(sha1InfoHeader)
 			fs.Debugf(o, "Reading sha1 from info - %q", o.sha1)
 		}
+		o.sha1 = cleanSHA1(o.sha1)
 	}
 	// Don't check length or hash on partial content
 	if resp.StatusCode == http.StatusPartialContent {
@@ -1707,7 +1764,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	if size == -1 {
 		// Check if the file is large enough for a chunked upload (needs to be at least two chunks)
-		buf := o.fs.getUploadBlock()
+		buf := o.fs.getBuf(false)
+
 		n, err := io.ReadFull(in, buf)
 		if err == nil {
 			bufReader := bufio.NewReader(in)
@@ -1717,22 +1775,24 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		if err == nil {
 			fs.Debugf(o, "File is big enough for chunked streaming")
-			up, err := o.fs.newLargeUpload(ctx, o, in, src)
+			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 			if err != nil {
-				o.fs.putUploadBlock(buf)
+				o.fs.putBuf(buf, false)
 				return err
 			}
+			// NB Stream returns the buffer and token
 			return up.Stream(ctx, buf)
 		} else if err == io.EOF || err == io.ErrUnexpectedEOF {
 			fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
-			defer o.fs.putUploadBlock(buf)
+			defer o.fs.putBuf(buf, false)
 			size = int64(n)
 			in = bytes.NewReader(buf[:n])
 		} else {
+			o.fs.putBuf(buf, false)
 			return err
 		}
 	} else if size > int64(o.fs.opt.UploadCutoff) {
-		up, err := o.fs.newLargeUpload(ctx, o, in, src)
+		up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 		if err != nil {
 			return err
 		}
@@ -1816,6 +1876,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Method:  "POST",
 		RootURL: upload.UploadURL,
 		Body:    in,
+		Options: options,
 		ExtraHeaders: map[string]string{
 			"Authorization":  upload.AuthorizationToken,
 			"X-Bz-File-Name": urlEncode(o.fs.opt.Enc.FromStandardPath(bucketPath)),

backend/b2/upload.go

@@ -20,7 +20,9 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
 )
 
 type hashAppendingReader struct {
@@ -68,20 +70,26 @@ func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader {
 
 // largeUpload is used to control the upload of large files which need chunking
 type largeUpload struct {
-	f        *Fs                             // parent Fs
-	o        *Object                         // object being uploaded
-	in       io.Reader                       // read the data from here
-	wrap     accounting.WrapFn               // account parts being transferred
-	id       string                          // ID of the file being uploaded
-	size     int64                           // total size
-	parts    int64                           // calculated number of parts, if known
-	sha1s    []string                        // slice of SHA1s for each part
-	uploadMu sync.Mutex                      // lock for upload variable
-	uploads  []*api.GetUploadPartURLResponse // result of get upload URL calls
+	f         *Fs                             // parent Fs
+	o         *Object                         // object being uploaded
+	doCopy    bool                            // doing copy rather than upload
+	what      string                          // text name of operation for logs
+	in        io.Reader                       // read the data from here
+	wrap      accounting.WrapFn               // account parts being transferred
+	id        string                          // ID of the file being uploaded
+	size      int64                           // total size
+	parts     int64                           // calculated number of parts, if known
+	sha1s     []string                        // slice of SHA1s for each part
+	uploadMu  sync.Mutex                      // lock for upload variable
+	uploads   []*api.GetUploadPartURLResponse // result of get upload URL calls
+	chunkSize int64                           // chunk size to use
+	src       *Object                         // if copying, object we are reading from
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) {
+//
+// If newInfo is set then metadata from that will be used instead of reading it from src
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	remote := o.remote
 	size := src.Size()
 	parts := int64(0)
@@ -89,8 +97,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
-		parts = size / int64(o.fs.opt.ChunkSize)
-		if size%int64(o.fs.opt.ChunkSize) != 0 {
+		parts = size / int64(chunkSize)
+		if size%int64(chunkSize) != 0 {
 			parts++
 		}
 		if parts > maxParts {
@@ -99,7 +107,6 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		sha1SliceSize = parts
 	}
 
-	modTime := src.ModTime(ctx)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_start_large_file",
@@ -110,18 +117,24 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		return nil, err
 	}
 	var request = api.StartLargeFileRequest{
-		BucketID:    bucketID,
-		Name:        f.opt.Enc.FromStandardPath(bucketPath),
-		ContentType: fs.MimeType(ctx, src),
-		Info: map[string]string{
-			timeKey: timeString(modTime),
-		},
+		BucketID: bucketID,
+		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
-	// Set the SHA1 if known
-	if !o.fs.opt.DisableCheckSum {
-		if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
-			request.Info[sha1Key] = calculatedSha1
+	if newInfo == nil {
+		modTime := src.ModTime(ctx)
+		request.ContentType = fs.MimeType(ctx, src)
+		request.Info = map[string]string{
+			timeKey: timeString(modTime),
 		}
+		// Set the SHA1 if known
+		if !o.fs.opt.DisableCheckSum || doCopy {
+			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
+				request.Info[sha1Key] = calculatedSha1
+			}
+		}
+	} else {
+		request.ContentType = newInfo.ContentType
+		request.Info = newInfo.Info
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
@@ -131,18 +144,24 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if err != nil {
 		return nil, err
 	}
+	up = &largeUpload{
+		f:         f,
+		o:         o,
+		doCopy:    doCopy,
+		what:      "upload",
+		id:        response.ID,
+		size:      size,
+		parts:     parts,
+		sha1s:     make([]string, sha1SliceSize),
+		chunkSize: int64(chunkSize),
+	}
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
-	in, wrap := accounting.UnWrap(in)
-	up = &largeUpload{
-		f:     f,
-		o:     o,
-		in:    in,
-		wrap:  wrap,
-		id:    response.ID,
-		size:  size,
-		parts: parts,
-		sha1s: make([]string, sha1SliceSize),
+	if doCopy {
+		up.what = "copy"
+		up.src = src.(*Object)
+	} else {
+		up.in, up.wrap = accounting.UnWrap(in)
 	}
 	return up, nil
 }
@@ -184,13 +203,6 @@ func (up *largeUpload) returnUploadURL(upload *api.GetUploadPartURLResponse) {
 	up.uploadMu.Unlock()
 }
 
-// clearUploadURL clears the current UploadURL and the AuthorizationToken
-func (up *largeUpload) clearUploadURL() {
-	up.uploadMu.Lock()
-	up.uploads = nil
-	up.uploadMu.Unlock()
-}
-
 // Transfer a chunk
 func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byte) error {
 	err := up.f.pacer.Call(func() (bool, error) {
@@ -263,9 +275,41 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 	return err
 }
 
+// Copy a chunk
+func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
+	err := up.f.pacer.Call(func() (bool, error) {
+		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
+		opts := rest.Opts{
+			Method: "POST",
+			Path:   "/b2_copy_part",
+		}
+		offset := (part - 1) * up.chunkSize // where we are in the source file
+		var request = api.CopyPartRequest{
+			SourceID:    up.src.id,
+			LargeFileID: up.id,
+			PartNumber:  part,
+			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
+		}
+		var response api.UploadPartResponse
+		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
+		retry, err := up.f.shouldRetry(ctx, resp, err)
+		if err != nil {
+			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
+		}
+		up.sha1s[part-1] = response.SHA1
+		return retry, err
+	})
+	if err != nil {
+		fs.Debugf(up.o, "Error copying chunk %d: %v", part, err)
+	} else {
+		fs.Debugf(up.o, "Done copying chunk %d", part)
+	}
+	return err
+}
+
 // finish closes off the large upload
 func (up *largeUpload) finish(ctx context.Context) error {
-	fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
+	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_finish_large_file",
@@ -287,6 +331,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
 
 // cancel aborts the large upload
 func (up *largeUpload) cancel(ctx context.Context) error {
+	fs.Debugf(up.o, "Cancelling large file %s", up.what)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_cancel_large_file",
@@ -299,139 +344,139 @@ func (up *largeUpload) cancel(ctx context.Context) error {
 		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
 		return up.f.shouldRetry(ctx, resp, err)
 	})
-	return err
-}
-
-func (up *largeUpload) managedTransferChunk(ctx context.Context, wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
-	wg.Add(1)
-	go func(part int64, buf []byte) {
-		defer wg.Done()
-		defer up.f.putUploadBlock(buf)
-		err := up.transferChunk(ctx, part, buf)
-		if err != nil {
-			select {
-			case errs <- err:
-			default:
-			}
-		}
-	}(part, buf)
-}
-
-func (up *largeUpload) finishOrCancelOnError(ctx context.Context, err error, errs chan error) error {
-	if err == nil {
-		select {
-		case err = <-errs:
-		default:
-		}
-	}
 	if err != nil {
-		fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
-		cancelErr := up.cancel(ctx)
-		if cancelErr != nil {
-			fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
-		}
-		return err
+		fs.Errorf(up.o, "Failed to cancel large file %s: %v", up.what, err)
 	}
-	return up.finish(ctx)
+	return err
 }
 
 // Stream uploads the chunks from the input, starting with a required initial
 // chunk. Assumes the file size is unknown and will upload until the input
 // reaches EOF.
+//
+// Note that initialUploadBlock must be returned to f.putBuf()
 func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
-	errs := make(chan error, 1)
-	hasMoreParts := true
-	var wg sync.WaitGroup
-
-	// Transfer initial chunk
+	var (
+		g, gCtx      = errgroup.WithContext(ctx)
+		hasMoreParts = true
+	)
 	up.size = int64(len(initialUploadBlock))
-	up.managedTransferChunk(ctx, &wg, errs, 1, initialUploadBlock)
+	g.Go(func() error {
+		for part := int64(1); hasMoreParts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			var buf []byte
+			if part == 1 {
+				buf = initialUploadBlock
+			} else {
+				buf = up.f.getBuf(false)
+			}
 
-outer:
-	for part := int64(2); hasMoreParts; part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+			// Fail fast, in case an errgroup managed function returns an error
+			// gCtx is cancelled. There is no point in uploading all the other parts.
+			if gCtx.Err() != nil {
+				up.f.putBuf(buf, false)
+				return nil
+			}
+
+			// Read the chunk
+			var n int
+			if part == 1 {
+				n = len(buf)
+			} else {
+				n, err = io.ReadFull(up.in, buf)
+				if err == io.ErrUnexpectedEOF {
+					fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
+					buf = buf[:n]
+					hasMoreParts = false
+				} else if err == io.EOF {
+					fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
+					up.f.putBuf(buf, false)
+					return nil
+				} else if err != nil {
+					// other kinds of errors indicate failure
+					up.f.putBuf(buf, false)
+					return err
+				}
+			}
+
+			// Keep stats up to date
+			up.parts = part
+			up.size += int64(n)
+			if part > maxParts {
+				up.f.putBuf(buf, false)
+				return errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+			}
+
+			part := part // for the closure
+			g.Go(func() (err error) {
+				defer up.f.putBuf(buf, false)
+				return up.transferChunk(gCtx, part, buf)
+			})
 		}
-
-		// Get a block of memory
-		buf := up.f.getUploadBlock()
-
-		// Read the chunk
-		var n int
-		n, err = io.ReadFull(up.in, buf)
-		if err == io.ErrUnexpectedEOF {
-			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
-			buf = buf[:n]
-			hasMoreParts = false
-			err = nil
-		} else if err == io.EOF {
-			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
-			up.f.putUploadBlock(buf)
-			err = nil
-			break outer
-		} else if err != nil {
-			// other kinds of errors indicate failure
-			up.f.putUploadBlock(buf)
-			break outer
-		}
-
-		// Keep stats up to date
-		up.parts = part
-		up.size += int64(n)
-		if part > maxParts {
-			err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
-			break outer
-		}
-
-		// Transfer the chunk
-		up.managedTransferChunk(ctx, &wg, errs, part, buf)
+		return nil
+	})
+	err = g.Wait()
+	if err != nil {
+		return err
 	}
-	wg.Wait()
 	up.sha1s = up.sha1s[:up.parts]
-
-	return up.finishOrCancelOnError(ctx, err, errs)
+	return up.finish(ctx)
 }
 
 // Upload uploads the chunks from the input
-func (up *largeUpload) Upload(ctx context.Context) error {
-	fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
-	remaining := up.size
-	errs := make(chan error, 1)
-	var wg sync.WaitGroup
-	var err error
-outer:
-	for part := int64(1); part <= up.parts; part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+func (up *largeUpload) Upload(ctx context.Context) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
+	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
+	var (
+		g, gCtx   = errgroup.WithContext(ctx)
+		remaining = up.size
+	)
+	g.Go(func() error {
+		for part := int64(1); part <= up.parts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			buf := up.f.getBuf(up.doCopy)
+
+			// Fail fast, in case an errgroup managed function returns an error
+			// gCtx is cancelled. There is no point in uploading all the other parts.
+			if gCtx.Err() != nil {
+				up.f.putBuf(buf, up.doCopy)
+				return nil
+			}
+
+			reqSize := remaining
+			if reqSize >= up.chunkSize {
+				reqSize = up.chunkSize
+			}
+
+			if !up.doCopy {
+				// Read the chunk
+				buf = buf[:reqSize]
+				_, err = io.ReadFull(up.in, buf)
+				if err != nil {
+					up.f.putBuf(buf, up.doCopy)
+					return err
+				}
+			}
+
+			part := part // for the closure
+			g.Go(func() (err error) {
+				defer up.f.putBuf(buf, up.doCopy)
+				if !up.doCopy {
+					err = up.transferChunk(gCtx, part, buf)
+				} else {
+					err = up.copyChunk(gCtx, part, reqSize)
+				}
+				return err
+			})
+			remaining -= reqSize
 		}
-
-		reqSize := remaining
-		if reqSize >= int64(up.f.opt.ChunkSize) {
-			reqSize = int64(up.f.opt.ChunkSize)
-		}
-
-		// Get a block of memory
-		buf := up.f.getUploadBlock()[:reqSize]
-
-		// Read the chunk
-		_, err = io.ReadFull(up.in, buf)
-		if err != nil {
-			up.f.putUploadBlock(buf)
-			break outer
-		}
-
-		// Transfer the chunk
-		up.managedTransferChunk(ctx, &wg, errs, part, buf)
-		remaining -= reqSize
+		return nil
+	})
+	err = g.Wait()
+	if err != nil {
+		return err
 	}
-	wg.Wait()
-
-	return up.finishOrCancelOnError(ctx, err, errs)
+	return up.finish(ctx)
 }

backend/box/api/types.go

@@ -222,3 +222,23 @@ type AppAuth struct {
 	PrivateKey  string `json:"privateKey"`
 	Passphrase  string `json:"passphrase"`
 }
+
+// User is returned from /users/me
+type User struct {
+	Type          string    `json:"type"`
+	ID            string    `json:"id"`
+	Name          string    `json:"name"`
+	Login         string    `json:"login"`
+	CreatedAt     time.Time `json:"created_at"`
+	ModifiedAt    time.Time `json:"modified_at"`
+	Language      string    `json:"language"`
+	Timezone      string    `json:"timezone"`
+	SpaceAmount   int64     `json:"space_amount"`
+	SpaceUsed     int64     `json:"space_used"`
+	MaxUploadSize int64     `json:"max_upload_size"`
+	Status        string    `json:"status"`
+	JobTitle      string    `json:"job_title"`
+	Phone         string    `json:"phone"`
+	Address       string    `json:"address"`
+	AvatarURL     string    `json:"avatar_url"`
+}

backend/box/box.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/jwtutil"
 
 	"github.com/youmark/pkcs8"
@@ -53,8 +54,7 @@ const (
 	rcloneEncryptedClientSecret = "sYbJYm99WB8jzeaLPU0OPDMJKIkZvD2qOn3SyEMfiJr03RdtDt3xcZEIudRhbIDL"
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
-	decayConstant               = 2   // bigger for slower decay, exponential
-	rootID                      = "0" // ID of root folder is always this
+	decayConstant               = 2 // bigger for slower decay, exponential
 	rootURL                     = "https://api.box.com/2.0"
 	uploadURL                   = "https://upload.box.com/api/2.0"
 	listChunks                  = 1000     // chunk size to read directory listings
@@ -89,27 +89,12 @@ func init() {
 			boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 			var err error
 			if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
-				boxConfig, err := getBoxConfig(jsonFile)
-				if err != nil {
-					log.Fatalf("Failed to configure token: %v", err)
-				}
-				privateKey, err := getDecryptedPrivateKey(boxConfig)
-				if err != nil {
-					log.Fatalf("Failed to configure token: %v", err)
-				}
-				claims, err := getClaims(boxConfig, boxSubType)
-				if err != nil {
-					log.Fatalf("Failed to configure token: %v", err)
-				}
-				signingHeaders := getSigningHeaders(boxConfig)
-				queryParams := getQueryParams(boxConfig)
-				client := fshttp.NewClient(fs.Config)
-				err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
+				err = refreshJWTToken(jsonFile, boxSubType, name, m)
 				if err != nil {
 					log.Fatalf("Failed to configure token with jwt authentication: %v", err)
 				}
 			} else {
-				err = oauthutil.Config("box", name, m, oauthConfig)
+				err = oauthutil.Config("box", name, m, oauthConfig, nil)
 				if err != nil {
 					log.Fatalf("Failed to configure token with oauth authentication: %v", err)
 				}
@@ -121,9 +106,14 @@ func init() {
 		}, {
 			Name: config.ConfigClientSecret,
 			Help: "Box App Client Secret\nLeave blank normally.",
+		}, {
+			Name:     "root_folder_id",
+			Help:     "Fill in for rclone to use a non root folder as its starting point.",
+			Default:  "0",
+			Advanced: true,
 		}, {
 			Name: "box_config_file",
-			Help: "Box App config.json location\nLeave blank normally.",
+			Help: "Box App config.json location\nLeave blank normally." + env.ShellExpandHelp,
 		}, {
 			Name:    "box_sub_type",
 			Default: "user",
@@ -163,6 +153,27 @@ func init() {
 	})
 }
 
+func refreshJWTToken(jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
+	jsonFile = env.ShellExpand(jsonFile)
+	boxConfig, err := getBoxConfig(jsonFile)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+	}
+	privateKey, err := getDecryptedPrivateKey(boxConfig)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+	}
+	claims, err := getClaims(boxConfig, boxSubType)
+	if err != nil {
+		log.Fatalf("Failed to configure token: %v", err)
+	}
+	signingHeaders := getSigningHeaders(boxConfig)
+	queryParams := getQueryParams(boxConfig)
+	client := fshttp.NewClient(fs.Config)
+	err = jwtutil.Config("box", name, claims, signingHeaders, queryParams, privateKey, m, client)
+	return err
+}
+
 func getBoxConfig(configFile string) (boxConfig *api.ConfigJSON, err error) {
 	file, err := ioutil.ReadFile(configFile)
 	if err != nil {
@@ -185,7 +196,6 @@ func getClaims(boxConfig *api.ConfigJSON, boxSubType string) (claims *jws.ClaimS
 		Iss: boxConfig.BoxAppSettings.ClientID,
 		Sub: boxConfig.EnterpriseID,
 		Aud: tokenURL,
-		Iat: time.Now().Unix(),
 		Exp: time.Now().Add(time.Second * 45).Unix(),
 		PrivateClaims: map[string]interface{}{
 			"box_sub_type": boxSubType,
@@ -236,6 +246,7 @@ type Options struct {
 	UploadCutoff  fs.SizeSuffix        `config:"upload_cutoff"`
 	CommitRetries int                  `config:"commit_retries"`
 	Enc           encoder.MultiEncoder `config:"encoding"`
+	RootFolderID  string               `config:"root_folder_id"`
 }
 
 // Fs represents a remote box
@@ -287,7 +298,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an box 'url'
+// parsePath parses a box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -318,7 +329,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -393,13 +404,27 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// Renew the token in the background
-	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-		_, err := f.readMetaDataForPath(ctx, "")
-		return err
-	})
+	jsonFile, ok := m.Get("box_config_file")
+	boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 
-	// Get rootID
+	// If using box config.json and JWT, renewing should just refresh the token and
+	// should do so whether there are uploads pending or not.
+	if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+			err := refreshJWTToken(jsonFile, boxSubType, name, m)
+			return err
+		})
+		f.tokenRenewer.Start()
+	} else {
+		// Renew the token in the background
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+			_, err := f.readMetaDataForPath(ctx, "")
+			return err
+		})
+	}
+
+	// Get rootFolderID
+	rootID := f.opt.RootFolderID
 	f.dirCache = dircache.New(root, rootID, f)
 
 	// Find the current root
@@ -592,10 +617,6 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -636,7 +657,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -692,13 +713,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -723,10 +738,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -860,6 +871,30 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
 	return info, nil
 }
 
+// About gets quota information
+func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/users/me",
+	}
+	var user api.User
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read user info")
+	}
+	// FIXME max upload size would be useful to use in Update
+	usage = &fs.Usage{
+		Used:  fs.NewUsageValue(user.SpaceUsed),                    // bytes in use
+		Total: fs.NewUsageValue(user.SpaceAmount),                  // bytes total
+		Free:  fs.NewUsageValue(user.SpaceAmount - user.SpaceUsed), // bytes free
+	}
+	return usage, nil
+}
+
 // Move src to this remote using server side move operations.
 //
 // This is stored with the remote path given
@@ -909,64 +944,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	_, err = f.move(ctx, "/folders/", srcID, leaf, directoryID)
+	_, err = f.move(ctx, "/folders/", srcID, dstLeaf, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -975,7 +960,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	id, err := f.dirCache.FindDir(ctx, remote, false)
 	var opts rest.Opts
 	if err == nil {
@@ -1014,6 +999,66 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
 	return info.SharedLink.URL, err
 }
 
+// deletePermanently permenently deletes a trashed file
+func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
+	opts := rest.Opts{
+		Method:     "DELETE",
+		NoResponse: true,
+	}
+	if itemType == api.ItemTypeFile {
+		opts.Path = "/files/" + id + "/trash"
+	} else {
+		opts.Path = "/folders/" + id + "/trash"
+	}
+	return f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.Call(ctx, &opts)
+		return shouldRetry(resp, err)
+	})
+}
+
+// CleanUp empties the trash
+func (f *Fs) CleanUp(ctx context.Context) (err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/folders/trash/items",
+		Parameters: url.Values{
+			"fields": []string{"type", "id"},
+		},
+	}
+	opts.Parameters.Set("limit", strconv.Itoa(listChunks))
+	offset := 0
+	for {
+		opts.Parameters.Set("offset", strconv.Itoa(offset))
+
+		var result api.FolderItems
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(resp, err)
+		})
+		if err != nil {
+			return errors.Wrap(err, "couldn't list trash")
+		}
+		for i := range result.Entries {
+			item := &result.Entries[i]
+			if item.Type == api.ItemTypeFolder || item.Type == api.ItemTypeFile {
+				err := f.deletePermanently(ctx, item.Type, item.ID)
+				if err != nil {
+					return errors.Wrap(err, "failed to delete file")
+				}
+			} else {
+				fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)
+				continue
+			}
+		}
+		offset += result.Limit
+		if offset >= result.TotalCount {
+			break
+		}
+	}
+	return
+}
+
 // DirCacheFlush resets the directory cache - used in testing as an
 // optional interface
 func (f *Fs) DirCacheFlush() {
@@ -1167,7 +1212,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // upload does a single non-multipart upload
 //
 // This is recommended for less than 50 MB of content
-func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time) (err error) {
+func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID string, modTime time.Time, options ...fs.OpenOption) (err error) {
 	upload := api.UploadFile{
 		Name:              o.fs.opt.Enc.FromStandardName(leaf),
 		ContentModifiedAt: api.Time(modTime),
@@ -1186,6 +1231,7 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 		MultipartContentName:  "contents",
 		MultipartFileName:     upload.Name,
 		RootURL:               uploadURL,
+		Options:               options,
 	}
 	// If object has an ID then it is existing so create a new version
 	if o.id != "" {
@@ -1220,16 +1266,16 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
 
 	// Upload with simple or multipart
 	if size <= int64(o.fs.opt.UploadCutoff) {
-		err = o.upload(ctx, in, leaf, directoryID, modTime)
+		err = o.upload(ctx, in, leaf, directoryID, modTime, options...)
 	} else {
-		err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime)
+		err = o.uploadMultipart(ctx, in, leaf, directoryID, size, modTime, options...)
 	}
 	return err
 }
@@ -1250,10 +1296,12 @@ var (
 	_ fs.Purger          = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/box/upload.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -54,7 +55,7 @@ func sha1Digest(digest []byte) string {
 }
 
 // uploadPart uploads a part in an upload session
-func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn) (response *api.UploadPartResponse, err error) {
+func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, totalSize int64, chunk []byte, wrap accounting.WrapFn, options ...fs.OpenOption) (response *api.UploadPartResponse, err error) {
 	chunkSize := int64(len(chunk))
 	sha1sum := sha1.Sum(chunk)
 	opts := rest.Opts{
@@ -64,6 +65,7 @@ func (o *Object) uploadPart(ctx context.Context, SessionID string, offset, total
 		ContentType:   "application/octet-stream",
 		ContentLength: &chunkSize,
 		ContentRange:  fmt.Sprintf("bytes %d-%d/%d", offset, offset+chunkSize-1, totalSize),
+		Options:       options,
 		ExtraHeaders: map[string]string{
 			"Digest": sha1Digest(sha1sum[:]),
 		},
@@ -171,7 +173,7 @@ func (o *Object) abortUpload(ctx context.Context, SessionID string) (err error)
 }
 
 // uploadMultipart uploads a file using multipart upload
-func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time) (err error) {
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, directoryID string, size int64, modTime time.Time, options ...fs.OpenOption) (err error) {
 	// Create upload session
 	session, err := o.createUploadSession(ctx, leaf, directoryID, size)
 	if err != nil {
@@ -181,15 +183,13 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize))
 
 	// Cancel the session if something went wrong
-	defer func() {
-		if err != nil {
-			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-			cancelErr := o.abortUpload(ctx, session.ID)
-			if cancelErr != nil {
-				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
-			}
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(o, "Cancelling multipart upload: %v", err)
+		cancelErr := o.abortUpload(ctx, session.ID)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-	}()
+	})()
 
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
@@ -236,7 +236,7 @@ outer:
 			defer wg.Done()
 			defer o.fs.uploadToken.Put()
 			fs.Debugf(o, "Uploading part %d/%d offset %v/%v part size %v", part+1, session.TotalParts, fs.SizeSuffix(position), fs.SizeSuffix(size), fs.SizeSuffix(chunkSize))
-			partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap)
+			partResponse, err := o.uploadPart(ctx, session.ID, position, size, buf, wrap, options...)
 			if err != nil {
 				err = errors.Wrap(err, "multipart upload failed to upload part")
 				select {

backend/cache/cache.go

@@ -65,6 +65,7 @@ func init() {
 		Name:        "cache",
 		Description: "Cache a remote",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to cache.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
@@ -86,7 +87,7 @@ func init() {
 			Advanced: true,
 		}, {
 			Name:     "plex_insecure",
-			Help:     "Skip all certificate verifications when connecting to the Plex server",
+			Help:     "Skip all certificate verification when connecting to the Plex server",
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -338,7 +339,7 @@ func parseRootPath(path string) (string, error) {
 	return strings.Trim(path, "/"), nil
 }
 
-// NewFs constructs a Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -519,9 +520,6 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// override only those features that use a temp fs and it doesn't support them
 	//f.features.ChangeNotify = f.ChangeNotify
 	if f.opt.TempWritePath != "" {
-		if f.tempFs.Features().Copy == nil {
-			f.features.Copy = nil
-		}
 		if f.tempFs.Features().Move == nil {
 			f.features.Move = nil
 		}
@@ -1532,6 +1530,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs())
 		return nil, fs.ErrorCantCopy
 	}
+	if f.opt.TempWritePath != "" && src.Fs() == f.tempFs {
+		return nil, fs.ErrorCantCopy
+	}
 	// the source must be a cached object or we abort
 	srcObj, ok := src.(*Object)
 	if !ok {
@@ -1828,6 +1829,19 @@ func (f *Fs) isRootInPath(p string) bool {
 	return strings.HasPrefix(p, f.Root()+"/")
 }
 
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
+	do := f.Fs.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	for _, dir := range dirs {
+		_ = f.cache.RemoveDir(dir.Remote())
+	}
+	return do(ctx, dirs)
+}
+
 // DirCacheFlush flushes the dir cache
 func (f *Fs) DirCacheFlush() {
 	_ = f.cache.RemoveDir("")
@@ -1882,6 +1896,31 @@ func (f *Fs) Disconnect(ctx context.Context) error {
 	return do(ctx)
 }
 
+var commandHelp = []fs.CommandHelp{
+	{
+		Name:  "stats",
+		Short: "Print stats on the cache backend in JSON format.",
+	},
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+	switch name {
+	case "stats":
+		return f.Stats()
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs             = (*Fs)(nil)
@@ -1899,4 +1938,6 @@ var (
 	_ fs.Abouter        = (*Fs)(nil)
 	_ fs.UserInfoer     = (*Fs)(nil)
 	_ fs.Disconnecter   = (*Fs)(nil)
+	_ fs.Commander      = (*Fs)(nil)
+	_ fs.MergeDirser    = (*Fs)(nil)
 )

backend/cache/cache_internal_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
@@ -299,6 +300,7 @@ func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {
 }
 
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
+	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -342,6 +344,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 }
 
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
+	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -1009,6 +1012,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		return nil, nil
 	}
 
+	// Config to pass to NewFs
+	m := configmap.Simple{}
+	for k, v := range r.runDefaultCfgMap {
+		m.Set(k, v)
+	}
+	for k, v := range flags {
+		m.Set(k, v)
+	}
+
 	// if the remote doesn't exist, create a new one with a local one for it
 	// identify which is the cache remote (it can be wrapped by a crypt too)
 	rootIsCrypt := false
@@ -1017,8 +1029,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		localRemote := remote + "-local"
 		config.FileSet(localRemote, "type", "local")
 		config.FileSet(localRemote, "nounc", "true")
-		config.FileSet(remote, "type", "cache")
-		config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
+		m.Set("type", "cache")
+		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
 		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
@@ -1028,8 +1040,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		if remoteType != "cache" {
 			if remoteType == "crypt" {
 				rootIsCrypt = true
-				config.FileSet(remote, "password", cryptPassword1)
-				config.FileSet(remote, "password2", cryptPassword2)
+				m.Set("password", cryptPassword1)
+				m.Set("password2", cryptPassword2)
 			}
 			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
@@ -1055,14 +1067,6 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 
 	fs.Config.LowLevelRetries = 1
 
-	m := configmap.Simple{}
-	for k, v := range r.runDefaultCfgMap {
-		m.Set(k, v)
-	}
-	for k, v := range flags {
-		m.Set(k, v)
-	}
-
 	// Instantiate root
 	if purge {
 		boltDb.PurgeTempUploads()

backend/cache/cache_test.go

@@ -18,7 +18,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
 		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})

backend/cache/storage_persistent.go

@@ -16,10 +16,10 @@ import (
 	"sync"
 	"time"
 
-	bolt "github.com/etcd-io/bbolt"
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/walk"
+	bolt "go.etcd.io/bbolt"
 )
 
 // Constants
@@ -980,15 +980,6 @@ func (b *Persistent) updatePendingUpload(remote string, fn func(item *tempUpload
 	})
 }
 
-// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
-// TO BE USED IN TESTING ONLY
-func (b *Persistent) SetPendingUploadToStarted(remote string) error {
-	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
-		item.Started = true
-		return nil
-	})
-}
-
 // ReconcileTempUploads will recursively look for all the files in the temp directory and add them to the queue
 func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) error {
 	return b.db.Update(func(tx *bolt.Tx) error {
@@ -1036,19 +1027,6 @@ func (b *Persistent) ReconcileTempUploads(ctx context.Context, cacheFs *Fs) erro
 	})
 }
 
-// PurgeTempUploads will remove all the pending uploads from the queue
-// TO BE USED IN TESTING ONLY
-func (b *Persistent) PurgeTempUploads() {
-	b.tempQueueMux.Lock()
-	defer b.tempQueueMux.Unlock()
-
-	_ = b.db.Update(func(tx *bolt.Tx) error {
-		_ = tx.DeleteBucket([]byte(tempBucket))
-		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
-		return nil
-	})
-}
-
 // Close should be called when the program ends gracefully
 func (b *Persistent) Close() {
 	b.cleanupMux.Lock()

backend/cache/utils_test.go

@@ -0,0 +1,23 @@
+package cache
+
+import bolt "go.etcd.io/bbolt"
+
+// PurgeTempUploads will remove all the pending uploads from the queue
+func (b *Persistent) PurgeTempUploads() {
+	b.tempQueueMux.Lock()
+	defer b.tempQueueMux.Unlock()
+
+	_ = b.db.Update(func(tx *bolt.Tx) error {
+		_ = tx.DeleteBucket([]byte(tempBucket))
+		_, _ = tx.CreateBucketIfNotExists([]byte(tempBucket))
+		return nil
+	})
+}
+
+// SetPendingUploadToStarted is a way to mark an entry as started (even if it's not already)
+func (b *Persistent) SetPendingUploadToStarted(remote string) error {
+	return b.updatePendingUpload(remote, func(item *tempUploadInfo) error {
+		item.Started = true
+		return nil
+	})
+}

backend/crypt/cipher.go

@@ -71,30 +71,6 @@ type ReadSeekCloser interface {
 // OpenRangeSeek opens the file handle at the offset with the limit given
 type OpenRangeSeek func(ctx context.Context, offset, limit int64) (io.ReadCloser, error)
 
-// Cipher is used to swap out the encryption implementations
-type Cipher interface {
-	// EncryptFileName encrypts a file path
-	EncryptFileName(string) string
-	// DecryptFileName decrypts a file path, returns error if decrypt was invalid
-	DecryptFileName(string) (string, error)
-	// EncryptDirName encrypts a directory path
-	EncryptDirName(string) string
-	// DecryptDirName decrypts a directory path, returns error if decrypt was invalid
-	DecryptDirName(string) (string, error)
-	// EncryptData
-	EncryptData(io.Reader) (io.Reader, error)
-	// DecryptData
-	DecryptData(io.ReadCloser) (io.ReadCloser, error)
-	// DecryptDataSeek decrypt at a given position
-	DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error)
-	// EncryptedSize calculates the size of the data when encrypted
-	EncryptedSize(int64) int64
-	// DecryptedSize calculates the size of the data when decrypted
-	DecryptedSize(int64) (int64, error)
-	// NameEncryptionMode returns the used mode for name handling
-	NameEncryptionMode() NameEncryptionMode
-}
-
 // NameEncryptionMode is the type of file name encryption in use
 type NameEncryptionMode int
 
@@ -136,7 +112,8 @@ func (mode NameEncryptionMode) String() (out string) {
 	return out
 }
 
-type cipher struct {
+// Cipher defines an encoding and decoding cipher for the crypt backend
+type Cipher struct {
 	dataKey        [32]byte                  // Key for secretbox
 	nameKey        [32]byte                  // 16,24 or 32 bytes
 	nameTweak      [nameCipherBlockSize]byte // used to tweak the name crypto
@@ -148,8 +125,8 @@ type cipher struct {
 }
 
 // newCipher initialises the cipher.  If salt is "" then it uses a built in salt val
-func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*cipher, error) {
-	c := &cipher{
+func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bool) (*Cipher, error) {
+	c := &Cipher{
 		mode:           mode,
 		cryptoRand:     rand.Reader,
 		dirNameEncrypt: dirNameEncrypt,
@@ -172,7 +149,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 //
 // Note that empty passsword makes all 0x00 keys which is used in the
 // tests.
-func (c *cipher) Key(password, salt string) (err error) {
+func (c *Cipher) Key(password, salt string) (err error) {
 	const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
 	var saltBytes = defaultSalt
 	if salt != "" {
@@ -196,12 +173,12 @@ func (c *cipher) Key(password, salt string) (err error) {
 }
 
 // getBlock gets a block from the pool of size blockSize
-func (c *cipher) getBlock() []byte {
+func (c *Cipher) getBlock() []byte {
 	return c.buffers.Get().([]byte)
 }
 
 // putBlock returns a block to the pool of size blockSize
-func (c *cipher) putBlock(buf []byte) {
+func (c *Cipher) putBlock(buf []byte) {
 	if len(buf) != blockSize {
 		panic("bad blocksize returned to pool")
 	}
@@ -240,13 +217,13 @@ func decodeFileName(in string) ([]byte, error) {
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
 // Rogaway.
 //
-// This makes for determinstic encryption which is what we want - the
+// This makes for deterministic encryption which is what we want - the
 // same filename must encrypt to the same thing.
 //
 // This means that
 //  * filenames with the same name will encrypt the same
 //  * filenames which start the same won't have a common prefix
-func (c *cipher) encryptSegment(plaintext string) string {
+func (c *Cipher) encryptSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -256,7 +233,7 @@ func (c *cipher) encryptSegment(plaintext string) string {
 }
 
 // decryptSegment decrypts a path segment
-func (c *cipher) decryptSegment(ciphertext string) (string, error) {
+func (c *Cipher) decryptSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -283,7 +260,7 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 }
 
 // Simple obfuscation routines
-func (c *cipher) obfuscateSegment(plaintext string) string {
+func (c *Cipher) obfuscateSegment(plaintext string) string {
 	if plaintext == "" {
 		return ""
 	}
@@ -370,7 +347,7 @@ func (c *cipher) obfuscateSegment(plaintext string) string {
 	return result.String()
 }
 
-func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
+func (c *Cipher) deobfuscateSegment(ciphertext string) (string, error) {
 	if ciphertext == "" {
 		return "", nil
 	}
@@ -457,7 +434,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 }
 
 // encryptFileName encrypts a file path
-func (c *cipher) encryptFileName(in string) string {
+func (c *Cipher) encryptFileName(in string) string {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		// Skip directory name encryption if the user chose to
@@ -475,7 +452,7 @@ func (c *cipher) encryptFileName(in string) string {
 }
 
 // EncryptFileName encrypts a file path
-func (c *cipher) EncryptFileName(in string) string {
+func (c *Cipher) EncryptFileName(in string) string {
 	if c.mode == NameEncryptionOff {
 		return in + encryptedSuffix
 	}
@@ -483,7 +460,7 @@ func (c *cipher) EncryptFileName(in string) string {
 }
 
 // EncryptDirName encrypts a directory path
-func (c *cipher) EncryptDirName(in string) string {
+func (c *Cipher) EncryptDirName(in string) string {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in
 	}
@@ -491,7 +468,7 @@ func (c *cipher) EncryptDirName(in string) string {
 }
 
 // decryptFileName decrypts a file path
-func (c *cipher) decryptFileName(in string) (string, error) {
+func (c *Cipher) decryptFileName(in string) (string, error) {
 	segments := strings.Split(in, "/")
 	for i := range segments {
 		var err error
@@ -514,7 +491,7 @@ func (c *cipher) decryptFileName(in string) (string, error) {
 }
 
 // DecryptFileName decrypts a file path
-func (c *cipher) DecryptFileName(in string) (string, error) {
+func (c *Cipher) DecryptFileName(in string) (string, error) {
 	if c.mode == NameEncryptionOff {
 		remainingLength := len(in) - len(encryptedSuffix)
 		if remainingLength > 0 && strings.HasSuffix(in, encryptedSuffix) {
@@ -526,14 +503,15 @@ func (c *cipher) DecryptFileName(in string) (string, error) {
 }
 
 // DecryptDirName decrypts a directory path
-func (c *cipher) DecryptDirName(in string) (string, error) {
+func (c *Cipher) DecryptDirName(in string) (string, error) {
 	if c.mode == NameEncryptionOff || !c.dirNameEncrypt {
 		return in, nil
 	}
 	return c.decryptFileName(in)
 }
 
-func (c *cipher) NameEncryptionMode() NameEncryptionMode {
+// NameEncryptionMode returns the encryption mode in use for names
+func (c *Cipher) NameEncryptionMode() NameEncryptionMode {
 	return c.mode
 }
 
@@ -581,7 +559,7 @@ func (n *nonce) increment() {
 	n.carry(0)
 }
 
-// add an uint64 to the nonce
+// add a uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
 	for i := 0; i < 8; i++ {
@@ -601,7 +579,7 @@ func (n *nonce) add(x uint64) {
 type encrypter struct {
 	mu       sync.Mutex
 	in       io.Reader
-	c        *cipher
+	c        *Cipher
 	nonce    nonce
 	buf      []byte
 	readBuf  []byte
@@ -611,7 +589,7 @@ type encrypter struct {
 }
 
 // newEncrypter creates a new file handle encrypting on the fly
-func (c *cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
+func (c *Cipher) newEncrypter(in io.Reader, nonce *nonce) (*encrypter, error) {
 	fh := &encrypter{
 		in:      in,
 		c:       c,
@@ -683,13 +661,19 @@ func (fh *encrypter) finish(err error) (int, error) {
 }
 
 // Encrypt data encrypts the data stream
-func (c *cipher) EncryptData(in io.Reader) (io.Reader, error) {
+func (c *Cipher) encryptData(in io.Reader) (io.Reader, *encrypter, error) {
 	in, wrap := accounting.UnWrap(in) // unwrap the accounting off the Reader
 	out, err := c.newEncrypter(in, nil)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
-	return wrap(out), nil // and wrap the accounting back on
+	return wrap(out), out, nil // and wrap the accounting back on
+}
+
+// EncryptData encrypts the data stream
+func (c *Cipher) EncryptData(in io.Reader) (io.Reader, error) {
+	out, _, err := c.encryptData(in)
+	return out, err
 }
 
 // decrypter decrypts an io.ReaderCloser on the fly
@@ -698,7 +682,7 @@ type decrypter struct {
 	rc           io.ReadCloser
 	nonce        nonce
 	initialNonce nonce
-	c            *cipher
+	c            *Cipher
 	buf          []byte
 	readBuf      []byte
 	bufIndex     int
@@ -709,7 +693,7 @@ type decrypter struct {
 }
 
 // newDecrypter creates a new file handle decrypting on the fly
-func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
+func (c *Cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	fh := &decrypter{
 		rc:      rc,
 		c:       c,
@@ -737,7 +721,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 }
 
 // newDecrypterSeek creates a new file handle decrypting on the fly
-func (c *cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
+func (c *Cipher) newDecrypterSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (fh *decrypter, err error) {
 	var rc io.ReadCloser
 	doRangeSeek := false
 	setLimit := false
@@ -1012,7 +996,7 @@ func (fh *decrypter) finishAndClose(err error) error {
 }
 
 // DecryptData decrypts the data stream
-func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
+func (c *Cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 	out, err := c.newDecrypter(rc)
 	if err != nil {
 		return nil, err
@@ -1025,7 +1009,7 @@ func (c *cipher) DecryptData(rc io.ReadCloser) (io.ReadCloser, error) {
 // The open function must return a ReadCloser opened to the offset supplied
 //
 // You must use this form of DecryptData if you might want to Seek the file handle
-func (c *cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
+func (c *Cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset, limit int64) (ReadSeekCloser, error) {
 	out, err := c.newDecrypterSeek(ctx, open, offset, limit)
 	if err != nil {
 		return nil, err
@@ -1034,7 +1018,7 @@ func (c *cipher) DecryptDataSeek(ctx context.Context, open OpenRangeSeek, offset
 }
 
 // EncryptedSize calculates the size of the data when encrypted
-func (c *cipher) EncryptedSize(size int64) int64 {
+func (c *Cipher) EncryptedSize(size int64) int64 {
 	blocks, residue := size/blockDataSize, size%blockDataSize
 	encryptedSize := int64(fileHeaderSize) + blocks*(blockHeaderSize+blockDataSize)
 	if residue != 0 {
@@ -1044,7 +1028,7 @@ func (c *cipher) EncryptedSize(size int64) int64 {
 }
 
 // DecryptedSize calculates the size of the data when decrypted
-func (c *cipher) DecryptedSize(size int64) (int64, error) {
+func (c *Cipher) DecryptedSize(size int64) (int64, error) {
 	size -= int64(fileHeaderSize)
 	if size < 0 {
 		return 0, ErrorEncryptedFileTooShort
@@ -1063,7 +1047,6 @@ func (c *cipher) DecryptedSize(size int64) (int64, error) {
 
 // check interfaces
 var (
-	_ Cipher         = (*cipher)(nil)
 	_ io.ReadCloser  = (*decrypter)(nil)
 	_ io.Seeker      = (*decrypter)(nil)
 	_ fs.RangeSeeker = (*decrypter)(nil)

backend/crypt/cipher_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/crypt/pkcs7"
+	"github.com/rclone/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -784,7 +785,7 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
 
-	in := &errorReader{io.ErrUnexpectedEOF}
+	in := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	fh, err := c.newEncrypter(in, nil)
 	assert.NoError(t, err)
 
@@ -793,14 +794,6 @@ func TestNewEncrypterErrUnexpectedEOF(t *testing.T) {
 	assert.Equal(t, int64(32), n)
 }
 
-type errorReader struct {
-	err error
-}
-
-func (er errorReader) Read(p []byte) (n int, err error) {
-	return 0, er.err
-}
-
 type closeDetector struct {
 	io.Reader
 	closed int
@@ -838,7 +831,7 @@ func TestNewDecrypter(t *testing.T) {
 		assert.Equal(t, 1, cd.closed)
 	}
 
-	er := &errorReader{errors.New("potato")}
+	er := &readers.ErrorReader{Err: errors.New("potato")}
 	cd = newCloseDetector(er)
 	fh, err = c.newDecrypter(cd)
 	assert.Nil(t, fh)
@@ -864,7 +857,7 @@ func TestNewDecrypterErrUnexpectedEOF(t *testing.T) {
 	c, err := newCipher(NameEncryptionStandard, "", "", true)
 	assert.NoError(t, err)
 
-	in2 := &errorReader{io.ErrUnexpectedEOF}
+	in2 := &readers.ErrorReader{Err: io.ErrUnexpectedEOF}
 	in1 := bytes.NewBuffer(file16)
 	in := ioutil.NopCloser(io.MultiReader(in1, in2))
 
@@ -936,7 +929,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 		assert.Equal(t, 0, n)
 	}
 
-	// Now try decoding it with a open/seek
+	// Now try decoding it with an open/seek
 	for _, offset := range trials {
 		for _, limit := range limits {
 			if offset+limit > len(plaintext) {
@@ -1118,7 +1111,7 @@ func TestDecrypterRead(t *testing.T) {
 
 	// Test producing an error on the file on Read the underlying file
 	in1 := bytes.NewBuffer(file1)
-	in2 := &errorReader{errors.New("potato")}
+	in2 := &readers.ErrorReader{Err: errors.New("potato")}
 	in := io.MultiReader(in1, in2)
 	cd := newCloseDetector(in)
 	fh, err := c.newDecrypter(cd)

backend/crypt/crypt.go

@@ -26,6 +26,7 @@ func init() {
 		Name:        "crypt",
 		Description: "Encrypt/Decrypt a remote",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name:     "remote",
 			Help:     "Remote to encrypt/decrypt.\nNormally should contain a ':' and a path, eg \"myremote:path/to/dir\",\n\"myremote:bucket\" or maybe \"myremote:\" (not recommended).",
@@ -71,6 +72,20 @@ NB If filename_encryption is "off" then this option will do nothing.`,
 			Name:       "password2",
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
 			IsPassword: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different crypt configs.
+
+Normally this option is not what you want, but if you have two crypts
+pointing to the same backend you can use it.
+
+This can be used, for example, to change file name encryption type
+without re-uploading all the data. Just make two crypt backends
+pointing to two different directories with the single changed
+parameter and use rclone move to move the files between the crypt
+remotes.`,
+			Advanced: true,
 		}, {
 			Name: "show_mapping",
 			Help: `For all files listed show how the names encrypt.
@@ -90,7 +105,7 @@ names, or for debugging purposes.`,
 }
 
 // newCipherForConfig constructs a Cipher for the given config name
-func newCipherForConfig(opt *Options) (Cipher, error) {
+func newCipherForConfig(opt *Options) (*Cipher, error) {
 	mode, err := NewNameEncryptionMode(opt.FilenameEncryption)
 	if err != nil {
 		return nil, err
@@ -117,7 +132,7 @@ func newCipherForConfig(opt *Options) (Cipher, error) {
 }
 
 // NewCipher constructs a Cipher for the given config
-func NewCipher(m configmap.Mapper) (Cipher, error) {
+func NewCipher(m configmap.Mapper) (*Cipher, error) {
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -180,6 +195,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		CanHaveEmptyDirectories: true,
 		SetTier:                 true,
 		GetTier:                 true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -192,6 +208,7 @@ type Options struct {
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
+	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
 }
 
@@ -203,7 +220,7 @@ type Fs struct {
 	root     string
 	opt      Options
 	features *fs.Features // optional features
-	cipher   Cipher
+	cipher   *Cipher
 }
 
 // Name of the remote (as passed into NewFs)
@@ -240,7 +257,7 @@ func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
 	*entries = append(*entries, f.newObject(obj))
 }
 
-// Encrypt an directory file name to entries.
+// Encrypt a directory file name to entries.
 func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
@@ -327,7 +344,7 @@ type putFn func(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ..
 // put implements Put or PutStream
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
 	// Encrypt the data into wrappedIn
-	wrappedIn, err := f.cipher.EncryptData(in)
+	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
 		return nil, err
 	}
@@ -351,7 +368,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options [
 	}
 
 	// Transfer the data
-	o, err := put(ctx, wrappedIn, f.newObjectInfo(src), options...)
+	o, err := put(ctx, wrappedIn, f.newObjectInfo(src, encrypter.nonce), options...)
 	if err != nil {
 		return nil, err
 	}
@@ -504,11 +521,11 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	if do == nil {
 		return nil, errors.New("can't PutUnchecked")
 	}
-	wrappedIn, err := f.cipher.EncryptData(in)
+	wrappedIn, encrypter, err := f.cipher.encryptData(in)
 	if err != nil {
 		return nil, err
 	}
-	o, err := do(ctx, wrappedIn, f.newObjectInfo(src))
+	o, err := do(ctx, wrappedIn, f.newObjectInfo(src, encrypter.nonce))
 	if err != nil {
 		return nil, err
 	}
@@ -561,6 +578,37 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 	return f.cipher.DecryptFileName(encryptedFileName)
 }
 
+// computeHashWithNonce takes the nonce and encrypts the contents of
+// src with it, and calculates the hash given by HashType on the fly
+//
+// Note that we break lots of encapsulation in this function.
+func (f *Fs) computeHashWithNonce(ctx context.Context, nonce nonce, src fs.Object, hashType hash.Type) (hashStr string, err error) {
+	// Open the src for input
+	in, err := src.Open(ctx)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to open src")
+	}
+	defer fs.CheckClose(in, &err)
+
+	// Now encrypt the src with the nonce
+	out, err := f.cipher.newEncrypter(in, &nonce)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make encrypter")
+	}
+
+	// pipe into hash
+	m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
+	if err != nil {
+		return "", errors.Wrap(err, "failed to make hasher")
+	}
+	_, err = io.Copy(m, out)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to hash data")
+	}
+
+	return m.Sums()[hashType], nil
+}
+
 // ComputeHash takes the nonce from o, and encrypts the contents of
 // src with it, and calculates the hash given by HashType on the fly
 //
@@ -572,7 +620,7 @@ func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType
 	if err != nil {
 		return "", errors.Wrap(err, "failed to open object to read nonce")
 	}
-	d, err := f.cipher.(*cipher).newDecrypter(in)
+	d, err := f.cipher.newDecrypter(in)
 	if err != nil {
 		_ = in.Close()
 		return "", errors.Wrap(err, "failed to open object to read nonce")
@@ -597,30 +645,7 @@ func (f *Fs) ComputeHash(ctx context.Context, o *Object, src fs.Object, hashType
 		return "", errors.Wrap(err, "failed to close nonce read")
 	}
 
-	// Open the src for input
-	in, err = src.Open(ctx)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to open src")
-	}
-	defer fs.CheckClose(in, &err)
-
-	// Now encrypt the src with the nonce
-	out, err := f.cipher.(*cipher).newEncrypter(in, &nonce)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to make encrypter")
-	}
-
-	// pipe into hash
-	m, err := hash.NewMultiHasherTypes(hash.NewHashSet(hashType))
-	if err != nil {
-		return "", errors.Wrap(err, "failed to make hasher")
-	}
-	_, err = io.Copy(m, out)
-	if err != nil {
-		return "", errors.Wrap(err, "failed to hash data")
-	}
-
-	return m.Sums()[hashType], nil
+	return f.computeHashWithNonce(ctx, nonce, src, hashType)
 }
 
 // MergeDirs merges the contents of all the directories passed
@@ -647,7 +672,7 @@ func (f *Fs) DirCacheFlush() {
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	do := f.Fs.Features().PublicLink
 	if do == nil {
 		return "", errors.New("PublicLink not supported")
@@ -655,9 +680,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
 	o, err := f.NewObject(ctx, remote)
 	if err != nil {
 		// assume it is a directory
-		return do(ctx, f.cipher.EncryptDirName(remote))
+		return do(ctx, f.cipher.EncryptDirName(remote), expire, unlink)
 	}
-	return do(ctx, o.(*Object).Object.Remote())
+	return do(ctx, o.(*Object).Object.Remote(), expire, unlink)
 }
 
 // ChangeNotify calls the passed function with a path
@@ -692,6 +717,67 @@ func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryT
 	do(ctx, wrappedNotifyFunc, pollIntervalChan)
 }
 
+var commandHelp = []fs.CommandHelp{
+	{
+		Name:  "encode",
+		Short: "Encode the given filename(s)",
+		Long: `This encodes the filenames given as arguments returning a list of
+strings of the encoded results.
+
+Usage Example:
+
+    rclone backend encode crypt: file1 [file2...]
+    rclone rc backend/command command=encode fs=crypt: file1 [file2...]
+`,
+	},
+	{
+		Name:  "decode",
+		Short: "Decode the given filename(s)",
+		Long: `This decodes the filenames given as arguments returning a list of
+strings of the decoded results. It will return an error if any of the
+inputs are invalid.
+
+Usage Example:
+
+    rclone backend decode crypt: encryptedfile1 [encryptedfile2...]
+    rclone rc backend/command command=decode fs=crypt: encryptedfile1 [encryptedfile2...]
+`,
+	},
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "decode":
+		out := make([]string, 0, len(arg))
+		for _, encryptedFileName := range arg {
+			fileName, err := f.DecryptFileName(encryptedFileName)
+			if err != nil {
+				return out, errors.Wrap(err, fmt.Sprintf("Failed to decrypt : %s", encryptedFileName))
+			}
+			out = append(out, fileName)
+		}
+		return out, nil
+	case "encode":
+		out := make([]string, 0, len(arg))
+		for _, fileName := range arg {
+			encryptedFileName := f.EncryptFileName(fileName)
+			out = append(out, encryptedFileName)
+		}
+		return out, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // Object describes a wrapped for being read from the Fs
 //
 // This decrypts the remote name and decrypts the data
@@ -833,13 +919,15 @@ func (f *Fs) Disconnect(ctx context.Context) error {
 // This encrypts the remote name and adjusts the size
 type ObjectInfo struct {
 	fs.ObjectInfo
-	f *Fs
+	f     *Fs
+	nonce nonce
 }
 
-func (f *Fs) newObjectInfo(src fs.ObjectInfo) *ObjectInfo {
+func (f *Fs) newObjectInfo(src fs.ObjectInfo, nonce nonce) *ObjectInfo {
 	return &ObjectInfo{
 		ObjectInfo: src,
 		f:          f,
+		nonce:      nonce,
 	}
 }
 
@@ -865,6 +953,23 @@ func (o *ObjectInfo) Size() int64 {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
+	var srcObj fs.Object
+	var ok bool
+	// Get the underlying object if there is one
+	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
+		// Prefer direct interface assertion
+	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
+		// Otherwise likely is an operations.OverrideRemote
+		srcObj = do.UnWrap()
+	} else {
+		return "", nil
+	}
+	// if this is wrapping a local object then we work out the hash
+	if srcObj.Fs().Features().IsLocal {
+		// Read the data and encrypt it to calculate the hash
+		fs.Debugf(o, "Computing %v hash of encrypted source", hash)
+		return o.f.computeHashWithNonce(ctx, o.nonce, srcObj, hash)
+	}
 	return "", nil
 }
 
@@ -903,6 +1008,7 @@ var (
 	_ fs.Copier          = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
+	_ fs.Commander       = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.CleanUpper      = (*Fs)(nil)

backend/crypt/crypt_internal_test.go

@@ -0,0 +1,143 @@
+package crypt
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+	"testing"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/lib/random"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type testWrapper struct {
+	fs.ObjectInfo
+}
+
+// UnWrap returns the Object that this Object is wrapping or nil if it
+// isn't wrapping anything
+func (o testWrapper) UnWrap() fs.Object {
+	if o, ok := o.ObjectInfo.(fs.Object); ok {
+		return o
+	}
+	return nil
+}
+
+// Create a temporary local fs to upload things from
+
+func makeTempLocalFs(t *testing.T) (localFs fs.Fs, cleanup func()) {
+	localFs, err := fs.TemporaryLocalFs()
+	require.NoError(t, err)
+	cleanup = func() {
+		require.NoError(t, localFs.Rmdir(context.Background(), ""))
+	}
+	return localFs, cleanup
+}
+
+// Upload a file to a remote
+func uploadFile(t *testing.T, f fs.Fs, remote, contents string) (obj fs.Object, cleanup func()) {
+	inBuf := bytes.NewBufferString(contents)
+	t1 := time.Date(2012, time.December, 17, 18, 32, 31, 0, time.UTC)
+	upSrc := object.NewStaticObjectInfo(remote, t1, int64(len(contents)), true, nil, nil)
+	obj, err := f.Put(context.Background(), inBuf, upSrc)
+	require.NoError(t, err)
+	cleanup = func() {
+		require.NoError(t, obj.Remove(context.Background()))
+	}
+	return obj, cleanup
+}
+
+// Test the ObjectInfo
+func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
+	var (
+		contents = random.String(100)
+		path     = "hash_test_object"
+		ctx      = context.Background()
+	)
+	if wrap {
+		path = "_wrap"
+	}
+
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
+
+	obj, cleanupObj := uploadFile(t, localFs, path, contents)
+	defer cleanupObj()
+
+	// encrypt the data
+	inBuf := bytes.NewBufferString(contents)
+	var outBuf bytes.Buffer
+	enc, err := f.cipher.newEncrypter(inBuf, nil)
+	require.NoError(t, err)
+	nonce := enc.nonce // read the nonce at the start
+	_, err = io.Copy(&outBuf, enc)
+	require.NoError(t, err)
+
+	var oi fs.ObjectInfo = obj
+	if wrap {
+		// wrap the object in an fs.ObjectUnwrapper if required
+		oi = testWrapper{oi}
+	}
+
+	// wrap the object in a crypt for upload using the nonce we
+	// saved from the encryptor
+	src := f.newObjectInfo(oi, nonce)
+
+	// Test ObjectInfo methods
+	assert.Equal(t, int64(outBuf.Len()), src.Size())
+	assert.Equal(t, f, src.Fs())
+	assert.NotEqual(t, path, src.Remote())
+
+	// Test ObjectInfo.Hash
+	wantHash := md5.Sum(outBuf.Bytes())
+	gotHash, err := src.Hash(ctx, hash.MD5)
+	require.NoError(t, err)
+	assert.Equal(t, fmt.Sprintf("%x", wantHash), gotHash)
+}
+
+func testComputeHash(t *testing.T, f *Fs) {
+	var (
+		contents = random.String(100)
+		path     = "compute_hash_test"
+		ctx      = context.Background()
+		hashType = f.Fs.Hashes().GetOne()
+	)
+
+	if hashType == hash.None {
+		t.Skipf("%v: does not support hashes", f.Fs)
+	}
+
+	localFs, cleanupLocalFs := makeTempLocalFs(t)
+	defer cleanupLocalFs()
+
+	// Upload a file to localFs as a test object
+	localObj, cleanupLocalObj := uploadFile(t, localFs, path, contents)
+	defer cleanupLocalObj()
+
+	// Upload the same data to the remote Fs also
+	remoteObj, cleanupRemoteObj := uploadFile(t, f, path, contents)
+	defer cleanupRemoteObj()
+
+	// Calculate the expected Hash of the remote object
+	computedHash, err := f.ComputeHash(ctx, remoteObj.(*Object), localObj, hashType)
+	require.NoError(t, err)
+
+	// Test computed hash matches remote object hash
+	remoteObjHash, err := remoteObj.(*Object).Object.Hash(ctx, hashType)
+	require.NoError(t, err)
+	assert.Equal(t, remoteObjHash, computedHash)
+}
+
+// InternalTest is called by fstests.Run to extra tests
+func (f *Fs) InternalTest(t *testing.T) {
+	t.Run("ObjectInfo", func(t *testing.T) { testObjectInfo(t, f, false) })
+	t.Run("ObjectInfoWrap", func(t *testing.T) { testObjectInfo(t, f, true) })
+	t.Run("ComputeHash", func(t *testing.T) { testComputeHash(t, f) })
+}

backend/drive/drive_internal_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/pkg/errors"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fstest/fstests"
 	"github.com/stretchr/testify/assert"
@@ -268,6 +269,98 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 	}
 }
 
+// TestIntegration/FsMkdir/FsPutFiles/Internal/Shortcuts
+func (f *Fs) InternalTestShortcuts(t *testing.T) {
+	const (
+		// from fstest/fstests/fstests.go
+		existingDir    = "hello? sausage"
+		existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
+		existingSubDir = "êé"
+	)
+	ctx := context.Background()
+	srcObj, err := f.NewObject(ctx, existingFile)
+	require.NoError(t, err)
+	srcHash, err := srcObj.Hash(ctx, hash.MD5)
+	require.NoError(t, err)
+	assert.NotEqual(t, "", srcHash)
+	t.Run("Errors", func(t *testing.T) {
+		_, err := f.makeShortcut(ctx, "", f, "")
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "can't be root")
+
+		_, err = f.makeShortcut(ctx, "notfound", f, "dst")
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "can't find source")
+
+		_, err = f.makeShortcut(ctx, existingFile, f, existingFile)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "not overwriting")
+		assert.Contains(t, err.Error(), "existing file")
+
+		_, err = f.makeShortcut(ctx, existingFile, f, existingDir)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "not overwriting")
+		assert.Contains(t, err.Error(), "existing directory")
+	})
+	t.Run("File", func(t *testing.T) {
+		dstObj, err := f.makeShortcut(ctx, existingFile, f, "shortcut.txt")
+		require.NoError(t, err)
+		require.NotNil(t, dstObj)
+		assert.Equal(t, "shortcut.txt", dstObj.Remote())
+		dstHash, err := dstObj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, srcHash, dstHash)
+		require.NoError(t, dstObj.Remove(ctx))
+	})
+	t.Run("Dir", func(t *testing.T) {
+		dstObj, err := f.makeShortcut(ctx, existingDir, f, "shortcutdir")
+		require.NoError(t, err)
+		require.Nil(t, dstObj)
+		entries, err := f.List(ctx, "shortcutdir")
+		require.NoError(t, err)
+		require.Equal(t, 1, len(entries))
+		require.Equal(t, "shortcutdir/"+existingSubDir, entries[0].Remote())
+		require.NoError(t, f.Rmdir(ctx, "shortcutdir"))
+	})
+	t.Run("Command", func(t *testing.T) {
+		_, err := f.Command(ctx, "shortcut", []string{"one"}, nil)
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "need exactly 2 arguments")
+
+		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
+			"target": "doesnotexistremote:",
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "couldn't find target")
+
+		_, err = f.Command(ctx, "shortcut", []string{"one", "two"}, map[string]string{
+			"target": ".",
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "target is not a drive backend")
+
+		dstObjI, err := f.Command(ctx, "shortcut", []string{existingFile, "shortcut2.txt"}, map[string]string{
+			"target": fs.ConfigString(f),
+		})
+		require.NoError(t, err)
+		dstObj := dstObjI.(*Object)
+		assert.Equal(t, "shortcut2.txt", dstObj.Remote())
+		dstHash, err := dstObj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, srcHash, dstHash)
+		require.NoError(t, dstObj.Remove(ctx))
+
+		dstObjI, err = f.Command(ctx, "shortcut", []string{existingFile, "shortcut3.txt"}, nil)
+		require.NoError(t, err)
+		dstObj = dstObjI.(*Object)
+		assert.Equal(t, "shortcut3.txt", dstObj.Remote())
+		dstHash, err = dstObj.Hash(ctx, hash.MD5)
+		require.NoError(t, err)
+		assert.Equal(t, srcHash, dstHash)
+		require.NoError(t, dstObj.Remove(ctx))
+	})
+}
+
 func (f *Fs) InternalTest(t *testing.T) {
 	// These tests all depend on each other so run them as nested tests
 	t.Run("DocumentImport", func(t *testing.T) {
@@ -282,6 +375,7 @@ func (f *Fs) InternalTest(t *testing.T) {
 			})
 		})
 	})
+	t.Run("Shortcuts", f.InternalTestShortcuts)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/dropbox/dropbox.go

@@ -117,7 +117,10 @@ func init() {
 		Description: "Dropbox",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.ConfigNoOffline("dropbox", name, m, dropboxConfig)
+			opt := oauthutil.Options{
+				NoOffline: true,
+			}
+			err := oauthutil.Config("dropbox", name, m, dropboxConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -225,7 +228,11 @@ func shouldRetry(err error) (bool, error) {
 		return false, err
 	}
 	baseErrString := errors.Cause(err).Error()
-	// handle any official Retry-After header from Dropbox's SDK first
+	// First check for Insufficient Space
+	if strings.Contains(baseErrString, "insufficient_space") {
+		return false, fserrors.FatalError(err)
+	}
+	// Then handle any official Retry-After header from Dropbox's SDK
 	switch e := err.(type) {
 	case auth.RateLimitAPIError:
 		if e.RateLimitError.RetryAfter > 0 {
@@ -775,11 +782,17 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	absPath := f.opt.Enc.FromStandardPath(path.Join(f.slashRoot, remote))
 	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
 	createArg := sharing.CreateSharedLinkWithSettingsArg{
 		Path: absPath,
+		// FIXME this gives settings_error/not_authorized/.. errors
+		// and the expires setting isn't in the documentation so remove
+		// for now.
+		// Settings: &sharing.SharedLinkSettings{
+		// 	Expires: time.Now().Add(time.Duration(expire)).UTC().Round(time.Second),
+		// },
 	}
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {

backend/fichier/api.go

@@ -17,6 +17,7 @@ import (
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	429, // Too Many Requests.
+	403, // Forbidden (may happen when request limit is exceeded)
 	500, // Internal Server Error
 	502, // Bad Gateway
 	503, // Service Unavailable
@@ -147,11 +148,6 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
 }
 
 func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
-
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -320,7 +316,7 @@ func (f *Fs) getUploadNode(ctx context.Context) (response *GetUploadNodeResponse
 	return response, err
 }
 
-func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string) (response *http.Response, err error) {
+func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName, folderID, uploadID, node string, options ...fs.OpenOption) (response *http.Response, err error) {
 	// fs.Debugf(f, "Uploading File `%s`", fileName)
 
 	fileName = f.opt.Enc.FromStandardName(fileName)
@@ -338,6 +334,7 @@ func (f *Fs) uploadFile(ctx context.Context, in io.Reader, size int64, fileName,
 		NoResponse:           true,
 		Body:                 in,
 		ContentLength:        &size,
+		Options:              options,
 		MultipartContentName: "file[]",
 		MultipartFileName:    fileName,
 		MultipartParams: map[string][]string{

backend/fichier/fichier.go

@@ -23,11 +23,12 @@ import (
 )
 
 const (
-	rootID        = "0"
-	apiBaseURL    = "https://api.1fichier.com/v1"
-	minSleep      = 334 * time.Millisecond // 3 API calls per second is recommended
-	maxSleep      = 5 * time.Second
-	decayConstant = 2 // bigger for slower decay, exponential
+	rootID         = "0"
+	apiBaseURL     = "https://api.1fichier.com/v1"
+	minSleep       = 400 * time.Millisecond // api is extremely rate limited now
+	maxSleep       = 5 * time.Second
+	decayConstant  = 2 // bigger for slower decay, exponential
+	attackConstant = 0 // start with max sleep
 )
 
 func init() {
@@ -185,7 +186,7 @@ func NewFs(name string, root string, config configmap.Mapper) (fs.Fs, error) {
 		name:       name,
 		root:       root,
 		opt:        *opt,
-		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		pacer:      fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant), pacer.AttackConstant(attackConstant))),
 		baseClient: &http.Client{},
 	}
 
@@ -263,7 +264,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -297,7 +298,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Put in to the remote path with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Put should either
 // return an error or upload it properly (rather than e.g. calling panic).
 //
@@ -333,12 +334,12 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		return nil, err
 	}
 
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
 
-	_, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL)
+	_, err = f.uploadFile(ctx, in, size, leaf, directoryID, nodeResponse.ID, nodeResponse.URL, options...)
 	if err != nil {
 		return nil, err
 	}
@@ -388,13 +389,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 //
 // Shouldn't return an error if it already exists
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -402,11 +397,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 //
 // Return an error if it doesn't exist or isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return err

backend/fichier/object.go

@@ -101,7 +101,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/ftp/ftp.go

@@ -50,8 +50,19 @@ func init() {
 			IsPassword: true,
 			Required:   true,
 		}, {
-			Name:    "tls",
-			Help:    "Use FTP over TLS (Implicit)",
+			Name: "tls",
+			Help: `Use FTPS over TLS (Implicit)
+When using implicit FTP over TLS the client will connect using TLS
+right from the start, which in turn breaks the compatibility with
+non-TLS-aware servers. This is usually served over port 990 rather
+than port 21. Cannot be used in combination with explicit FTP.`,
+			Default: false,
+		}, {
+			Name: "explicit_tls",
+			Help: `Use FTP over TLS (Explicit)
+When using explicit FTP over TLS the client explicitly request
+security from the server in order to upgrade a plain text connection
+to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Default: false,
 		}, {
 			Name:     "concurrency",
@@ -72,7 +83,7 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			// The FTP protocal can't handle trailing spaces (for instance
+			// The FTP protocol can't handle trailing spaces (for instance
 			// pureftpd turns them into _)
 			//
 			// proftpd can't handle '*' in file names
@@ -90,6 +101,7 @@ type Options struct {
 	Pass              string               `config:"pass"`
 	Port              string               `config:"port"`
 	TLS               bool                 `config:"tls"`
+	ExplicitTLS       bool                 `config:"explicit_tls"`
 	Concurrency       int                  `config:"concurrency"`
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
@@ -152,12 +164,21 @@ func (f *Fs) Features() *fs.Features {
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
 	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
-	if f.opt.TLS {
+	if f.opt.TLS && f.opt.ExplicitTLS {
+		fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+	} else if f.opt.TLS {
 		tlsConfig := &tls.Config{
 			ServerName:         f.opt.Host,
 			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
 		}
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
+	} else if f.opt.ExplicitTLS {
+		tlsConfig := &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
+		}
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
@@ -799,10 +820,12 @@ func (f *ftpReadCloser) Close() error {
 		f.f.putFtpConnection(&f.c, nil)
 	}
 	// mask the error if it was caused by a premature close
+	// NB StatusAboutToSend is to work around a bug in pureftpd
+	// See: https://github.com/rclone/rclone/issues/3445#issuecomment-521654257
 	switch errX := err.(type) {
 	case *textproto.Error:
 		switch errX.Code {
-		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable:
+		case ftp.StatusTransfertAborted, ftp.StatusFileUnavailable, ftp.StatusAboutToSend:
 			err = nil
 		}
 	}

backend/googlecloudstorage/googlecloudstorage.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"os"
 	"path"
 	"strings"
 	"time"
@@ -38,6 +37,7 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"golang.org/x/oauth2"
@@ -82,7 +82,7 @@ func init() {
 			if saFile != "" || saCreds != "" {
 				return
 			}
-			err := oauthutil.Config("google cloud storage", name, m, storageConfig)
+			err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -98,7 +98,7 @@ func init() {
 			Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",
 		}, {
 			Name: "service_account_file",
-			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
 			Name: "service_account_credentials",
 			Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
@@ -242,6 +242,9 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			}, {
 				Value: "COLDLINE",
 				Help:  "Coldline storage class",
+			}, {
+				Value: "ARCHIVE",
+				Help:  "Archive storage class",
 			}, {
 				Value: "DURABLE_REDUCED_AVAILABILITY",
 				Help:  "Durable reduced availability storage class",
@@ -402,7 +405,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, errors.Wrap(err, "error opening service account credentials file")
 		}
@@ -555,7 +558,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				continue
 			}
 			remote = remote[len(prefix):]
-			isDirectory := strings.HasSuffix(remote, "/")
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
@@ -1063,6 +1066,33 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentType: fs.MimeType(ctx, src),
 		Metadata:    metadataFromModTime(modTime),
 	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			object.CacheControl = value
+		case "content-disposition":
+			object.ContentDisposition = value
+		case "content-encoding":
+			object.ContentEncoding = value
+		case "content-language":
+			object.ContentLanguage = value
+		case "content-type":
+			object.ContentType = value
+		default:
+			const googMetaPrefix = "x-goog-meta-"
+			if strings.HasPrefix(lowerKey, googMetaPrefix) {
+				metaKey := lowerKey[len(googMetaPrefix):]
+				object.Metadata[metaKey] = value
+			} else {
+				fs.Errorf(o, "Don't know how to set key %q on upload", key)
+			}
+		}
+	}
 	var newObject *storage.Object
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 		insertObject := o.fs.svc.Objects.Insert(bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name)

backend/googlephotos/albums_test.go

@@ -30,7 +30,7 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a1)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
+		"one": {a1},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1": a1,
@@ -39,7 +39,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"one": a1,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one"},
+		"": {"one"},
 	}, albums.path)
 
 	a2 := &api.Album{
@@ -49,8 +49,8 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a2)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
-		"two": []*api.Album{a2},
+		"one": {a1},
+		"two": {a2},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1": a1,
@@ -61,7 +61,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"two": a2,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one", "two"},
+		"": {"one", "two"},
 	}, albums.path)
 
 	// Add a duplicate
@@ -72,8 +72,8 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a2a)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
-		"two": []*api.Album{a2, a2a},
+		"one": {a1},
+		"two": {a2, a2a},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":  a1,
@@ -86,7 +86,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one", "two {2}", "two {2a}"},
+		"": {"one", "two {2}", "two {2a}"},
 	}, albums.path)
 
 	// Add a sub directory
@@ -97,9 +97,9 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":    a1,
@@ -114,8 +114,8 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	// Add a weird path
@@ -126,10 +126,10 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a0)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"{0}":     []*api.Album{a0},
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"{0}":     {a0},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"0":    a0,
@@ -146,8 +146,8 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}", "{0}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}", "{0}"},
+		"one": {"sub"},
 	}, albums.path)
 }
 
@@ -181,9 +181,9 @@ func TestAlbumsDel(t *testing.T) {
 	albums.add(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":    a1,
@@ -198,16 +198,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a1)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"2":    a2,
@@ -220,16 +220,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a2)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"2a":   a2a,
@@ -240,16 +240,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a2a)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1sub": a1sub,
@@ -258,16 +258,16 @@ func TestAlbumsDel(t *testing.T) {
 		"one/sub": a1sub,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one"},
-		"one": []string{"sub"},
+		"":    {"one"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{}, albums.byID)
 	assert.Equal(t, map[string]*api.Album{}, albums.byTitle)

backend/googlephotos/api/types.go

@@ -17,7 +17,7 @@ type Error struct {
 	Details ErrorDetails `json:"error"`
 }
 
-// Error statisfies error interface
+// Error satisfies error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Details.Message, e.Details.Code, e.Details.Status)
 }

backend/googlephotos/googlephotos.go

@@ -96,7 +96,7 @@ func init() {
 			}
 
 			// Do the oauth
-			err = oauthutil.Config("google photos", name, m, oauthConfig)
+			err = oauthutil.Config("google photos", name, m, oauthConfig, nil)
 			if err != nil {
 				golog.Fatalf("Failed to configure token: %v", err)
 			}
@@ -134,14 +134,20 @@ rclone mount needs to know the size of files in advance of reading
 them, so setting this flag when using rclone mount is recommended if
 you want to read the media.`,
 			Advanced: true,
+		}, {
+			Name:     "start_year",
+			Default:  2000,
+			Help:     `Year limits the photos to be downloaded to those which are uploaded after the given year`,
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ReadOnly bool `config:"read_only"`
-	ReadSize bool `config:"read_size"`
+	ReadOnly  bool `config:"read_only"`
+	ReadSize  bool `config:"read_size"`
+	StartYear int  `config:"start_year"`
 }
 
 // Fs represents a remote storage server
@@ -202,6 +208,11 @@ func (f *Fs) dirTime() time.Time {
 	return f.startTime
 }
 
+// startYear returns the start year
+func (f *Fs) startYear() int {
+	return f.opt.StartYear
+}
+
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	429, // Too Many Requests.
@@ -224,6 +235,10 @@ func errorHandler(resp *http.Response) error {
 	if err != nil {
 		body = nil
 	}
+	// Google sends 404 messages as images so be prepared for that
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "image/") {
+		body = []byte("Image not found or broken")
+	}
 	var e = api.Error{
 		Details: api.ErrorDetails{
 			Code:    resp.StatusCode,
@@ -943,8 +958,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Upload the media item in exchange for an UploadToken
 	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/uploads",
+		Method:  "POST",
+		Path:    "/uploads",
+		Options: options,
 		ExtraHeaders: map[string]string{
 			"X-Goog-Upload-File-Name": fileName,
 			"X-Goog-Upload-Protocol":  "raw",
@@ -1003,7 +1019,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Add upload to internal storage
 	if pattern.isUpload {
+		o.fs.uploadedMu.Lock()
 		o.fs.uploaded.AddEntry(o)
+		o.fs.uploadedMu.Unlock()
 	}
 	return nil
 }

backend/googlephotos/pattern.go

@@ -23,6 +23,7 @@ type lister interface {
 	listAlbums(ctx context.Context, shared bool) (all *albums, err error)
 	listUploads(ctx context.Context, dir string) (entries fs.DirEntries, err error)
 	dirTime() time.Time
+	startYear() int
 }
 
 // dirPattern describes a single directory pattern
@@ -52,6 +53,7 @@ var patterns = dirPatterns{
 				fs.NewDir(prefix+"album", f.dirTime()),
 				fs.NewDir(prefix+"shared-album", f.dirTime()),
 				fs.NewDir(prefix+"upload", f.dirTime()),
+				fs.NewDir(prefix+"feature", f.dirTime()),
 			}, nil
 		},
 	},
@@ -189,6 +191,28 @@ var patterns = dirPatterns{
 		re:     `^shared-album/(.+?)/([^/]+)$`,
 		isFile: true,
 	},
+	{
+		re: `^feature$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			return fs.DirEntries{
+				fs.NewDir(prefix+"favorites", f.dirTime()),
+			}, nil
+		},
+	},
+	{
+		re: `^feature/favorites$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			filter := featureFilter(ctx, f, match)
+			if err != nil {
+				return nil, err
+			}
+			return f.listDir(ctx, prefix, filter)
+		},
+	},
+	{
+		re:     `^feature/favorites/([^/]+)$`,
+		isFile: true,
+	},
 }.mustCompile()
 
 // mustCompile compiles the regexps in the dirPatterns
@@ -200,7 +224,7 @@ func (ds dirPatterns) mustCompile() dirPatterns {
 	return ds
 }
 
-// match finds the path passed in in the matching structure and
+// match finds the path passed in the matching structure and
 // returns the parameters and a pointer to the match, or nil.
 func (ds dirPatterns) match(root string, itemPath string, isFile bool) (match []string, prefix string, pattern *dirPattern) {
 	itemPath = strings.Trim(itemPath, "/")
@@ -222,11 +246,10 @@ func (ds dirPatterns) match(root string, itemPath string, isFile bool) (match []
 	return nil, "", nil
 }
 
-// Return the years from 2000 to today
-// FIXME make configurable?
+// Return the years from startYear to today
 func years(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
 	currentYear := f.dirTime().Year()
-	for year := 2000; year <= currentYear; year++ {
+	for year := f.startYear(); year <= currentYear; year++ {
 		entries = append(entries, fs.NewDir(prefix+fmt.Sprint(year), f.dirTime()))
 	}
 	return entries, nil
@@ -290,6 +313,24 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 	return sf, nil
 }
 
+// featureFilter creates a filter for the Feature enum
+//
+// The API only supports one feature, FAVORITES, so hardcode that feature
+//
+// https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
+func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {
+	sf = api.SearchFilter{
+		Filters: &api.Filters{
+			FeatureFilter: &api.FeatureFilter{
+				IncludedFeatures: []string{
+					"FAVORITES",
+				},
+			},
+		},
+	}
+	return sf
+}
+
 // Turns an albumPath into entries
 //
 // These can either be synthetic directory entries if the album path

backend/googlephotos/pattern_test.go

@@ -59,6 +59,11 @@ func (f *testLister) dirTime() time.Time {
 	return startTime
 }
 
+// mock startYear for testing
+func (f *testLister) startYear() int {
+	return 2000
+}
+
 func TestPatternMatch(t *testing.T) {
 	for testNumber, test := range []struct {
 		// input
@@ -150,6 +155,38 @@ func TestPatternMatch(t *testing.T) {
 			wantPrefix:  "file.jpg/",
 			wantPattern: &patterns[5],
 		},
+		{
+			root:        "",
+			itemPath:    "feature",
+			isFile:      false,
+			wantMatch:   []string{"feature"},
+			wantPrefix:  "feature/",
+			wantPattern: &patterns[23],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature",
+			itemPath:    "favorites",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "favorites/",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "file.jpg",
+			isFile:      true,
+			wantMatch:   []string{"feature/favorites/file.jpg", "file.jpg"},
+			wantPrefix:  "file.jpg/",
+			wantPattern: &patterns[25],
+		},
 	} {
 		t.Run(fmt.Sprintf("#%d,root=%q,itemPath=%q,isFile=%v", testNumber, test.root, test.itemPath, test.isFile), func(t *testing.T) {
 			gotMatch, gotPrefix, gotPattern := patterns.match(test.root, test.itemPath, test.isFile)

backend/http/http_internal_test.go

@@ -166,8 +166,7 @@ func TestNewObject(t *testing.T) {
 	require.NoError(t, err)
 	tFile := fi.ModTime()
 
-	dt, ok := fstest.CheckTimeEqualWithPrecision(tObj, tFile, time.Second)
-	assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", o.Remote(), dt, time.Second, tObj, tFile, time.Second))
+	fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second)
 
 	// check object not found
 	o, err = f.NewObject(context.Background(), "not found.txt")

backend/hubic/auth.go

@@ -21,7 +21,7 @@ func newAuth(f *Fs) *auth {
 	}
 }
 
-// Request constructs a http.Request for authentication
+// Request constructs an http.Request for authentication
 //
 // returns nil for not needed
 func (a *auth) Request(*swift.Connection) (r *http.Request, err error) {

backend/hubic/hubic.go

@@ -39,7 +39,7 @@ var (
 	// Description of how to auth for this app
 	oauthConfig = &oauth2.Config{
 		Scopes: []string{
-			"credentials.r", // Read Openstack credentials
+			"credentials.r", // Read OpenStack credentials
 		},
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  "https://api.hubic.com/oauth/auth/",
@@ -58,7 +58,7 @@ func init() {
 		Description: "Hubic",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("hubic", name, m, oauthConfig)
+			err := oauthutil.Config("hubic", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -76,8 +76,8 @@ func init() {
 // credentials is the JSON returned from the Hubic API to read the
 // OpenStack credentials
 type credentials struct {
-	Token    string `json:"token"`    // Openstack token
-	Endpoint string `json:"endpoint"` // Openstack endpoint
+	Token    string `json:"token"`    // OpenStack token
+	Endpoint string `json:"endpoint"` // OpenStack endpoint
 	Expires  string `json:"expires"`  // Expires date - eg "2015-11-09T14:24:56+01:00"
 }
 

backend/jottacloud/api/types.go

@@ -164,6 +164,12 @@ type CustomerInfo struct {
 	IOSHash           string      `json:"ios_hash"`
 }
 
+// TrashResponse is returned when emptying the Trash
+type TrashResponse struct {
+	Folders int64 `json:"folders"`
+	Files   int64 `json:"files"`
+}
+
 // XML structures returned by the old API
 
 // Flag is a hacky type for checking if an attribute is present

backend/jottacloud/jottacloud.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -26,6 +27,7 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -39,26 +41,33 @@ import (
 
 // Globals
 const (
-	minSleep          = 10 * time.Millisecond
-	maxSleep          = 2 * time.Second
-	decayConstant     = 2 // bigger for slower decay, exponential
-	defaultDevice     = "Jotta"
-	defaultMountpoint = "Archive"
-	rootURL           = "https://www.jottacloud.com/jfs/"
-	apiURL            = "https://api.jottacloud.com/"
-	baseURL           = "https://www.jottacloud.com/"
-	defaultTokenURL   = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
-	cachePrefix       = "rclone-jcmd5-"
-	configDevice      = "device"
-	configMountpoint  = "mountpoint"
-	configTokenURL    = "tokenURL"
-	configVersion     = 1
+	minSleep           = 10 * time.Millisecond
+	maxSleep           = 2 * time.Second
+	decayConstant      = 2 // bigger for slower decay, exponential
+	defaultDevice      = "Jotta"
+	defaultMountpoint  = "Archive"
+	rootURL            = "https://jfs.jottacloud.com/jfs/"
+	apiURL             = "https://api.jottacloud.com/"
+	baseURL            = "https://www.jottacloud.com/"
+	defaultTokenURL    = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
+	cachePrefix        = "rclone-jcmd5-"
+	configDevice       = "device"
+	configMountpoint   = "mountpoint"
+	configTokenURL     = "tokenURL"
+	configClientID     = "client_id"
+	configClientSecret = "client_secret"
+	configVersion      = 1
+
+	v1tokenURL              = "https://api.jottacloud.com/auth/v1/token"
+	v1registerURL           = "https://api.jottacloud.com/auth/v1/register"
+	v1ClientID              = "nibfk8biu12ju7hpqomr8b1e40"
+	v1EncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
+	v1configVersion         = 0
 )
 
 var (
 	// Description of how to auth for this app for a personal account
 	oauthConfig = &oauth2.Config{
-		ClientID: "jottacli",
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  defaultTokenURL,
 			TokenURL: defaultTokenURL,
@@ -72,7 +81,7 @@ func init() {
 	// needs to be done early so we can use oauth during config
 	fs.Register(&fs.RegInfo{
 		Name:        "jottacloud",
-		Description: "JottaCloud",
+		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
 			ctx := context.TODO()
@@ -83,7 +92,7 @@ func init() {
 				if err != nil {
 					log.Fatalf("Failed to parse config version - corrupted config")
 				}
-				refresh = ver != configVersion
+				refresh = (ver != configVersion) && (ver != v1configVersion)
 			}
 
 			if refresh {
@@ -98,42 +107,12 @@ func init() {
 				}
 			}
 
-			clientConfig := *fs.Config
-			clientConfig.UserAgent = "JottaCli 0.6.18626 windows-amd64"
-			srv := rest.NewClient(fshttp.NewClient(&clientConfig))
-
-			fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
-			fmt.Printf("Login Token> ")
-			loginToken := config.ReadLine()
-
-			token, err := doAuth(ctx, srv, loginToken, m)
-			if err != nil {
-				log.Fatalf("Failed to get oauth token: %s", err)
-			}
-			err = oauthutil.PutToken(name, m, &token, true)
-			if err != nil {
-				log.Fatalf("Error while saving token: %s", err)
-			}
-
-			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+			fmt.Printf("Use legacy authentification?.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n")
 			if config.Confirm(false) {
-				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
-				if err != nil {
-					log.Fatalf("Failed to load oAuthClient: %s", err)
-				}
-
-				srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
-				apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
-
-				device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
-				if err != nil {
-					log.Fatalf("Failed to setup mountpoint: %s", err)
-				}
-				m.Set(configDevice, device)
-				m.Set(configMountpoint, mountpoint)
+				v1config(ctx, name, m)
+			} else {
+				v2config(ctx, name, m)
 			}
-
-			m.Set("configVersion", strconv.Itoa(configVersion))
 		},
 		Options: []fs.Option{{
 			Name:     "md5_memory_limit",
@@ -141,13 +120,13 @@ func init() {
 			Default:  fs.SizeSuffix(10 * 1024 * 1024),
 			Advanced: true,
 		}, {
-			Name:     "hard_delete",
-			Help:     "Delete files permanently rather than putting them into the trash.",
+			Name:     "trashed_only",
+			Help:     "Only show files that are in the trash.\nThis will show trashed files in their original directory structure.",
 			Default:  false,
 			Advanced: true,
 		}, {
-			Name:     "unlink",
-			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
+			Name:     "hard_delete",
+			Help:     "Delete files permanently rather than putting them into the trash.",
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -174,8 +153,8 @@ type Options struct {
 	Device             string               `config:"device"`
 	Mountpoint         string               `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix        `config:"md5_memory_limit"`
+	TrashedOnly        bool                 `config:"trashed_only"`
 	HardDelete         bool                 `config:"hard_delete"`
-	Unlink             bool                 `config:"unlink"`
 	UploadThreshold    fs.SizeSuffix        `config:"upload_resume_limit"`
 	Enc                encoder.MultiEncoder `config:"encoding"`
 }
@@ -229,7 +208,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an box 'url'
+// parsePath parses a box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -251,8 +230,181 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// doAuth runs the actual token request
-func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
+// v1config configure a jottacloud backend using legacy authentification
+func v1config(ctx context.Context, name string, m configmap.Mapper) {
+	srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+	fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
+	if config.Confirm(false) {
+		deviceRegistration, err := registerDevice(ctx, srv)
+		if err != nil {
+			log.Fatalf("Failed to register device: %v", err)
+		}
+
+		m.Set(configClientID, deviceRegistration.ClientID)
+		m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
+		fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+	}
+
+	clientID, ok := m.Get(configClientID)
+	if !ok {
+		clientID = v1ClientID
+	}
+	clientSecret, ok := m.Get(configClientSecret)
+	if !ok {
+		clientSecret = v1EncryptedClientSecret
+	}
+	oauthConfig.ClientID = clientID
+	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
+	oauthConfig.Endpoint.AuthURL = v1tokenURL
+	oauthConfig.Endpoint.TokenURL = v1tokenURL
+
+	fmt.Printf("Username> ")
+	username := config.ReadLine()
+	password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
+
+	token, err := doAuthV1(ctx, srv, username, password)
+	if err != nil {
+		log.Fatalf("Failed to get oauth token: %s", err)
+	}
+	err = oauthutil.PutToken(name, m, &token, true)
+	if err != nil {
+		log.Fatalf("Error while saving token: %s", err)
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(v1configVersion))
+}
+
+// registerDevice register a new device for use with the jottacloud API
+func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
+	// random generator to generate random device names
+	seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
+	randonDeviceNamePartLength := 21
+	randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
+	charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	for i := range randomDeviceNamePart {
+		randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
+	}
+	randomDeviceName := "rclone-" + string(randomDeviceNamePart)
+	fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
+
+	values := url.Values{}
+	values.Set("device_id", randomDeviceName)
+
+	opts := rest.Opts{
+		Method:       "POST",
+		RootURL:      v1registerURL,
+		ContentType:  "application/x-www-form-urlencoded",
+		ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
+		Parameters:   values,
+	}
+
+	var deviceRegistration *api.DeviceRegistrationResponse
+	_, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
+	return deviceRegistration, err
+}
+
+// doAuthV1 runs the actual token request for V1 authentification
+func doAuthV1(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
+	// prepare out token request with username and password
+	values := url.Values{}
+	values.Set("grant_type", "PASSWORD")
+	values.Set("password", password)
+	values.Set("username", username)
+	values.Set("client_id", oauthConfig.ClientID)
+	values.Set("client_secret", oauthConfig.ClientSecret)
+	opts := rest.Opts{
+		Method:      "POST",
+		RootURL:     oauthConfig.Endpoint.AuthURL,
+		ContentType: "application/x-www-form-urlencoded",
+		Parameters:  values,
+	}
+
+	// do the first request
+	var jsonToken api.TokenJSON
+	resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
+	if err != nil {
+		// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
+		if resp != nil {
+			if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
+				fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
+				fmt.Printf("Enter verification code> ")
+				authCode := config.ReadLine()
+
+				authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
+				opts.ExtraHeaders = make(map[string]string)
+				opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
+				resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
+			}
+		}
+	}
+
+	token.AccessToken = jsonToken.AccessToken
+	token.RefreshToken = jsonToken.RefreshToken
+	token.TokenType = jsonToken.TokenType
+	token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
+	return token, err
+}
+
+// v2config configure a jottacloud backend using the modern JottaCli token based authentification
+func v2config(ctx context.Context, name string, m configmap.Mapper) {
+	srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+	fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
+	fmt.Printf("Login Token> ")
+	loginToken := config.ReadLine()
+
+	token, err := doAuthV2(ctx, srv, loginToken, m)
+	if err != nil {
+		log.Fatalf("Failed to get oauth token: %s", err)
+	}
+	err = oauthutil.PutToken(name, m, &token, true)
+	if err != nil {
+		log.Fatalf("Error while saving token: %s", err)
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oauthConfig.ClientID = "jottacli"
+		oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(configVersion))
+}
+
+// doAuthV2 runs the actual token request for V2 authentification
+func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
 	loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
 	if err != nil {
 		return token, err
@@ -448,7 +600,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// Jottacloud want's '+' to be URL encoded even though the RFC states it's not reserved
+// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
 func urlPathEscape(in string) string {
 	return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
 }
@@ -458,11 +610,34 @@ func (f *Fs) filePathRaw(file string) string {
 	return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
-// filePath returns a escaped file path (f.root, file)
+// filePath returns an escaped file path (f.root, file)
 func (f *Fs) filePath(file string) string {
 	return urlPathEscape(f.filePathRaw(file))
 }
 
+// Jottacloud requires the grant_type 'refresh_token' string
+// to be uppercase and throws a 400 Bad Request if we use the
+// lower case used by the oauth2 module
+//
+// This filter catches all refresh requests, reads the body,
+// changes the case and then sends it on
+func grantTypeFilter(req *http.Request) {
+	if v1tokenURL == req.URL.String() {
+		// read the entire body
+		refreshBody, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			return
+		}
+		_ = req.Body.Close()
+
+		// make the refresh token upper case
+		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
+
+		// set the new ReadCloser (with a dummy Close())
+		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+	}
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	ctx := context.TODO()
@@ -474,28 +649,55 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Check config version
-	var ok bool
-	var version string
-	if version, ok = m.Get("configVersion"); ok {
-		ver, err := strconv.Atoi(version)
+	var ver int
+	version, ok := m.Get("configVersion")
+	if ok {
+		ver, err = strconv.Atoi(version)
 		if err != nil {
 			return nil, errors.New("Failed to parse config version")
 		}
-		ok = ver == configVersion
+		ok = (ver == configVersion) || (ver == v1configVersion)
 	}
 	if !ok {
 		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
 
-	// if custome endpoints are set use them else stick with defaults
-	if tokenURL, ok := m.Get(configTokenURL); ok {
-		oauthConfig.Endpoint.TokenURL = tokenURL
-		// jottacloud is weird. we need to use the tokenURL as authURL
-		oauthConfig.Endpoint.AuthURL = tokenURL
+	baseClient := fshttp.NewClient(fs.Config)
+
+	if ver == configVersion {
+		oauthConfig.ClientID = "jottacli"
+		// if custom endpoints are set use them else stick with defaults
+		if tokenURL, ok := m.Get(configTokenURL); ok {
+			oauthConfig.Endpoint.TokenURL = tokenURL
+			// jottacloud is weird. we need to use the tokenURL as authURL
+			oauthConfig.Endpoint.AuthURL = tokenURL
+		}
+	} else if ver == v1configVersion {
+		clientID, ok := m.Get(configClientID)
+		if !ok {
+			clientID = v1ClientID
+		}
+		clientSecret, ok := m.Get(configClientSecret)
+		if !ok {
+			clientSecret = v1EncryptedClientSecret
+		}
+		oauthConfig.ClientID = clientID
+		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
+		oauthConfig.Endpoint.TokenURL = v1tokenURL
+		oauthConfig.Endpoint.AuthURL = v1tokenURL
+
+		// add the request filter to fix token refresh
+		if do, ok := baseClient.Transport.(interface {
+			SetRequestFilter(f func(req *http.Request))
+		}); ok {
+			do.SetRequestFilter(grantTypeFilter)
+		} else {
+			fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
+		}
 	}
 
 	// Create OAuth Client
-	baseClient := fshttp.NewClient(fs.Config)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
@@ -519,6 +721,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:           true,
 	}).Fill(f)
 	f.srv.SetErrorHandler(errorHandler)
+	if opt.TrashedOnly { // we cannot support showing Trashed Files when using ListR right now
+		f.features.ListR = nil
+	}
 
 	// Renew the token in the background
 	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
@@ -638,13 +843,13 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		return nil, errors.Wrap(err, "couldn't list files")
 	}
 
-	if result.Deleted {
+	if bool(result.Deleted) && !f.opt.TrashedOnly {
 		return nil, fs.ErrorDirNotFound
 	}
 
 	for i := range result.Folders {
 		item := &result.Folders[i]
-		if item.Deleted {
+		if !f.opt.TrashedOnly && bool(item.Deleted) {
 			continue
 		}
 		remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
@@ -654,8 +859,14 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 
 	for i := range result.Files {
 		item := &result.Files[i]
-		if item.Deleted || item.State != "COMPLETED" {
-			continue
+		if f.opt.TrashedOnly {
+			if !item.Deleted || item.State != "COMPLETED" {
+				continue
+			}
+		} else {
+			if item.Deleted || item.State != "COMPLETED" {
+				continue
+			}
 		}
 		remote := path.Join(dir, f.opt.Enc.ToStandardName(item.Name))
 		o, err := f.newObjectWithInfo(ctx, remote, item)
@@ -876,7 +1087,8 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		retry, _ := shouldRetry(resp, err)
+		return (retry && resp.StatusCode != 500), err
 	})
 	if err != nil {
 		return nil, err
@@ -980,6 +1192,18 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
+	// surprise! jottacloud fucked up dirmove - the api spits out an error but
+	// dir gets moved regardless
+	if apiErr, ok := err.(*api.Error); ok {
+		if apiErr.StatusCode == 500 {
+			_, err := f.NewObject(ctx, dstRemote)
+			if err == fs.ErrorNotAFile {
+				log.Printf("FIXME: ignoring DirMove error - move succeeded anyway\n")
+				return nil
+			}
+			return err
+		}
+	}
 	if err != nil {
 		return errors.Wrap(err, "couldn't move directory")
 	}
@@ -987,14 +1211,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	opts := rest.Opts{
 		Method:     "GET",
 		Path:       f.filePath(remote),
 		Parameters: url.Values{},
 	}
 
-	if f.opt.Unlink {
+	if unlink {
 		opts.Parameters.Set("mode", "disableShare")
 	} else {
 		opts.Parameters.Set("mode", "enableShare")
@@ -1014,12 +1238,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 		}
 	}
 	if err != nil {
-		if f.opt.Unlink {
+		if unlink {
 			return "", errors.Wrap(err, "couldn't remove public link")
 		}
 		return "", errors.Wrap(err, "couldn't create public link")
 	}
-	if f.opt.Unlink {
+	if unlink {
 		if result.PublicSharePath != "" {
 			return "", errors.Errorf("couldn't remove public link - %q", result.PublicSharePath)
 		}
@@ -1049,6 +1273,22 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	return usage, nil
 }
 
+// CleanUp empties the trash
+func (f *Fs) CleanUp(ctx context.Context) error {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "files/v1/purge_trash",
+	}
+
+	var info api.TrashResponse
+	_, err := f.apiSrv.CallJSON(ctx, &opts, nil, &info)
+	if err != nil {
+		return errors.Wrap(err, "couldn't empty trash")
+	}
+
+	return nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
@@ -1074,7 +1314,7 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// filePath returns a escaped file path (f.root, remote)
+// filePath returns an escaped file path (f.root, remote)
 func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
@@ -1122,7 +1362,7 @@ func (o *Object) readMetaData(ctx context.Context, force bool) (err error) {
 	if err != nil {
 		return err
 	}
-	if info.Deleted {
+	if bool(info.Deleted) && !o.fs.opt.TrashedOnly {
 		return fs.ErrorObjectNotFound
 	}
 	return o.setMetaData(info)
@@ -1179,7 +1419,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The cleanup function should be called when out is finished with
 // regardless of whether this function returned an error or not.
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
-	// we need a MD5
+	// we need an MD5
 	md5Hasher := md5.New()
 	// use the teeReader to write to the local file AND calculate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)
@@ -1259,6 +1499,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	opts := rest.Opts{
 		Method:       "POST",
 		Path:         "files/v1/allocate",
+		Options:      options,
 		ExtraHeaders: make(map[string]string),
 	}
 	fileDate := api.Time(src.ModTime(ctx)).APIString()
@@ -1355,6 +1596,7 @@ var (
 	_ fs.ListRer      = (*Fs)(nil)
 	_ fs.PublicLinker = (*Fs)(nil)
 	_ fs.Abouter      = (*Fs)(nil)
+	_ fs.CleanUpper   = (*Fs)(nil)
 	_ fs.Object       = (*Object)(nil)
 	_ fs.MimeTyper    = (*Object)(nil)
 )

backend/koofr/koofr.go

@@ -79,7 +79,7 @@ type Options struct {
 	Enc      encoder.MultiEncoder `config:"encoding"`
 }
 
-// A Fs is a representation of a remote Koofr Fs
+// An Fs is a representation of a remote Koofr Fs
 type Fs struct {
 	name     string
 	mountID  string
@@ -250,7 +250,7 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
 }
 
-// fullPath constructs a full, absolute path from a Fs root relative path,
+// fullPath constructs a full, absolute path from an Fs root relative path,
 func (f *Fs) fullPath(part string) string {
 	return f.opt.Enc.FromStandardPath(path.Join("/", f.root, part))
 }
@@ -421,7 +421,7 @@ func translateErrorsObject(err error) error {
 }
 
 // mkdir creates a directory at the given remote path. Creates ancestors if
-// neccessary
+// necessary
 func (f *Fs) mkdir(fullPath string) error {
 	if fullPath == "/" {
 		return nil
@@ -603,7 +603,7 @@ func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link,
 }
 
 // PublicLink creates a public link to the remote path
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
 	if err != nil {
 		return "", translateErrorsDir(err)

backend/local/local.go

@@ -41,6 +41,7 @@ func init() {
 		Name:        "local",
 		Description: "Local Disk",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name: "nounc",
 			Help: "Disable UNC (long path names) conversion on Windows",
@@ -89,7 +90,24 @@ are being uploaded and aborts with a message which starts "can't copy
 
 However on some file systems this modification time check may fail (eg
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
-check can be disabled with this flag.`,
+check can be disabled with this flag.
+
+If this flag is set, rclone will use its best efforts to transfer a
+file which is being updated. If the file is only having things
+appended to it (eg a log) then rclone will transfer the log file with
+the size it had the first time rclone saw it.
+
+If the file is being modified throughout (not just appended to) then
+the transfer may fail with a hash check failure.
+
+In detail, once the file has had stat() called on it for the first
+time we:
+
+- Only transfer the size that stat gave
+- Only checksum the size that stat gave
+- Don't update the stat info for the file
+
+`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -117,6 +135,16 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_sparse",
+			Help: `Disable sparse files for multi-thread downloads
+
+On Windows platforms rclone will make sparse files when doing
+multi-thread downloads. This avoids long pauses on large files where
+the OS zeros the file. However sparse files may be undesirable as they
+cause disk fragmentation and can be slow to work with.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -138,6 +166,7 @@ type Options struct {
 	OneFileSystem     bool                 `config:"one_file_system"`
 	CaseSensitive     bool                 `config:"case_sensitive"`
 	CaseInsensitive   bool                 `config:"case_insensitive"`
+	NoSparse          bool                 `config:"no_sparse"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -697,6 +726,50 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Supported()
 }
 
+var commandHelp = []fs.CommandHelp{
+	{
+		Name:  "noop",
+		Short: "A null operation for testing backend commands",
+		Long: `This is a test command which has some options
+you can try to change the output.`,
+		Opts: map[string]string{
+			"echo":  "echo the input arguments",
+			"error": "return an error based on option value",
+		},
+	},
+}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (interface{}, error) {
+	switch name {
+	case "noop":
+		if txt, ok := opt["error"]; ok {
+			if txt == "" {
+				txt = "unspecified error"
+			}
+			return nil, errors.New(txt)
+		}
+		if _, ok := opt["echo"]; ok {
+			out := map[string]interface{}{}
+			out["name"] = name
+			out["arg"] = arg
+			out["opt"] = opt
+			return out, nil
+		}
+		return nil, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -723,8 +796,17 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	oldtime := o.modTime
 	oldsize := o.size
 	err := o.lstat()
+	var changed bool
 	if err != nil {
-		return "", errors.Wrap(err, "hash: failed to stat")
+		if os.IsNotExist(errors.Cause(err)) {
+			// If file not found then we assume any accumulated
+			// hashes are OK - this will error on Open
+			changed = true
+		} else {
+			return "", errors.Wrap(err, "hash: failed to stat")
+		}
+	} else {
+		changed = !o.modTime.Equal(oldtime) || oldsize != o.size
 	}
 
 	o.fs.objectHashesMu.Lock()
@@ -732,7 +814,7 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	hashValue, hashFound := o.hashes[r]
 	o.fs.objectHashesMu.Unlock()
 
-	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil || !hashFound {
+	if changed || hashes == nil || !hashFound {
 		var in io.ReadCloser
 
 		if !o.translatedLink {
@@ -744,6 +826,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		} else {
 			in, err = o.openTranslatedLink(0, -1)
 		}
+		// If not checking for updates, only read size given
+		if o.fs.opt.NoCheckUpdated {
+			in = readers.NewLimitedReadCloser(in, o.size)
+		}
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -890,6 +976,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 	}
 
+	// If not checking updated then limit to current size.  This means if
+	// file is being extended, readers will read a o.Size() bytes rather
+	// than the new size making for a consistent upload.
+	if limit < 0 && o.fs.opt.NoCheckUpdated {
+		limit = o.size
+	}
+
 	// Handle a translated link
 	if o.translatedLink {
 		return o.openTranslatedLink(offset, limit)
@@ -977,7 +1070,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			}
 		}
 		// Pre-allocate the file for performance reasons
-		err = preAllocate(src.Size(), f)
+		err = file.PreAllocate(src.Size(), f)
 		if err != nil {
 			fs.Debugf(o, "Failed to pre-allocate: %v", err)
 		}
@@ -1041,6 +1134,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.lstat()
 }
 
+var sparseWarning sync.Once
+
 // OpenWriterAt opens with a handle for random access writes
 //
 // Pass in the remote desired and the size if known.
@@ -1064,15 +1159,30 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 		return nil, err
 	}
 	// Pre-allocate the file for performance reasons
-	err = preAllocate(size, out)
+	err = file.PreAllocate(size, out)
 	if err != nil {
 		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
+	if !f.opt.NoSparse && file.SetSparseImplemented {
+		sparseWarning.Do(func() {
+			fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
+		})
+		// Set the file to be a sparse file (important on Windows)
+		err = file.SetSparse(out)
+		if err != nil {
+			fs.Debugf(o, "Failed to set sparse: %v", err)
+		}
+	}
+
 	return out, nil
 }
 
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
+	// if not checking updated then don't update the stat
+	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
+		return
+	}
 	// Don't overwrite the info if we don't need to
 	// this avoids upsetting the race detector
 	if o.size != info.Size() {
@@ -1086,7 +1196,7 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	}
 }
 
-// Stat a Object into info
+// Stat an Object into info
 func (o *Object) lstat() error {
 	info, err := o.fs.lstat(o.path)
 	if err == nil {
@@ -1158,6 +1268,7 @@ var (
 	_ fs.PutStreamer    = &Fs{}
 	_ fs.Mover          = &Fs{}
 	_ fs.DirMover       = &Fs{}
+	_ fs.Commander      = &Fs{}
 	_ fs.OpenWriterAter = &Fs{}
 	_ fs.Object         = &Object{}
 )

backend/local/preallocate_other.go

@@ -1,10 +0,0 @@
-//+build !windows,!linux
-
-package local
-
-import "os"
-
-// preAllocate the file for performance reasons
-func preAllocate(size int64, out *os.File) error {
-	return nil
-}

backend/mailru/mailru.go

@@ -402,7 +402,7 @@ func (q *quirks) parseQuirks(option string) {
 			// "Accept-Encoding: gzip" header. However, enabling compression
 			// might be good for performance.
 			// Use this quirk to investigate the performance impact.
-			// Remove this quirk if perfomance does not improve.
+			// Remove this quirk if performance does not improve.
 			q.gzip = true
 		case "insecure":
 			// The mailru disk-o protocol is not documented. To compare HTTP
@@ -1450,7 +1450,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	// fs.Debugf(f, ">>> PublicLink %q", remote)
 
 	token, err := f.accessToken()

backend/mega/mega.go

@@ -150,7 +150,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an mega 'url'
+// parsePath parses a mega 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -836,7 +836,7 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	root, err := f.findRoot(false)
 	if err != nil {
 		return "", errors.Wrap(err, "PublicLink failed to find root node")
@@ -871,7 +871,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 			return errors.Errorf("MergeDirs failed to find node for: %v", srcDir)
 		}
 
-		// list the the objects
+		// list the objects
 		infos := []*mega.Node{}
 		_, err := f.list(ctx, srcDirNode, func(info *mega.Node) bool {
 			infos = append(infos, info)

backend/onedrive/api/types.go

@@ -272,19 +272,19 @@ type CreateShareLinkResponse struct {
 	} `json:"link"`
 }
 
-// AsyncOperationStatus provides information on the status of a asynchronous job progress.
+// AsyncOperationStatus provides information on the status of an asynchronous job progress.
 //
 // The following API calls return AsyncOperationStatus resources:
 //
 // Copy Item
 // Upload From URL
 type AsyncOperationStatus struct {
-	PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
+	PercentageComplete float64 `json:"percentageComplete"` // A float value between 0 and 100 that indicates the percentage complete.
 	Status             string  `json:"status"`             // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
 }
 
 // GetID returns a normalized ID of the item
-// If DriveID is known it will be prefixed to the ID with # seperator
+// If DriveID is known it will be prefixed to the ID with # separator
 // Can be parsed using onedrive.parseNormalizedID(normalizedID)
 func (i *Item) GetID() string {
 	if i.IsRemote() && i.RemoteItem.ID != "" {

backend/onedrive/onedrive.go

@@ -79,7 +79,7 @@ func init() {
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
 			ctx := context.TODO()
-			err := oauthutil.Config("onedrive", name, m, oauthConfig)
+			err := oauthutil.Config("onedrive", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 				return
@@ -184,6 +184,28 @@ func init() {
 					log.Fatalf("Failed to query available drives: %v", err)
 				}
 
+				// Also call /me/drive as sometimes /me/drives doesn't return it #4068
+				if opts.Path == "/me/drives" {
+					opts.Path = "/me/drive"
+					meDrive := driveResource{}
+					_, err := srv.CallJSON(ctx, &opts, nil, &meDrive)
+					if err != nil {
+						log.Fatalf("Failed to query available drives: %v", err)
+					}
+					found := false
+					for _, drive := range drives.Drives {
+						if drive.DriveID == meDrive.DriveID {
+							found = true
+							break
+						}
+					}
+					// add the me drive if not found already
+					if !found {
+						fs.Debugf(nil, "Adding %v to drives list from /me/drive", meDrive)
+						drives.Drives = append(drives.Drives, meDrive)
+					}
+				}
+
 				if len(drives.Drives) == 0 {
 					log.Fatalf("No drives found")
 				} else {
@@ -226,8 +248,9 @@ func init() {
 			Name: "chunk_size",
 			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
 
-Above this size files will be chunked - must be multiple of 320k (327,680 bytes). Note
-that the chunks will be buffered into memory.`,
+Above this size files will be chunked - must be multiple of 320k (327,680 bytes) and
+should not exceed 250M (262,144,000 bytes) else you may encounter \"Microsoft.SharePoint.Client.InvalidClientQueryException: The request message is too big.\"
+Note that the chunks will be buffered into memory.`,
 			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
@@ -251,6 +274,16 @@ delete OneNote files or otherwise want them to show up in directory
 listing, set this option.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different onedrive configs.
+
+This can be useful if you wish to do a server side copy between two
+different Onedrives.  Note that this isn't enabled by default
+because it isn't easy to tell if it will work between any two
+configurations.`,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -303,11 +336,12 @@ listing, set this option.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize          fs.SizeSuffix        `config:"chunk_size"`
-	DriveID            string               `config:"drive_id"`
-	DriveType          string               `config:"drive_type"`
-	ExposeOneNoteFiles bool                 `config:"expose_onenote_files"`
-	Enc                encoder.MultiEncoder `config:"encoding"`
+	ChunkSize               fs.SizeSuffix        `config:"chunk_size"`
+	DriveID                 string               `config:"drive_id"`
+	DriveType               string               `config:"drive_type"`
+	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
+	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
+	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote one drive
@@ -362,7 +396,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an one drive 'url'
+// parsePath parses a one drive 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -402,6 +436,8 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 					fs.Debugf(nil, "Too many requests. Trying again in %d seconds.", retryAfter)
 				}
 			}
+		case 507: // Insufficient Storage
+			return false, fserrors.FatalError(err)
 		}
 	}
 	return retry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
@@ -462,13 +498,13 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var dirCacheRootIDExists bool
-		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
+		rootNormalizedID, err = f.dirCache.RootID(ctx, false)
+		dirCacheRootIDExists := err == nil
 		if f.root == "" {
 			// if f.root == "", it means f.root is the absolute root of the drive
 			// and its ID should have been found in NewFs
 			dirCacheFoundRoot = dirCacheRootIDExists
-		} else if _, err := f.dirCache.RootParentID(); err == nil {
+		} else if _, err := f.dirCache.RootParentID(ctx, false); err == nil {
 			// if root is in a folder, it must have a parent folder, and
 			// if dirCache has found root in NewFs, the parent folder's ID
 			// should be present.
@@ -576,6 +612,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		CaseInsensitive:         true,
 		ReadMimeType:            true,
 		CanHaveEmptyDirectories: true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f)
 	f.srv.SetErrorHandler(errorHandler)
 
@@ -776,10 +813,6 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -827,7 +860,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, leaf, directoryID, err
 	}
@@ -858,13 +891,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -887,10 +914,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -988,10 +1011,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, err
 	}
 
-	srcPath := srcObj.rootPath()
-	dstPath := f.rootPath(remote)
-	if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
-		return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
+	// Check we aren't overwriting a file on the same remote
+	if srcObj.fs == f {
+		srcPath := srcObj.rootPath()
+		dstPath := f.rootPath(remote)
+		if strings.ToLower(srcPath) == strings.ToLower(dstPath) {
+			return nil, errors.Errorf("can't copy %q -> %q as are same name when lowercase", srcPath, dstPath)
+		}
 	}
 
 	// Create temporary object
@@ -1133,50 +1159,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, dstDirectoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
 	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
 	_, srcDriveID, _ := parseNormalizedID(srcID)
 
 	if dstDriveID != srcDriveID {
@@ -1185,18 +1174,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorCantDirMove
 	}
 
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
 	// Get timestamps of src so they can be preserved
 	srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(ctx, srcID, "")
 	if err != nil {
@@ -1206,7 +1183,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	// Do the move
 	opts := newOptsCall(srcID, "PATCH", "")
 	move := api.MoveItemRequest{
-		Name: f.opt.Enc.FromStandardName(leaf),
+		Name: f.opt.Enc.FromStandardName(dstLeaf),
 		ParentReference: &api.ItemReference{
 			DriveID: dstDriveID,
 			ID:      parsedDstDirID,
@@ -1270,8 +1247,8 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(QuickXorHashType)
 }
 
-// PublicLink returns a link for downloading without accout.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+// PublicLink returns a link for downloading without account.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
 	if err != nil {
 		return "", err
@@ -1572,7 +1549,7 @@ func (o *Object) getPosition(ctx context.Context, url string) (pos int64, err er
 }
 
 // uploadFragment uploads a part
-func (o *Object) uploadFragment(ctx context.Context, url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64) (info *api.Item, err error) {
+func (o *Object) uploadFragment(ctx context.Context, url string, start int64, totalSize int64, chunk io.ReadSeeker, chunkSize int64, options ...fs.OpenOption) (info *api.Item, err error) {
 	//	var response api.UploadFragmentResponse
 	var resp *http.Response
 	var body []byte
@@ -1585,6 +1562,7 @@ func (o *Object) uploadFragment(ctx context.Context, url string, start int64, to
 			ContentLength: &toSend,
 			ContentRange:  fmt.Sprintf("bytes %d-%d/%d", start+skip, start+chunkSize-1, totalSize),
 			Body:          chunk,
+			Options:       options,
 		}
 		_, _ = chunk.Seek(skip, io.SeekStart)
 		resp, err = o.fs.srv.Call(ctx, &opts)
@@ -1642,46 +1620,27 @@ func (o *Object) cancelUploadSession(ctx context.Context, url string) (err error
 }
 
 // uploadMultipart uploads a file using multipart upload
-func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
+func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
 	if size <= 0 {
 		return nil, errors.New("unknown-sized upload not supported")
 	}
 
-	uploadURLChan := make(chan string, 1)
-	gracefulCancel := func() {
-		uploadURL, ok := <-uploadURLChan
-		// Reading from uploadURLChan blocks the atexit process until
-		// we are able to use uploadURL to cancel the upload
-		if !ok { // createUploadSession failed - no need to cancel upload
-			return
-		}
-
-		fs.Debugf(o, "Cancelling multipart upload")
-		cancelErr := o.cancelUploadSession(ctx, uploadURL)
-		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
-		}
-	}
-	cancelFuncHandle := atexit.Register(gracefulCancel)
-
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(ctx, modTime)
 	if err != nil {
-		close(uploadURLChan)
-		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
-	uploadURLChan <- uploadURL
 
-	defer func() {
-		if err != nil {
-			fs.Debugf(o, "Error encountered during upload: %v", err)
-			gracefulCancel()
+	// Cancel the session if something went wrong
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(o, "Cancelling multipart upload: %v", err)
+		cancelErr := o.cancelUploadSession(ctx, uploadURL)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-		atexit.Unregister(cancelFuncHandle)
-	}()
+	})()
 
 	// Upload the chunks
 	remaining := size
@@ -1693,7 +1652,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 		}
 		seg := readers.NewRepeatableReader(io.LimitReader(in, n))
 		fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)
-		info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n)
+		info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)
 		if err != nil {
 			return nil, err
 		}
@@ -1706,7 +1665,7 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 
 // Update the content of a remote file within 4MB size in one single request
 // This function will set modtime after uploading, which will create a new version for the remote file
-func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time) (info *api.Item, err error) {
+func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64, modTime time.Time, options ...fs.OpenOption) (info *api.Item, err error) {
 	if size < 0 || size > int64(fs.SizeSuffix(4*1024*1024)) {
 		return nil, errors.New("size passed into uploadSinglepart must be >= 0 and <= 4MiB")
 	}
@@ -1723,6 +1682,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(o.fs.opt.Enc.FromStandardName(leaf)) + ":/content",
 			ContentLength: &size,
 			Body:          in,
+			Options:       options,
 		}
 	} else {
 		opts = rest.Opts{
@@ -1730,6 +1690,7 @@ func (o *Object) uploadSinglepart(ctx context.Context, in io.Reader, size int64,
 			Path:          "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
 			ContentLength: &size,
 			Body:          in,
+			Options:       options,
 		}
 	}
 
@@ -1771,9 +1732,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	var info *api.Item
 	if size > 0 {
-		info, err = o.uploadMultipart(ctx, in, size, modTime)
+		info, err = o.uploadMultipart(ctx, in, size, modTime, options...)
 	} else if size == 0 {
-		info, err = o.uploadSinglepart(ctx, in, size, modTime)
+		info, err = o.uploadSinglepart(ctx, in, size, modTime, options...)
 	} else {
 		return errors.New("unknown-sized upload not supported")
 	}

backend/opendrive/opendrive.go

@@ -63,7 +63,7 @@ func init() {
 			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
 			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
 			//
-			// Additionally names can't begin or end with a ASCII whitespace.
+			// Additionally names can't begin or end with an ASCII whitespace.
 			// List of replaced characters:
 			//     (space)           -> '␠'  // SYMBOL FOR SPACE
 			//     (horizontal tab)  -> '␉'  // SYMBOL FOR HORIZONTAL TABULATION
@@ -280,13 +280,7 @@ func errorHandler(resp *http.Response) error {
 // Mkdir creates the folder if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	// fs.Debugf(nil, "Mkdir(\"%s\")", dir)
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -312,10 +306,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -483,58 +473,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -546,9 +486,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		moveFolderData := moveCopyFolder{
 			SessionID:     f.session.SessionID,
 			FolderID:      srcID,
-			DstFolderID:   directoryID,
+			DstFolderID:   dstDirectoryID,
 			Move:          "true",
-			NewFolderName: leaf,
+			NewFolderName: dstLeaf,
 		}
 		opts := rest.Opts{
 			Method: "POST",
@@ -620,7 +560,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, leaf, directoryID, err
 	}
@@ -677,7 +617,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	}
 
 	if "" == o.id {
-		// We need to create a ID for this file
+		// We need to create an ID for this file
 		var resp *http.Response
 		response := createFileResponse{}
 		err := o.fs.pacer.Call(func() (bool, error) {
@@ -687,8 +627,9 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 				Name:      leaf,
 			}
 			opts := rest.Opts{
-				Method: "POST",
-				Path:   "/upload/create_file.json",
+				Method:  "POST",
+				Options: options,
+				Path:    "/upload/create_file.json",
 			}
 			resp, err = o.fs.srv.CallJSON(ctx, &opts, &createFileData, &response)
 			return o.fs.shouldRetry(resp, err)
@@ -802,10 +743,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	// fs.Debugf(nil, "List(%v)", dir)
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -970,8 +907,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		openUploadData := openUpload{SessionID: o.fs.session.SessionID, FileID: o.id, Size: size}
 		// fs.Debugf(nil, "PreOpen: %#v", openUploadData)
 		opts := rest.Opts{
-			Method: "POST",
-			Path:   "/upload/open_file_upload.json",
+			Method:  "POST",
+			Options: options,
+			Path:    "/upload/open_file_upload.json",
 		}
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &openUploadData, &openResponse)
 		return o.fs.shouldRetry(resp, err)
@@ -1077,7 +1015,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 func (o *Object) readMetaData(ctx context.Context) (err error) {
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return fs.ErrorObjectNotFound

backend/opendrive/types.go

@@ -18,13 +18,13 @@ func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }
 
-// Account describes a OpenDRIVE account
+// Account describes an OpenDRIVE account
 type Account struct {
 	Username string `json:"username"`
 	Password string `json:"passwd"`
 }
 
-// UserSessionInfo describes a OpenDRIVE session
+// UserSessionInfo describes an OpenDRIVE session
 type UserSessionInfo struct {
 	Username string `json:"username"`
 	Password string `json:"passwd"`
@@ -45,7 +45,7 @@ type UserSessionInfo struct {
 	PartnerUsersDomain string          `json:"PartnerUsersDomain"`
 }
 
-// FolderList describes a OpenDRIVE listing
+// FolderList describes an OpenDRIVE listing
 type FolderList struct {
 	// DirUpdateTime    string   `json:"DirUpdateTime,string"`
 	Name             string   `json:"Name"`
@@ -56,7 +56,7 @@ type FolderList struct {
 	Files            []File   `json:"Files"`
 }
 
-// Folder describes a OpenDRIVE folder
+// Folder describes an OpenDRIVE folder
 type Folder struct {
 	FolderID      string `json:"FolderID"`
 	Name          string `json:"Name"`
@@ -109,7 +109,7 @@ type removeFolder struct {
 	FolderID  string `json:"folder_id"`
 }
 
-// File describes a OpenDRIVE file
+// File describes an OpenDRIVE file
 type File struct {
 	FileID            string `json:"FileId"`
 	FileHash          string `json:"FileHash"`

backend/pcloud/api/types.go

@@ -152,6 +152,14 @@ type ChecksumFileResult struct {
 	Metadata Item `json:"metadata"`
 }
 
+// PubLinkResult is returned from /getfilepublink and /getfolderpublink
+type PubLinkResult struct {
+	Error
+	LinkID   int    `json:"linkid"`
+	Link     string `json:"link"`
+	LinkCode string `json:"code"`
+}
+
 // UserInfo is returned from /userinfo
 type UserInfo struct {
 	Error

backend/pcloud/pcloud.go

@@ -41,8 +41,7 @@ const (
 	rcloneEncryptedClientSecret = "ej1OIF39VOQQ0PXaSdK9ztkLw3tdLNscW2157TKNQdQKkICR4uU7aFg4eFM"
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
-	decayConstant               = 2    // bigger for slower decay, exponential
-	rootID                      = "d0" // ID of root folder is always this
+	decayConstant               = 2 // bigger for slower decay, exponential
 	rootURL                     = "https://api.pcloud.com"
 )
 
@@ -68,7 +67,10 @@ func init() {
 		Description: "Pcloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("pcloud", name, m, oauthConfig)
+			opt := oauthutil.Options{
+				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
+			}
+			err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -89,13 +91,19 @@ func init() {
 			Default: (encoder.Display |
 				encoder.EncodeBackSlash |
 				encoder.EncodeInvalidUtf8),
+		}, {
+			Name:     "root_folder_id",
+			Help:     "Fill in for rclone to use a non root folder as its starting point.",
+			Default:  "d0",
+			Advanced: true,
 		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	Enc encoder.MultiEncoder `config:"encoding"`
+	Enc          encoder.MultiEncoder `config:"encoding"`
+	RootFolderID string               `config:"root_folder_id"`
 }
 
 // Fs represents a remote pcloud
@@ -147,7 +155,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an pcloud 'url'
+// parsePath parses a pcloud 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -190,7 +198,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -265,7 +273,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return err
 	})
 
-	// Get rootID
+	// Get rootFolderID
+	rootID := f.opt.RootFolderID
 	f.dirCache = dircache.New(root, rootID, f)
 
 	// Find the current root
@@ -446,10 +455,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -490,7 +495,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -521,13 +526,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -539,10 +538,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -650,7 +645,7 @@ func (f *Fs) Purge(ctx context.Context) error {
 
 // CleanUp empties the trash
 func (f *Fs) CleanUp(ctx context.Context) error {
-	err := f.dirCache.FindRoot(ctx, false)
+	rootID, err := f.dirCache.RootID(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -659,7 +654,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		Path:       "/trash_clear",
 		Parameters: url.Values{},
 	}
-	opts.Parameters.Set("folderid", dirIDtoNumber(f.dirCache.RootID()))
+	opts.Parameters.Set("folderid", dirIDtoNumber(rootID))
 	var resp *http.Response
 	var result api.Error
 	return f.pacer.Call(func() (bool, error) {
@@ -732,58 +727,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -795,8 +740,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		Parameters: url.Values{},
 	}
 	opts.Parameters.Set("folderid", dirIDtoNumber(srcID))
-	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(leaf))
-	opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID))
+	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(dstLeaf))
+	opts.Parameters.Set("tofolderid", dirIDtoNumber(dstDirectoryID))
 	var resp *http.Response
 	var result api.ItemResult
 	err = f.pacer.Call(func() (bool, error) {
@@ -818,6 +763,61 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
+func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (string, error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/getfolderpublink",
+		Parameters: url.Values{},
+	}
+	var result api.PubLinkResult
+	opts.Parameters.Set("folderid", dirIDtoNumber(dirID))
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
+		err = result.Error.Update(err)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return result.Link, err
+}
+
+func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (string, error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/getfilepublink",
+		Parameters: url.Values{},
+	}
+	var result api.PubLinkResult
+	opts.Parameters.Set("path", path)
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
+		err = result.Error.Update(err)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return result.Link, nil
+}
+
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	err := f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return "", err
+	}
+
+	dirID, err := f.dirCache.FindDir(ctx, remote, false)
+	if err == fs.ErrorDirNotFound {
+		return f.linkFile(ctx, remote, expire)
+	}
+	if err != nil {
+		return "", err
+	}
+	return f.linkDir(ctx, dirID, expire)
+}
+
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	opts := rest.Opts{
@@ -1047,7 +1047,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
@@ -1074,6 +1074,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		ContentLength:    &size,
 		Parameters:       url.Values{},
 		TransferEncoding: []string{"identity"}, // pcloud doesn't like chunked encoding
+		Options:          options,
 	}
 	leaf = o.fs.opt.Enc.FromStandardName(leaf)
 	opts.Parameters.Set("filename", leaf)
@@ -1151,6 +1152,7 @@ var (
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)

backend/premiumizeme/api/types.go

@@ -10,7 +10,7 @@ type Response struct {
 	Status  string `json:"status"`
 }
 
-// Error statisfies the error interface
+// Error satisfies the error interface
 func (e *Response) Error() string {
 	return fmt.Sprintf("%s: %s", e.Status, e.Message)
 }

backend/premiumizeme/premiumizeme.go

@@ -79,7 +79,7 @@ func init() {
 		Description: "premiumize.me",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("premiumizeme", name, m, oauthConfig)
+			err := oauthutil.Config("premiumizeme", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -183,7 +183,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -450,10 +450,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -493,7 +489,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -517,7 +513,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return existingObj, existingObj.Update(ctx, in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
-		return f.PutUnchecked(ctx, in, src)
+		return f.PutUnchecked(ctx, in, src, options...)
 	default:
 		return nil, err
 	}
@@ -544,13 +540,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -562,10 +552,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -745,75 +731,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
-
-	// Find ID of src parent, not creating subdirs
-	var srcLeaf, srcDirectoryID string
-	findPath = srcRemote
-	if srcRemote == "" {
-		findPath = srcFs.root
-	}
-	srcLeaf, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
+	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	err = f.move(ctx, false, srcID, srcLeaf, leaf, srcDirectoryID, directoryID)
+	err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -822,7 +747,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	_, err := f.dirCache.FindDir(ctx, remote, false)
 	if err == nil {
 		return "", fs.ErrorCantShareDirectories
@@ -990,7 +915,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	size := src.Size()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
@@ -1002,6 +927,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		Method:     "POST",
 		Path:       "/folder/uploadinfo",
 		Parameters: o.fs.baseParams(),
+		Options:    options,
 		MultipartParams: url.Values{
 			"id": {directoryID},
 		},

backend/putio/fs.go

@@ -18,6 +18,7 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/oauthutil"
@@ -34,7 +35,8 @@ type Fs struct {
 	client      *putio.Client      // client for making API calls to Put.io
 	pacer       *fs.Pacer          // To pace the API calls
 	dirCache    *dircache.DirCache // Map of directory path to directory id
-	oAuthClient *http.Client
+	httpClient  *http.Client       // base http client
+	oAuthClient *http.Client       // http client with oauth Authorization
 }
 
 // ------------------------------------------------------------
@@ -59,6 +61,12 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
+// parsePath parses a putio 'url'
+func parsePath(path string) (root string) {
+	root = strings.Trim(path, "/")
+	return
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) {
 	// defer log.Trace(name, "root=%v", root)("f=%+v, err=%v", &f, &err)
@@ -68,7 +76,9 @@ func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) {
 	if err != nil {
 		return nil, err
 	}
-	oAuthClient, _, err := oauthutil.NewClient(name, m, putioConfig)
+	root = parsePath(root)
+	httpClient := fshttp.NewClient(fs.Config)
+	oAuthClient, _, err := oauthutil.NewClientWithBaseClient(name, m, putioConfig, httpClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure putio")
 	}
@@ -78,6 +88,7 @@ func NewFs(name, root string, m configmap.Mapper) (f fs.Fs, err error) {
 		opt:         *opt,
 		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		client:      putio.NewClient(oAuthClient),
+		httpClient:  httpClient,
 		oAuthClient: oAuthClient,
 	}
 	p.features = (&fs.Features{
@@ -186,10 +197,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -249,11 +256,11 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	// defer log.Trace(f, "src=%+v", src)("o=%+v, err=%v", &o, &err)
 	size := src.Size()
 	remote := src.Remote()
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
-	loc, err := f.createUpload(ctx, leaf, size, directoryID, src.ModTime(ctx))
+	loc, err := f.createUpload(ctx, leaf, size, directoryID, src.ModTime(ctx), options)
 	if err != nil {
 		return nil, err
 	}
@@ -273,7 +280,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	return f.newObjectWithInfo(ctx, remote, entry)
 }
 
-func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID string, modTime time.Time) (location string, err error) {
+func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID string, modTime time.Time, options []fs.OpenOption) (location string, err error) {
 	// defer log.Trace(f, "name=%v, size=%v, parentID=%v, modTime=%v", name, size, parentID, modTime.String())("location=%v, err=%v", location, &err)
 	err = f.pacer.Call(func() (bool, error) {
 		req, err := http.NewRequest("POST", "https://upload.put.io/files/", nil)
@@ -288,6 +295,7 @@ func (f *Fs) createUpload(ctx context.Context, name string, size int64, parentID
 		b64parentID := base64.StdEncoding.EncodeToString([]byte(parentID))
 		b64modifiedAt := base64.StdEncoding.EncodeToString([]byte(modTime.Format(time.RFC3339)))
 		req.Header.Set("upload-metadata", fmt.Sprintf("name %s,no-torrent %s,parent_id %s,updated-at %s", b64name, b64true, b64parentID, b64modifiedAt))
+		fs.OpenOptionAddHTTPHeaders(req.Header, options)
 		resp, err := f.oAuthClient.Do(req)
 		retry, err := shouldRetry(err)
 		if retry {
@@ -446,13 +454,7 @@ func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
-	err = f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err = f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -516,12 +518,11 @@ func (f *Fs) Purge(ctx context.Context) (err error) {
 	if f.root == "" {
 		return errors.New("can't purge root directory")
 	}
-	err = f.dirCache.FindRoot(ctx, false)
+	rootIDs, err := f.dirCache.RootID(ctx, false)
 	if err != nil {
 		return err
 	}
-
-	rootID := atoi(f.dirCache.RootID())
+	rootID := atoi(rootIDs)
 	// Let putio delete the filesystem tree
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "deleting file: %d", rootID)
@@ -547,7 +548,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 	if !ok {
 		return nil, fs.ErrorCantCopy
 	}
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -586,7 +587,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 	if !ok {
 		return nil, fs.ErrorCantMove
 	}
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -624,57 +625,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	if !ok {
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, dstDirectoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -683,7 +635,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		params := url.Values{}
 		params.Set("file_id", srcID)
 		params.Set("parent_id", dstDirectoryID)
-		params.Set("name", f.opt.Enc.FromStandardName(leaf))
+		params.Set("name", f.opt.Enc.FromStandardName(dstLeaf))
 		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
 		if err != nil {
 			return false, err

backend/putio/object.go

@@ -125,7 +125,7 @@ func (o *Object) setMetadataFromEntry(info putio.File) error {
 // Reads the entry for a file from putio
 func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
 	// defer log.Trace(o, "")("f=%+v, err=%v", f, &err)
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -241,14 +241,17 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			req.Header.Set(header, value)
 		}
 		// fs.Debugf(o, "opening file: id=%d", o.file.ID)
-		resp, err = http.DefaultClient.Do(req)
+		resp, err = o.fs.httpClient.Do(req)
 		return shouldRetry(err)
 	})
 	if perr, ok := err.(*putio.ErrorResponse); ok && perr.Response.StatusCode >= 400 && perr.Response.StatusCode <= 499 {
 		_ = resp.Body.Close()
 		return nil, fserrors.NoRetryError(err)
 	}
-	return resp.Body, err
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, nil
 }
 
 // Update the already existing object

backend/putio/putio.go

@@ -60,7 +60,10 @@ func init() {
 		Description: "Put.io",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.ConfigNoOffline("putio", name, m, putioConfig)
+			opt := oauthutil.Options{
+				NoOffline: true,
+			}
+			err := oauthutil.Config("putio", name, m, putioConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}

backend/qingstor/qingstor.go

@@ -56,7 +56,7 @@ func init() {
 			Help: "QingStor Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
 		}, {
 			Name: "endpoint",
-			Help: "Enter a endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
+			Help: "Enter an endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
 		}, {
 			Name: "zone",
 			Help: "Zone to connect to.\nDefault is \"pek3a\".",
@@ -203,10 +203,10 @@ func (o *Object) split() (bucket, bucketPath string) {
 	return o.fs.split(o.remote)
 }
 
-// Split an URL into three parts: protocol host and port
+// Split a URL into three parts: protocol host and port
 func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) {
 	/*
-	  Pattern to match a endpoint,
+	  Pattern to match an endpoint,
 	  eg: "http(s)://qingstor.com:443" --> "http(s)", "qingstor.com", 443
 	      "http(s)//qingstor.com"      --> "http(s)", "qingstor.com", ""
 	      "qingstor.com"               --> "", "qingstor.com", ""
@@ -864,6 +864,76 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	})
 }
 
+// cleanUpBucket removes all pending multipart uploads for a given bucket
+func (f *Fs) cleanUpBucket(ctx context.Context, bucket string) (err error) {
+	fs.Infof(f, "cleaning bucket %q of pending multipart uploads older than 24 hours", bucket)
+	bucketInit, err := f.svc.Bucket(bucket, f.zone)
+	if err != nil {
+		return err
+	}
+	maxLimit := int(listLimitSize)
+	var marker *string
+	for {
+		req := qs.ListMultipartUploadsInput{
+			Limit:     &maxLimit,
+			KeyMarker: marker,
+		}
+		var resp *qs.ListMultipartUploadsOutput
+		resp, err = bucketInit.ListMultipartUploads(&req)
+		if err != nil {
+			return errors.Wrap(err, "clean up bucket list multipart uploads")
+		}
+		for _, upload := range resp.Uploads {
+			if upload.Created != nil && upload.Key != nil && upload.UploadID != nil {
+				age := time.Since(*upload.Created)
+				if age > 24*time.Hour {
+					fs.Infof(f, "removing pending multipart upload for %q dated %v (%v ago)", *upload.Key, upload.Created, age)
+					req := qs.AbortMultipartUploadInput{
+						UploadID: upload.UploadID,
+					}
+					_, abortErr := bucketInit.AbortMultipartUpload(*upload.Key, &req)
+					if abortErr != nil {
+						err = errors.Wrapf(abortErr, "failed to remove multipart upload for %q", *upload.Key)
+						fs.Errorf(f, "%v", err)
+					}
+				} else {
+					fs.Debugf(f, "ignoring pending multipart upload for %q dated %v (%v ago)", *upload.Key, upload.Created, age)
+				}
+			}
+		}
+		if resp.HasMore != nil && !*resp.HasMore {
+			break
+		}
+		// Use NextMarker if set, otherwise use last Key
+		if resp.NextKeyMarker == nil || *resp.NextKeyMarker == "" {
+			fs.Errorf(f, "Expecting NextKeyMarker but didn't find one")
+			break
+		} else {
+			marker = resp.NextKeyMarker
+		}
+	}
+	return err
+}
+
+// CleanUp removes all pending multipart uploads
+func (f *Fs) CleanUp(ctx context.Context) (err error) {
+	if f.rootBucket != "" {
+		return f.cleanUpBucket(ctx, f.rootBucket)
+	}
+	entries, err := f.listBuckets(ctx)
+	if err != nil {
+		return err
+	}
+	for _, entry := range entries {
+		cleanErr := f.cleanUpBucket(ctx, f.opt.Enc.FromStandardName(entry.Remote()))
+		if err != nil {
+			fs.Errorf(f, "Failed to cleanup bucket: %q", cleanErr)
+			err = cleanErr
+		}
+	}
+	return err
+}
+
 // readMetaData gets the metadata if it hasn't already been fetched
 //
 // it also sets the info
@@ -1090,9 +1160,10 @@ func (o *Object) MimeType(ctx context.Context) string {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs        = &Fs{}
-	_ fs.Copier    = &Fs{}
-	_ fs.Object    = &Object{}
-	_ fs.ListRer   = &Fs{}
-	_ fs.MimeTyper = &Object{}
+	_ fs.Fs         = &Fs{}
+	_ fs.CleanUpper = &Fs{}
+	_ fs.Copier     = &Fs{}
+	_ fs.Object     = &Object{}
+	_ fs.ListRer    = &Fs{}
+	_ fs.MimeTyper  = &Object{}
 )

backend/qingstor/upload.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/atexit"
 	qs "github.com/yunify/qingstor-sdk-go/v3/service"
 )
 
@@ -22,7 +23,7 @@ const (
 	// maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
 	// maxMultiPartSize = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
 	minMultiPartSize = 1024 * 1024 * 4 // The minimum allowed part size when uploading a part to QingStor
-	maxMultiParts    = 10000           // The maximum allowed number of parts in an multi-part upload
+	maxMultiParts    = 10000           // The maximum allowed number of parts in a multi-part upload
 )
 
 const (
@@ -168,7 +169,7 @@ func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
 	return err
 }
 
-// Upload upload a object into QingStor
+// Upload upload an object into QingStor
 func (u *uploader) upload() error {
 	u.init()
 
@@ -255,7 +256,7 @@ func (mu *multiUploader) readChunk(ch chan chunk) {
 	}
 }
 
-// initiate init an Multiple Object and obtain UploadID
+// initiate init a Multiple Object and obtain UploadID
 func (mu *multiUploader) initiate() error {
 	bucketInit, _ := mu.bucketInit()
 	req := qs.InitiateMultipartUploadInput{
@@ -297,7 +298,7 @@ func (mu *multiUploader) send(c chunk) error {
 	return err
 }
 
-// complete complete an multipart upload
+// complete complete a multipart upload
 func (mu *multiUploader) complete() error {
 	var err error
 	if err = mu.getErr(); err != nil {
@@ -324,7 +325,7 @@ func (mu *multiUploader) complete() error {
 	return err
 }
 
-// abort abort an multipart upload
+// abort abort a multipart upload
 func (mu *multiUploader) abort() error {
 	var err error
 	bucketInit, _ := mu.bucketInit()
@@ -341,13 +342,21 @@ func (mu *multiUploader) abort() error {
 }
 
 // multiPartUpload upload a multiple object into QingStor
-func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error {
-	var err error
-	//Initiate an multi-part upload
+func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
+	// Initiate a multi-part upload
 	if err = mu.initiate(); err != nil {
 		return err
 	}
 
+	// Cancel the session if something went wrong
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(mu, "Cancelling multipart upload: %v", err)
+		cancelErr := mu.abort()
+		if cancelErr != nil {
+			fs.Logf(mu, "Failed to cancel multipart upload: %v", cancelErr)
+		}
+	})()
+
 	ch := make(chan chunk, mu.cfg.concurrency)
 	for i := 0; i < mu.cfg.concurrency; i++ {
 		mu.wg.Add(1)
@@ -400,9 +409,5 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error {
 	close(ch)
 	mu.wg.Wait()
 	// Complete Multipart Upload
-	err = mu.complete()
-	if mu.getErr() != nil || err != nil {
-		_ = mu.abort()
-	}
-	return err
+	return mu.complete()
 }

backend/s3/s3.go

@@ -40,6 +40,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -52,13 +53,16 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
+	"github.com/rclone/rclone/lib/structs"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -68,6 +72,7 @@ func init() {
 		Name:        "s3",
 		Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name: fs.ConfigProvider,
 			Help: "Choose your S3 provider.",
@@ -95,6 +100,9 @@ func init() {
 			}, {
 				Value: "Netease",
 				Help:  "Netease Object Storage (NOS)",
+			}, {
+				Value: "Scaleway",
+				Help:  "Scaleway Object Storage",
 			}, {
 				Value: "StackPath",
 				Help:  "StackPath Object Storage",
@@ -175,10 +183,21 @@ func init() {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "Scaleway",
+			Examples: []fs.OptionExample{{
+				Value: "nl-ams",
+				Help:  "Amsterdam, The Netherlands",
+			}, {
+				Value: "fr-par",
+				Help:  "Paris, France",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba",
+			Provider: "!AWS,Alibaba,Scaleway",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -360,6 +379,17 @@ func init() {
 				Value: "oss-me-east-1.aliyuncs.com",
 				Help:  "Middle East 1 (Dubai)",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Scaleway Object Storage.",
+			Provider: "Scaleway",
+			Examples: []fs.OptionExample{{
+				Value: "s3.nl-ams.scw.cloud",
+				Help:  "Amsterdam Endpoint",
+			}, {
+				Value: "s3.fr-par.scw.cloud",
+				Help:  "Paris Endpoint",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for StackPath Object Storage.",
@@ -377,7 +407,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,Alibaba,StackPath",
+			Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -564,7 +594,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba,StackPath",
+			Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -641,7 +671,7 @@ isn't set then "acl" is used instead.`,
 		}, {
 			Name:     "server_side_encryption",
 			Help:     "The server-side encryption algorithm used when storing this object in S3.",
-			Provider: "AWS",
+			Provider: "AWS,Ceph,Minio",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "None",
@@ -652,10 +682,22 @@ isn't set then "acl" is used instead.`,
 				Value: "aws:kms",
 				Help:  "aws:kms",
 			}},
+		}, {
+			Name:     "sse_customer_algorithm",
+			Help:     "If using SSE-C, the server-side encryption algorithm used when storing this object in S3.",
+			Provider: "AWS,Ceph,Minio",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "None",
+			}, {
+				Value: "AES256",
+				Help:  "AES256",
+			}},
 		}, {
 			Name:     "sse_kms_key_id",
 			Help:     "If using KMS ID you must provide the ARN of Key.",
-			Provider: "AWS",
+			Provider: "AWS,Ceph,Minio",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "None",
@@ -663,6 +705,24 @@ isn't set then "acl" is used instead.`,
 				Value: "arn:aws:kms:us-east-1:*",
 				Help:  "arn:aws:kms:*",
 			}},
+		}, {
+			Name:     "sse_customer_key",
+			Help:     "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
+			Provider: "AWS,Ceph,Minio",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "None",
+			}},
+		}, {
+			Name:     "sse_customer_key_md5",
+			Help:     "If using SSE-C you must provide the secret encryption key MD5 checksum.",
+			Provider: "AWS,Ceph,Minio",
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "None",
+			}},
 		}, {
 			Name:     "storage_class",
 			Help:     "The storage class to use when storing new objects in S3.",
@@ -710,6 +770,21 @@ isn't set then "acl" is used instead.`,
 				Value: "STANDARD_IA",
 				Help:  "Infrequent access storage mode.",
 			}},
+		}, {
+			// Mapping from here: https://www.scaleway.com/en/docs/object-storage-glacier/#-Scaleway-Storage-Classes
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing new objects in S3.",
+			Provider: "Scaleway",
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "Default",
+			}, {
+				Value: "STANDARD",
+				Help:  "The Standard class for any upload; suitable for on-demand content like streaming or CDN.",
+			}, {
+				Value: "GLACIER",
+				Help:  "Archived storage; prices are lower, but it needs to be restored first to be accessed.",
+			}},
 		}, {
 			Name: "upload_cutoff",
 			Help: `Cutoff for switching to chunked upload
@@ -743,6 +818,21 @@ file you can stream upload is 48GB.  If you wish to stream upload
 larger files then you will need to increase chunk_size.`,
 			Default:  minChunkSize,
 			Advanced: true,
+		}, {
+			Name: "max_upload_parts",
+			Help: `Maximum number of parts in a multipart upload.
+
+This option defines the maximum number of multipart chunks to use
+when doing a multipart upload.
+
+This can be useful if a service does not support the AWS S3
+specification of 10,000 chunks.
+
+Rclone will automatically increase the chunk size when uploading a
+large file of a known size to stay below this number of chunks limit.
+`,
+			Default:  maxUploadParts,
+			Advanced: true,
 		}, {
 			Name: "copy_cutoff",
 			Help: `Cutoff for switching to multipart copy
@@ -754,8 +844,13 @@ The minimum is 0 and the maximum is 5GB.`,
 			Default:  fs.SizeSuffix(maxSizeForCopy),
 			Advanced: true,
 		}, {
-			Name:     "disable_checksum",
-			Help:     "Don't store MD5 checksum with object metadata",
+			Name: "disable_checksum",
+			Help: `Don't store MD5 checksum with object metadata
+
+Normally rclone will calculate the MD5 checksum of the input before
+uploading it so it can add it to metadata on the object. This is great
+for data integrity checking but can cause long delays for large files
+to start uploading.`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -874,6 +969,7 @@ const (
 
 	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
 	memoryPoolUseMmap   = false
+	maxExpireDuration   = fs.Duration(7 * 24 * time.Hour) // max expiry is 1 week
 )
 
 // Options defines the configuration for this backend
@@ -889,10 +985,14 @@ type Options struct {
 	BucketACL             string               `config:"bucket_acl"`
 	ServerSideEncryption  string               `config:"server_side_encryption"`
 	SSEKMSKeyID           string               `config:"sse_kms_key_id"`
+	SSECustomerAlgorithm  string               `config:"sse_customer_algorithm"`
+	SSECustomerKey        string               `config:"sse_customer_key"`
+	SSECustomerKeyMD5     string               `config:"sse_customer_key_md5"`
 	StorageClass          string               `config:"storage_class"`
 	UploadCutoff          fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff            fs.SizeSuffix        `config:"copy_cutoff"`
 	ChunkSize             fs.SizeSuffix        `config:"chunk_size"`
+	MaxUploadParts        int64                `config:"max_upload_parts"`
 	DisableChecksum       bool                 `config:"disable_checksum"`
 	SessionToken          string               `config:"session_token"`
 	UploadConcurrency     int                  `config:"upload_concurrency"`
@@ -908,19 +1008,18 @@ type Options struct {
 
 // Fs represents a remote s3 server
 type Fs struct {
-	name          string               // the name of the remote
-	root          string               // root of the bucket - ignore all objects above this
-	opt           Options              // parsed options
-	features      *fs.Features         // optional features
-	c             *s3.S3               // the connection to the s3 server
-	ses           *session.Session     // the s3 session
-	rootBucket    string               // bucket part of root (if any)
-	rootDirectory string               // directory part of root (if any)
-	cache         *bucket.Cache        // cache for bucket creation status
-	pacer         *fs.Pacer            // To pace the API calls
-	srv           *http.Client         // a plain http client
-	poolMu        sync.Mutex           // mutex protecting memory pools map
-	pools         map[int64]*pool.Pool // memory pools
+	name          string           // the name of the remote
+	root          string           // root of the bucket - ignore all objects above this
+	opt           Options          // parsed options
+	features      *fs.Features     // optional features
+	c             *s3.S3           // the connection to the s3 server
+	ses           *session.Session // the s3 session
+	rootBucket    string           // bucket part of root (if any)
+	rootDirectory string           // directory part of root (if any)
+	cache         *bucket.Cache    // cache for bucket creation status
+	pacer         *fs.Pacer        // To pace the API calls
+	srv           *http.Client     // a plain http client
+	pool          *pool.Pool       // memory pool
 }
 
 // Object describes a s3 object
@@ -1039,6 +1138,12 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	def := defaults.Get()
 	def.Config.HTTPClient = lowTimeoutClient
 
+	// start a new AWS session
+	awsSession, err := session.NewSession()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "NewSession")
+	}
+
 	// first provider to supply a credential set "wins"
 	providers := []credentials.Provider{
 		// use static credentials if they're present (checked by provider)
@@ -1058,7 +1163,7 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 
 		// Pick up IAM role in case we're on EC2
 		&ec2rolecreds.EC2RoleProvider{
-			Client: ec2metadata.New(session.New(), &aws.Config{
+			Client: ec2metadata.New(awsSession, &aws.Config{
 				HTTPClient: lowTimeoutClient,
 			}),
 			ExpiryWindow: 3 * time.Minute,
@@ -1089,15 +1194,20 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
+	if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
+	if opt.Provider == "Scaleway" && opt.MaxUploadParts > 1000 {
+		opt.MaxUploadParts = 1000
+	}
 	awsConfig := aws.NewConfig().
-		WithMaxRetries(fs.Config.LowLevelRetries).
+		WithMaxRetries(0). // Rely on rclone's retry logic
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle).
-		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
+		WithS3UseAccelerate(opt.UseAccelerateEndpoint).
+		WithS3UsEast1RegionalEndpoint(endpoints.RegionalS3UsEast1Endpoint)
+
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
@@ -1200,20 +1310,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, err
 	}
 
-	pc := fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep)))
-	// Set pacer retries to 0 because we are relying on SDK retry mechanism.
-	// Setting it to 1 because in context of pacer it means 1 attempt.
-	pc.SetRetries(1)
-
 	f := &Fs{
 		name:  name,
 		opt:   *opt,
 		c:     c,
 		ses:   ses,
-		pacer: pc,
+		pacer: fs.NewPacer(pacer.NewS3(pacer.MinSleep(minSleep))),
 		cache: bucket.NewCache(),
 		srv:   fshttp.NewClient(fs.Config),
-		pools: make(map[int64]*pool.Pool),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			opt.UploadConcurrency*fs.Config.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 
 	f.setRoot(root)
@@ -1463,7 +1573,7 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
 				continue
 			}
 			remote = remote[len(prefix):]
-			isDirectory := strings.HasSuffix(remote, "/")
+			isDirectory := remote == "" || strings.HasSuffix(remote, "/")
 			if addBucket {
 				remote = path.Join(bucket, remote)
 			}
@@ -1695,12 +1805,12 @@ func (f *Fs) makeBucket(ctx context.Context, bucket string) error {
 		if err == nil {
 			fs.Infof(f, "Bucket %q created with ACL %q", bucket, f.opt.BucketACL)
 		}
-		if err, ok := err.(awserr.Error); ok {
-			if err.Code() == "BucketAlreadyOwnedByYou" {
+		if awsErr, ok := err.(awserr.Error); ok {
+			if code := awsErr.Code(); code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" {
 				err = nil
 			}
 		}
-		return nil
+		return err
 	}, func() (bool, error) {
 		return f.bucketExists(ctx, bucket)
 	})
@@ -1796,21 +1906,19 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
 	}
 	uid := cout.UploadId
 
-	defer func() {
-		if err != nil {
-			// We can try to abort the upload, but ignore the error.
-			fs.Debugf(nil, "Cancelling multipart copy")
-			_ = f.pacer.Call(func() (bool, error) {
-				_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
-					Bucket:       &dstBucket,
-					Key:          &dstPath,
-					UploadId:     uid,
-					RequestPayer: req.RequestPayer,
-				})
-				return f.shouldRetry(err)
+	defer atexit.OnError(&err, func() {
+		// Try to abort the upload, but ignore the error.
+		fs.Debugf(nil, "Cancelling multipart copy")
+		_ = f.pacer.Call(func() (bool, error) {
+			_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
+				Bucket:       &dstBucket,
+				Key:          &dstPath,
+				UploadId:     uid,
+				RequestPayer: req.RequestPayer,
 			})
-		}
-	}()
+			return f.shouldRetry(err)
+		})
+	})()
 
 	partSize := int64(f.opt.CopyCutoff)
 	numParts := (srcSize-1)/partSize + 1
@@ -1904,19 +2012,157 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 func (f *Fs) getMemoryPool(size int64) *pool.Pool {
-	f.poolMu.Lock()
-	defer f.poolMu.Unlock()
-
-	_, ok := f.pools[size]
-	if !ok {
-		f.pools[size] = pool.New(
-			time.Duration(f.opt.MemoryPoolFlushTime),
-			int(f.opt.ChunkSize),
-			f.opt.UploadConcurrency*fs.Config.Transfers,
-			f.opt.MemoryPoolUseMmap,
+	if size == int64(f.opt.ChunkSize) {
+		return f.pool
+	}
+
+	return pool.New(
+		time.Duration(f.opt.MemoryPoolFlushTime),
+		int(size),
+		f.opt.UploadConcurrency*fs.Config.Transfers,
+		f.opt.MemoryPoolUseMmap,
+	)
+}
+
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
+	if strings.HasSuffix(remote, "/") {
+		return "", fs.ErrorCantShareDirectories
+	}
+	if _, err := f.NewObject(ctx, remote); err != nil {
+		return "", err
+	}
+	if expire > maxExpireDuration {
+		fs.Logf(f, "Public Link: Reducing expiry to %v as %v is greater than the max time allowed", maxExpireDuration, expire)
+		expire = maxExpireDuration
+	}
+	bucket, bucketPath := f.split(remote)
+	httpReq, _ := f.c.GetObjectRequest(&s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &bucketPath,
+	})
+
+	return httpReq.Presign(time.Duration(expire))
+}
+
+var commandHelp = []fs.CommandHelp{{
+	Name:  "restore",
+	Short: "Restore objects from GLACIER to normal storage",
+	Long: `This command can be used to restore one or more objects from GLACIER
+to normal storage.
+
+Usage Examples:
+
+    rclone backend restore s3:bucket/path/to/object [-o priority=PRIORITY] [-o lifetime=DAYS]
+    rclone backend restore s3:bucket/path/to/directory [-o priority=PRIORITY] [-o lifetime=DAYS]
+    rclone backend restore s3:bucket [-o priority=PRIORITY] [-o lifetime=DAYS]
+
+This flag also obeys the filters. Test first with -i/--interactive or --dry-run flags
+
+    rclone -i backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+
+All the objects shown will be marked for restore, then
+
+    rclone backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+
+It returns a list of status dictionaries with Remote and Status
+keys. The Status will be OK if it was successfull or an error message
+if not.
+
+    [
+        {
+            "Status": "OK",
+            "Path": "test.txt"
+        },
+        {
+            "Status": "OK",
+            "Path": "test/file4.txt"
+        }
+    ]
+
+`,
+	Opts: map[string]string{
+		"priority":    "Priority of restore: Standard|Expedited|Bulk",
+		"lifetime":    "Lifetime of the active copy in days",
+		"description": "The optional description for the job.",
+	},
+}}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "restore":
+		req := s3.RestoreObjectInput{
+			//Bucket:         &f.rootBucket,
+			//Key:            &encodedDirectory,
+			RestoreRequest: &s3.RestoreRequest{},
+		}
+		if lifetime := opt["lifetime"]; lifetime != "" {
+			ilifetime, err := strconv.ParseInt(lifetime, 10, 64)
+			if err != nil {
+				return nil, errors.Wrap(err, "bad lifetime")
+			}
+			req.RestoreRequest.Days = &ilifetime
+		}
+		if priority := opt["priority"]; priority != "" {
+			req.RestoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{
+				Tier: &priority,
+			}
+		}
+		if description := opt["description"]; description != "" {
+			req.RestoreRequest.Description = &description
+		}
+		type status struct {
+			Status string
+			Remote string
+		}
+		var (
+			outMu sync.Mutex
+			out   = []status{}
 		)
+		err = operations.ListFn(ctx, f, func(obj fs.Object) {
+			// Remember this is run --checkers times concurrently
+			o, ok := obj.(*Object)
+			st := status{Status: "OK", Remote: obj.Remote()}
+			defer func() {
+				outMu.Lock()
+				out = append(out, st)
+				outMu.Unlock()
+			}()
+			if operations.SkipDestructive(ctx, obj, "restore") {
+				return
+			}
+			if !ok {
+				st.Status = "Not an S3 object"
+				return
+			}
+			bucket, bucketPath := o.split()
+			reqCopy := req
+			reqCopy.Bucket = &bucket
+			reqCopy.Key = &bucketPath
+			err = f.pacer.Call(func() (bool, error) {
+				_, err = f.c.RestoreObject(&reqCopy)
+				return f.shouldRetry(err)
+			})
+			if err != nil {
+				st.Status = err.Error()
+			}
+		})
+		if err != nil {
+			return out, err
+		}
+		return out, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
 	}
-	return f.pools[size]
 }
 
 // ------------------------------------------------------------
@@ -1993,11 +2239,17 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
 			if awsErr.StatusCode() == http.StatusNotFound {
+				// NotFound indicates bucket was OK
+				// NoSuchBucket would be returned if bucket was bad
+				if awsErr.Code() == "NotFound" {
+					o.fs.cache.MarkOK(bucket)
+				}
 				return fs.ErrorObjectNotFound
 			}
 		}
 		return err
 	}
+	o.fs.cache.MarkOK(bucket)
 	var size int64
 	// Ignore missing Content-Length assuming it is 0
 	// Some versions of ceph do this due their apache proxies
@@ -2083,22 +2335,35 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		Bucket: &bucket,
 		Key:    &bucketPath,
 	}
+	if o.fs.opt.SSECustomerAlgorithm != "" {
+		req.SSECustomerAlgorithm = &o.fs.opt.SSECustomerAlgorithm
+	}
+	if o.fs.opt.SSECustomerKey != "" {
+		req.SSECustomerKey = &o.fs.opt.SSECustomerKey
+	}
+	if o.fs.opt.SSECustomerKeyMD5 != "" {
+		req.SSECustomerKeyMD5 = &o.fs.opt.SSECustomerKeyMD5
+	}
+	httpReq, resp := o.fs.c.GetObjectRequest(&req)
 	fs.FixRangeOption(options, o.bytes)
 	for _, option := range options {
 		switch option.(type) {
 		case *fs.RangeOption, *fs.SeekOption:
 			_, value := option.Header()
 			req.Range = &value
+		case *fs.HTTPOption:
+			key, value := option.Header()
+			httpReq.HTTPRequest.Header.Add(key, value)
 		default:
 			if option.Mandatory() {
 				fs.Logf(o, "Unsupported mandatory option: %v", option)
 			}
 		}
 	}
-	var resp *s3.GetObjectOutput
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
-		resp, err = o.fs.c.GetObjectWithContext(ctx, &req)
+		httpReq.HTTPRequest = httpReq.HTTPRequest.WithContext(ctx)
+		err = httpReq.Send()
 		return o.fs.shouldRetry(err)
 	})
 	if err, ok := err.(awserr.RequestFailure); ok {
@@ -2124,6 +2389,13 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	tokens := pacer.NewTokenDispenser(concurrency)
 
+	uploadParts := f.opt.MaxUploadParts
+	if uploadParts < 1 {
+		uploadParts = 1
+	} else if uploadParts > maxUploadParts {
+		uploadParts = maxUploadParts
+	}
+
 	// calculate size of parts
 	partSize := int(f.opt.ChunkSize)
 
@@ -2133,31 +2405,24 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	if size == -1 {
 		warnStreamUpload.Do(func() {
 			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
-				f.opt.ChunkSize, fs.SizeSuffix(partSize*maxUploadParts))
+				f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
 		})
 	} else {
 		// Adjust partSize until the number of parts is small enough.
-		if size/int64(partSize) >= maxUploadParts {
+		if size/int64(partSize) >= uploadParts {
 			// Calculate partition size rounded up to the nearest MB
-			partSize = int((((size / maxUploadParts) >> 20) + 1) << 20)
+			partSize = int((((size / uploadParts) >> 20) + 1) << 20)
 		}
 	}
 
 	memPool := f.getMemoryPool(int64(partSize))
 
+	var mReq s3.CreateMultipartUploadInput
+	structs.SetFrom(&mReq, req)
 	var cout *s3.CreateMultipartUploadOutput
 	err = f.pacer.Call(func() (bool, error) {
 		var err error
-		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
-			Bucket:               req.Bucket,
-			ACL:                  req.ACL,
-			Key:                  req.Key,
-			ContentType:          req.ContentType,
-			Metadata:             req.Metadata,
-			ServerSideEncryption: req.ServerSideEncryption,
-			SSEKMSKeyId:          req.SSEKMSKeyId,
-			StorageClass:         req.StorageClass,
-		})
+		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &mReq)
 		return f.shouldRetry(err)
 	})
 	if err != nil {
@@ -2165,27 +2430,24 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	uid := cout.UploadId
 
-	defer func() {
+	defer atexit.OnError(&err, func() {
 		if o.fs.opt.LeavePartsOnError {
 			return
 		}
-		if err != nil {
-			// We can try to abort the upload, but ignore the error.
-			fs.Debugf(o, "Cancelling multipart upload")
-			errCancel := f.pacer.Call(func() (bool, error) {
-				_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
-					Bucket:       req.Bucket,
-					Key:          req.Key,
-					UploadId:     uid,
-					RequestPayer: req.RequestPayer,
-				})
-				return f.shouldRetry(err)
+		fs.Debugf(o, "Cancelling multipart upload")
+		errCancel := f.pacer.Call(func() (bool, error) {
+			_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
+				Bucket:       req.Bucket,
+				Key:          req.Key,
+				UploadId:     uid,
+				RequestPayer: req.RequestPayer,
 			})
-			if errCancel != nil {
-				fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
-			}
+			return f.shouldRetry(err)
+		})
+		if errCancel != nil {
+			fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
 		}
-	}()
+	})()
 
 	var (
 		g, gCtx  = errgroup.WithContext(ctx)
@@ -2200,9 +2462,16 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 		tokens.Get()
 		buf := memPool.Get()
 
+		free := func() {
+			// return the memory and token
+			memPool.Put(buf)
+			tokens.Put()
+		}
+
 		// Fail fast, in case an errgroup managed function returns an error
 		// gCtx is cancelled. There is no point in uploading all the other parts.
 		if gCtx.Err() != nil {
+			free()
 			break
 		}
 
@@ -2211,10 +2480,12 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 		n, err = readers.ReadFill(in, buf) // this can never return 0, nil
 		if err == io.EOF {
 			if n == 0 && partNum != 1 { // end if no data and if not first chunk
+				free()
 				break
 			}
 			finished = true
 		} else if err != nil {
+			free()
 			return errors.Wrap(err, "multipart upload failed to read source")
 		}
 		buf = buf[:n]
@@ -2223,6 +2494,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 		fs.Debugf(o, "multipart upload starting chunk %d size %v offset %v/%v", partNum, fs.SizeSuffix(n), fs.SizeSuffix(off), fs.SizeSuffix(size))
 		off += int64(n)
 		g.Go(func() (err error) {
+			defer free()
 			partLength := int64(len(buf))
 
 			// create checksum of buffer for integrity checking
@@ -2260,11 +2532,6 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 
 				return false, nil
 			})
-
-			// return the memory and token
-			memPool.Put(buf[:partSize])
-			tokens.Put()
-
 			if err != nil {
 				return errors.Wrap(err, "multipart upload failed to upload part")
 			}
@@ -2350,12 +2617,50 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if o.fs.opt.ServerSideEncryption != "" {
 		req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
 	}
+	if o.fs.opt.SSECustomerAlgorithm != "" {
+		req.SSECustomerAlgorithm = &o.fs.opt.SSECustomerAlgorithm
+	}
+	if o.fs.opt.SSECustomerKey != "" {
+		req.SSECustomerKey = &o.fs.opt.SSECustomerKey
+	}
+	if o.fs.opt.SSECustomerKeyMD5 != "" {
+		req.SSECustomerKeyMD5 = &o.fs.opt.SSECustomerKeyMD5
+	}
 	if o.fs.opt.SSEKMSKeyID != "" {
 		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
 	}
 	if o.fs.opt.StorageClass != "" {
 		req.StorageClass = &o.fs.opt.StorageClass
 	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			req.CacheControl = aws.String(value)
+		case "content-disposition":
+			req.ContentDisposition = aws.String(value)
+		case "content-encoding":
+			req.ContentEncoding = aws.String(value)
+		case "content-language":
+			req.ContentLanguage = aws.String(value)
+		case "content-type":
+			req.ContentType = aws.String(value)
+		case "x-amz-tagging":
+			req.Tagging = aws.String(value)
+		default:
+			const amzMetaPrefix = "x-amz-meta-"
+			if strings.HasPrefix(lowerKey, amzMetaPrefix) {
+				metaKey := lowerKey[len(amzMetaPrefix):]
+				req.Metadata[metaKey] = aws.String(value)
+			} else {
+				fs.Errorf(o, "Don't know how to set key %q on upload", key)
+			}
+		}
+	}
 
 	if multipart {
 		err = o.uploadMultipart(ctx, &req, size, in)
@@ -2477,6 +2782,7 @@ var (
 	_ fs.Copier      = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.ListRer     = &Fs{}
+	_ fs.Commander   = &Fs{}
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
 	_ fs.GetTierer   = &Object{}

backend/seafile/api/types.go

@@ -0,0 +1,153 @@
+package api
+
+// Some api objects are duplicated with only small differences,
+// it's because the returned JSON objects are very inconsistent between api calls
+
+// AuthenticationRequest contains user credentials
+type AuthenticationRequest struct {
+	Username string `json:"username"`
+	Password string `json:"password"`
+}
+
+// AuthenticationResult is returned by a call to the authentication api
+type AuthenticationResult struct {
+	Token  string   `json:"token"`
+	Errors []string `json:"non_field_errors"`
+}
+
+// AccountInfo contains simple user properties
+type AccountInfo struct {
+	Usage int64  `json:"usage"`
+	Total int64  `json:"total"`
+	Email string `json:"email"`
+	Name  string `json:"name"`
+}
+
+// ServerInfo contains server information
+type ServerInfo struct {
+	Version string `json:"version"`
+}
+
+// DefaultLibrary when none specified
+type DefaultLibrary struct {
+	ID     string `json:"repo_id"`
+	Exists bool   `json:"exists"`
+}
+
+// CreateLibraryRequest contains the information needed to create a library
+type CreateLibraryRequest struct {
+	Name        string `json:"name"`
+	Description string `json:"desc"`
+	Password    string `json:"passwd"`
+}
+
+// Library properties. Please note not all properties are going to be useful for rclone
+type Library struct {
+	Encrypted bool   `json:"encrypted"`
+	Owner     string `json:"owner"`
+	ID        string `json:"id"`
+	Size      int    `json:"size"`
+	Name      string `json:"name"`
+	Modified  int64  `json:"mtime"`
+}
+
+// CreateLibrary properties. Seafile is not consistent and returns different types for different API calls
+type CreateLibrary struct {
+	ID   string `json:"repo_id"`
+	Name string `json:"repo_name"`
+}
+
+// FileType is either "dir" or "file"
+type FileType string
+
+// File types
+var (
+	FileTypeDir  FileType = "dir"
+	FileTypeFile FileType = "file"
+)
+
+// FileDetail contains file properties (for older api v2.0)
+type FileDetail struct {
+	ID       string   `json:"id"`
+	Type     FileType `json:"type"`
+	Name     string   `json:"name"`
+	Size     int64    `json:"size"`
+	Parent   string   `json:"parent_dir"`
+	Modified string   `json:"last_modified"`
+}
+
+// DirEntries contains a list of DirEntry
+type DirEntries struct {
+	Entries []DirEntry `json:"dirent_list"`
+}
+
+// DirEntry contains a directory entry
+type DirEntry struct {
+	ID       string   `json:"id"`
+	Type     FileType `json:"type"`
+	Name     string   `json:"name"`
+	Size     int64    `json:"size"`
+	Path     string   `json:"parent_dir"`
+	Modified int64    `json:"mtime"`
+}
+
+// Operation is move, copy or rename
+type Operation string
+
+// Operations
+var (
+	CopyFileOperation   Operation = "copy"
+	MoveFileOperation   Operation = "move"
+	RenameFileOperation Operation = "rename"
+)
+
+// FileOperationRequest is sent to the api to copy, move or rename a file
+type FileOperationRequest struct {
+	Operation            Operation `json:"operation"`
+	DestinationLibraryID string    `json:"dst_repo"` // For copy/move operation
+	DestinationPath      string    `json:"dst_dir"`  // For copy/move operation
+	NewName              string    `json:"newname"`  // Only to be used by the rename operation
+}
+
+// FileInfo is returned by a server file copy/move/rename (new api v2.1)
+type FileInfo struct {
+	Type      string `json:"type"`
+	LibraryID string `json:"repo_id"`
+	Path      string `json:"parent_dir"`
+	Name      string `json:"obj_name"`
+	ID        string `json:"obj_id"`
+	Size      int64  `json:"size"`
+}
+
+// CreateDirRequest only contain an operation field
+type CreateDirRequest struct {
+	Operation string `json:"operation"`
+}
+
+// DirectoryDetail contains the directory details specific to the getDirectoryDetails call
+type DirectoryDetail struct {
+	ID   string `json:"repo_id"`
+	Name string `json:"name"`
+	Path string `json:"path"`
+}
+
+// ShareLinkRequest contains the information needed to create or list shared links
+type ShareLinkRequest struct {
+	LibraryID string `json:"repo_id"`
+	Path      string `json:"path"`
+}
+
+// SharedLink contains the information returned by a call to shared link creation
+type SharedLink struct {
+	Link      string `json:"link"`
+	IsExpired bool   `json:"is_expired"`
+}
+
+// BatchSourceDestRequest contains JSON parameters for sending a batch copy or move operation
+type BatchSourceDestRequest struct {
+	SrcLibraryID string   `json:"src_repo_id"`
+	SrcParentDir string   `json:"src_parent_dir"`
+	SrcItems     []string `json:"src_dirents"`
+	DstLibraryID string   `json:"dst_repo_id"`
+	DstParentDir string   `json:"dst_parent_dir"`
+}

backend/seafile/object.go

@@ -0,0 +1,127 @@
+package seafile
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+)
+
+// Object describes a seafile object (also commonly called a file)
+type Object struct {
+	fs            *Fs       // what this object is part of
+	id            string    // internal ID of object
+	remote        string    // The remote path (full path containing library name if target at root)
+	pathInLibrary string    // Path of the object without the library name
+	size          int64     // size of the object
+	modTime       time.Time // modification time of the object
+	libraryID     string    // Needed to download the file
+}
+
+// ==================== Interface fs.DirEntry ====================
+
+// Return a string version
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+	return o.remote
+}
+
+// Remote returns the remote string
+func (o *Object) Remote() string {
+	return o.remote
+}
+
+// ModTime returns last modified time
+func (o *Object) ModTime(context.Context) time.Time {
+	return o.modTime
+}
+
+// Size returns the size of an object in bytes
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// ==================== Interface fs.ObjectInfo ====================
+
+// Fs returns the parent Fs
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, ty hash.Type) (string, error) {
+	return "", hash.ErrUnsupported
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+	return true
+}
+
+// ==================== Interface fs.Object ====================
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	return fs.ErrorCantSetModTime
+}
+
+// Open opens the file for read.  Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloser, error) {
+	downloadLink, err := o.fs.getDownloadLink(ctx, o.libraryID, o.pathInLibrary)
+	if err != nil {
+		return nil, err
+	}
+	reader, err := o.fs.download(ctx, downloadLink, o.Size(), options...)
+	if err != nil {
+		return nil, err
+	}
+	return reader, nil
+}
+
+// Update in to the object with the modTime given of the given size
+//
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
+// return an error or update the object properly (rather than e.g. calling panic).
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	// The upload sometimes return a temporary 500 error
+	// We cannot use the pacer to retry uploading the file as the upload link is single use only
+	for retry := 0; retry <= 3; retry++ {
+		uploadLink, err := o.fs.getUploadLink(ctx, o.libraryID)
+		if err != nil {
+			return err
+		}
+
+		uploaded, err := o.fs.upload(ctx, in, uploadLink, o.pathInLibrary)
+		if err == ErrorInternalDuringUpload {
+			// This is a temporary error, try again with a new upload link
+			continue
+		}
+		if err != nil {
+			return err
+		}
+		// Set the properties from the upload back to the object
+		o.size = uploaded.Size
+		o.id = uploaded.ID
+
+		return nil
+	}
+	return ErrorInternalDuringUpload
+}
+
+// Remove this object
+func (o *Object) Remove(ctx context.Context) error {
+	return o.fs.deleteFile(ctx, o.libraryID, o.pathInLibrary)
+}
+
+// ==================== Optional Interface fs.IDer ====================
+
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}

backend/seafile/pacer.go

@@ -0,0 +1,67 @@
+package seafile
+
+import (
+	"fmt"
+	"net/url"
+	"sync"
+	"time"
+
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/pacer"
+)
+
+const (
+	minSleep      = 100 * time.Millisecond
+	maxSleep      = 10 * time.Second
+	decayConstant = 2 // bigger for slower decay, exponential
+)
+
+// Use only one pacer per server URL
+var (
+	pacers     map[string]*fs.Pacer
+	pacerMutex sync.Mutex
+)
+
+func init() {
+	pacers = make(map[string]*fs.Pacer, 0)
+}
+
+// getPacer returns the unique pacer for that remote URL
+func getPacer(remote string) *fs.Pacer {
+	pacerMutex.Lock()
+	defer pacerMutex.Unlock()
+
+	remote = parseRemote(remote)
+	if existing, found := pacers[remote]; found {
+		return existing
+	}
+
+	pacers[remote] = fs.NewPacer(
+		pacer.NewDefault(
+			pacer.MinSleep(minSleep),
+			pacer.MaxSleep(maxSleep),
+			pacer.DecayConstant(decayConstant),
+		),
+	)
+	return pacers[remote]
+}
+
+// parseRemote formats a remote url into "hostname:port"
+func parseRemote(remote string) string {
+	remoteURL, err := url.Parse(remote)
+	if err != nil {
+		// Return a default value in the very unlikely event we're not going to parse remote
+		fs.Infof(nil, "Cannot parse remote %s", remote)
+		return "default"
+	}
+	host := remoteURL.Hostname()
+	port := remoteURL.Port()
+	if port == "" {
+		if remoteURL.Scheme == "https" {
+			port = "443"
+		} else {
+			port = "80"
+		}
+	}
+	return fmt.Sprintf("%s:%s", host, port)
+}

backend/seafile/seafile_internal_test.go

@@ -0,0 +1,123 @@
+package seafile
+
+import (
+	"path"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type pathData struct {
+	configLibrary   string // Library specified in the config
+	configRoot      string // Root directory specified in the config
+	argumentPath    string // Path given as an argument in the command line
+	expectedLibrary string
+	expectedPath    string
+}
+
+// Test the method to split a library name and a path
+// from a mix of configuration data and path command line argument
+func TestSplitPath(t *testing.T) {
+	testData := []pathData{
+		pathData{
+			configLibrary:   "",
+			configRoot:      "",
+			argumentPath:    "",
+			expectedLibrary: "",
+			expectedPath:    "",
+		},
+		pathData{
+			configLibrary:   "",
+			configRoot:      "",
+			argumentPath:    "Library",
+			expectedLibrary: "Library",
+			expectedPath:    "",
+		},
+		pathData{
+			configLibrary:   "",
+			configRoot:      "",
+			argumentPath:    path.Join("Library", "path", "to", "file"),
+			expectedLibrary: "Library",
+			expectedPath:    path.Join("path", "to", "file"),
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      "",
+			argumentPath:    "",
+			expectedLibrary: "Library",
+			expectedPath:    "",
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      "",
+			argumentPath:    "path",
+			expectedLibrary: "Library",
+			expectedPath:    "path",
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      "",
+			argumentPath:    path.Join("path", "to", "file"),
+			expectedLibrary: "Library",
+			expectedPath:    path.Join("path", "to", "file"),
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      "root",
+			argumentPath:    "",
+			expectedLibrary: "Library",
+			expectedPath:    "root",
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      path.Join("root", "path"),
+			argumentPath:    "",
+			expectedLibrary: "Library",
+			expectedPath:    path.Join("root", "path"),
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      "root",
+			argumentPath:    "path",
+			expectedLibrary: "Library",
+			expectedPath:    path.Join("root", "path"),
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      "root",
+			argumentPath:    path.Join("path", "to", "file"),
+			expectedLibrary: "Library",
+			expectedPath:    path.Join("root", "path", "to", "file"),
+		},
+		pathData{
+			configLibrary:   "Library",
+			configRoot:      path.Join("root", "path"),
+			argumentPath:    path.Join("subpath", "to", "file"),
+			expectedLibrary: "Library",
+			expectedPath:    path.Join("root", "path", "subpath", "to", "file"),
+		},
+	}
+	for _, test := range testData {
+		fs := &Fs{
+			libraryName:   test.configLibrary,
+			rootDirectory: test.configRoot,
+		}
+		libraryName, path := fs.splitPath(test.argumentPath)
+
+		assert.Equal(t, test.expectedLibrary, libraryName)
+		assert.Equal(t, test.expectedPath, path)
+	}
+}
+
+func TestSplitPathIntoSlice(t *testing.T) {
+	testData := map[string][]string{
+		"1":     {"1"},
+		"/1":    {"1"},
+		"/1/":   {"1"},
+		"1/2/3": {"1", "2", "3"},
+	}
+	for input, expected := range testData {
+		output := splitPath(input)
+		assert.Equal(t, expected, output)
+	}
+}

backend/seafile/seafile_test.go

@@ -0,0 +1,17 @@
+// Test Seafile filesystem interface
+package seafile_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/seafile"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSeafile:",
+		NilObject:  (*seafile.Object)(nil),
+	})
+}

backend/sftp/sftp.go

@@ -69,9 +69,12 @@ func init() {
 			Name:       "pass",
 			Help:       "SSH password, leave blank to use ssh-agent.",
 			IsPassword: true,
+		}, {
+			Name: "key_pem",
+			Help: "Raw PEM-encoded private key, If specified, will override key_file parameter.",
 		}, {
 			Name: "key_file",
-			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
+			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent." + env.ShellExpandHelp,
 		}, {
 			Name: "key_file_pass",
 			Help: `The passphrase to decrypt the PEM-encoded private key file.
@@ -91,7 +94,7 @@ when the ssh-agent contains many keys.`,
 			Name: "use_insecure_cipher",
 			Help: `Enable the use of insecure ciphers and key exchange methods. 
 
-This enables the use of the the following insecure ciphers and key exchange methods:
+This enables the use of the following insecure ciphers and key exchange methods:
 
 - aes128-cbc
 - aes192-cbc
@@ -172,6 +175,7 @@ type Options struct {
 	User              string `config:"user"`
 	Port              string `config:"port"`
 	Pass              string `config:"pass"`
+	KeyPem            string `config:"key_pem"`
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
 	KeyUseAgent       bool   `config:"key_use_agent"`
@@ -390,8 +394,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
-	// Add ssh agent-auth if no password or file specified
-	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword) || opt.KeyUseAgent {
+	//keyPem := env.ShellExpand(opt.KeyPem)
+	// Add ssh agent-auth if no password or file or key PEM specified
+	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword && opt.KeyPem == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -427,10 +432,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Load key file if specified
-	if keyFile != "" {
-		key, err := ioutil.ReadFile(keyFile)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to read private key file")
+	if keyFile != "" || opt.KeyPem != "" {
+		var key []byte
+		if opt.KeyPem == "" {
+			key, err = ioutil.ReadFile(keyFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to read private key file")
+			}
+		} else {
+			// wrap in quotes because the config is a coming as a literal without them.
+			opt.KeyPem, err = strconv.Unquote("\"" + opt.KeyPem + "\"")
+			if err != nil {
+				return nil, errors.Wrap(err, "pem key not formatted properly")
+			}
+			key = []byte(opt.KeyPem)
 		}
 		clearpass := ""
 		if opt.KeyFilePass != "" {
@@ -470,7 +485,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return NewFsWithConnection(ctx, name, root, m, opt, sshConfig)
 }
 
-// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
+// NewFsWithConnection creates a new Fs object from the name and root and an ssh.ClientConfig. It connects to
 // the host specified in the ssh.ClientConfig
 func NewFsWithConnection(ctx context.Context, name string, root string, m configmap.Mapper, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
@@ -1021,7 +1036,7 @@ func parseHash(bytes []byte) string {
 
 // Parses the byte array output from the SSH session
 // returned by an invocation of df into
-// the disk size, used space, and avaliable space on the disk, in that order.
+// the disk size, used space, and available space on the disk, in that order.
 // Only works when `df` has output info on only one disk
 func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
 	spaceTotal, spaceUsed, spaceAvail = -1, -1, -1
@@ -1102,19 +1117,18 @@ func (o *Object) stat() error {
 //
 // it also updates the info field
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	if !o.fs.opt.SetModTime {
-		return nil
+	if o.fs.opt.SetModTime {
+		c, err := o.fs.getSftpConnection()
+		if err != nil {
+			return errors.Wrap(err, "SetModTime")
+		}
+		err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
+		o.fs.putSftpConnection(&c, err)
+		if err != nil {
+			return errors.Wrap(err, "SetModTime failed")
+		}
 	}
-	c, err := o.fs.getSftpConnection()
-	if err != nil {
-		return errors.Wrap(err, "SetModTime")
-	}
-	err = c.sftpClient.Chtimes(o.path(), modTime, modTime)
-	o.fs.putSftpConnection(&c, err)
-	if err != nil {
-		return errors.Wrap(err, "SetModTime failed")
-	}
-	err = o.stat()
+	err := o.stat()
 	if err != nil {
 		return errors.Wrap(err, "SetModTime stat failed")
 	}

backend/sharefile/api/types.go

@@ -102,7 +102,7 @@ type UploadSpecification struct {
 	MaxNumberOfThreads int    `json:"MaxNumberOfThreads"` // Specifies the max number of chunks that can be sent simultaneously for threaded uploads
 }
 
-// UploadFinishResponse is returnes from calling UploadSpecification.FinishURI
+// UploadFinishResponse is returns from calling UploadSpecification.FinishURI
 type UploadFinishResponse struct {
 	Error        bool   `json:"error"`
 	ErrorMessage string `json:"errorMessage"`

backend/sharefile/sharefile.go

@@ -152,7 +152,10 @@ func init() {
 				oauthConfig.Endpoint.TokenURL = endpoint + tokenPath
 				return nil
 			}
-			err := oauthutil.ConfigWithCallback("sharefile", name, m, oauthConfig, checkAuth)
+			opt := oauthutil.Options{
+				CheckAuth: checkAuth,
+			}
+			err := oauthutil.Config("sharefile", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -347,7 +350,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string, directoriesOnly b
 
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -697,10 +700,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -740,7 +739,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -796,13 +795,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -814,10 +807,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1030,75 +1019,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
-
-	// Find ID of src parent, not creating subdirs
-	var srcLeaf, srcDirectoryID string
-	findPath = srcRemote
-	if srcRemote == "" {
-		findPath = srcFs.root
-	}
-	srcLeaf, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
+	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	_, err = f.move(ctx, false, srcID, srcLeaf, leaf, srcDirectoryID, directoryID)
+	_, err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -1399,7 +1327,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	isLargeFile := size < 0 || size > int64(o.fs.opt.UploadCutoff)
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
@@ -1429,8 +1357,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	var resp *http.Response
 	var info api.UploadSpecification
 	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/Items(" + directoryID + ")/Upload2",
+		Method:  "POST",
+		Path:    "/Items(" + directoryID + ")/Upload2",
+		Options: options,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(ctx, &opts, &req, &info)

backend/sugarsync/sugarsync.go

@@ -230,7 +230,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an sugarsync 'url'
+// parsePath parses a sugarsync 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -255,7 +255,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.File, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -663,10 +663,6 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -709,7 +705,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -733,7 +729,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return existingObj, existingObj.Update(ctx, in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
-		return f.PutUnchecked(ctx, in, src)
+		return f.PutUnchecked(ctx, in, src, options...)
 	default:
 		return nil, err
 	}
@@ -765,13 +761,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -807,10 +797,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	directoryID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1033,64 +1019,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	err = f.moveDir(ctx, srcID, leaf, directoryID)
+	err = f.moveDir(ctx, srcID, dstLeaf, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -1099,7 +1035,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	obj, err := f.NewObject(ctx, remote)
 	if err != nil {
 		return "", err
@@ -1289,7 +1225,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
@@ -1320,6 +1256,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		RootURL:    o.id,
 		Path:       "/data",
 		NoResponse: true,
+		Options:    options,
 		Body:       in,
 	}
 	if size >= 0 {

backend/swift/swift.go

@@ -73,7 +73,7 @@ copy operations.`,
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "swift",
-		Description: "Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
+		Description: "OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
 		NewFs:       NewFs,
 		Options: append([]fs.Option{{
 			Name:    "env_auth",
@@ -284,7 +284,7 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
 func shouldRetry(err error) (bool, error) {
-	// If this is an swift.Error object extract the HTTP error code
+	// If this is a swift.Error object extract the HTTP error code
 	if swiftError, ok := err.(*swift.Error); ok {
 		for _, e := range retryErrorCodes {
 			if swiftError.StatusCode == e {
@@ -1124,6 +1124,9 @@ func (o *Object) removeSegments(except string) error {
 	// remove the segments container if empty, ignore errors
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.c.ContainerDelete(segmentsContainer)
+		if err == swift.ContainerNotFound || err == swift.ContainerNotEmpty {
+			return false, err
+		}
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -1253,7 +1256,7 @@ func deleteChunks(o *Object, segmentsContainer string, segmentInfos []string) {
 			fs.Debugf(o, "Delete segment file %q on %q", v, segmentsContainer)
 			e := o.fs.c.ObjectDelete(segmentsContainer, v)
 			if e != nil {
-				fs.Errorf(o, "Error occured in delete segment file %q on %q , error: %q", v, segmentsContainer, e)
+				fs.Errorf(o, "Error occurred in delete segment file %q on %q, error: %q", v, segmentsContainer, e)
 			}
 		}
 	}
@@ -1285,6 +1288,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	m.SetModTime(modTime)
 	contentType := fs.MimeType(ctx, src)
 	headers := m.ObjectHeaders()
+	fs.OpenOptionAddHeaders(options, headers)
 	uniquePrefix := ""
 	if size > int64(o.fs.opt.ChunkSize) || (size == -1 && !o.fs.opt.NoChunk) {
 		uniquePrefix, err = o.updateChunks(in, headers, size, contentType)

backend/tardigrade/fs.go

@@ -0,0 +1,686 @@
+// +build go1.13,!plan9
+
+// Package tardigrade provides an interface to Tardigrade decentralized object storage.
+package tardigrade
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
+	"github.com/rclone/rclone/fs/config/configmap"
+	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/fserrors"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/bucket"
+	"golang.org/x/text/unicode/norm"
+
+	"storj.io/uplink"
+)
+
+const (
+	existingProvider = "existing"
+	newProvider      = "new"
+)
+
+var satMap = map[string]string{
+	"us-central-1.tardigrade.io":  "12EayRS2V1kEsWESU9QMRseFhdxYxKicsiFmxrsLZHeLUtdps3S@us-central-1.tardigrade.io:7777",
+	"europe-west-1.tardigrade.io": "12L9ZFwhzVpuEKMUNUqkaTLGzwY9G24tbiigLiXpmZWKwmcNDDs@europe-west-1.tardigrade.io:7777",
+	"asia-east-1.tardigrade.io":   "121RTSDpyNZVcEU84Ticf2L1ntiuUimbWgfATz21tuvgk3vzoA6@asia-east-1.tardigrade.io:7777",
+}
+
+// Register with Fs
+func init() {
+	fs.Register(&fs.RegInfo{
+		Name:        "tardigrade",
+		Description: "Tardigrade Decentralized Cloud Storage",
+		NewFs:       NewFs,
+		Config: func(name string, configMapper configmap.Mapper) {
+			provider, _ := configMapper.Get(fs.ConfigProvider)
+
+			config.FileDeleteKey(name, fs.ConfigProvider)
+
+			if provider == newProvider {
+				satelliteString, _ := configMapper.Get("satellite_address")
+				apiKey, _ := configMapper.Get("api_key")
+				passphrase, _ := configMapper.Get("passphrase")
+
+				// satelliteString contains always default and passphrase can be empty
+				if apiKey == "" {
+					return
+				}
+
+				satellite, found := satMap[satelliteString]
+				if !found {
+					satellite = satelliteString
+				}
+
+				access, err := uplink.RequestAccessWithPassphrase(context.TODO(), satellite, apiKey, passphrase)
+				if err != nil {
+					log.Fatalf("Couldn't create access grant: %v", err)
+				}
+
+				serialziedAccess, err := access.Serialize()
+				if err != nil {
+					log.Fatalf("Couldn't serialize access grant: %v", err)
+				}
+				configMapper.Set("satellite_address", satellite)
+				configMapper.Set("access_grant", serialziedAccess)
+			} else if provider == existingProvider {
+				config.FileDeleteKey(name, "satellite_address")
+				config.FileDeleteKey(name, "api_key")
+				config.FileDeleteKey(name, "passphrase")
+			} else {
+				log.Fatalf("Invalid provider type: %s", provider)
+			}
+		},
+		Options: []fs.Option{
+			{
+				Name:     fs.ConfigProvider,
+				Help:     "Choose an authentication method.",
+				Required: true,
+				Default:  existingProvider,
+				Examples: []fs.OptionExample{{
+					Value: "existing",
+					Help:  "Use an existing access grant.",
+				}, {
+					Value: newProvider,
+					Help:  "Create a new access grant from satellite address, API key, and passphrase.",
+				},
+				}},
+			{
+				Name:     "access_grant",
+				Help:     "Access Grant.",
+				Required: false,
+				Provider: "existing",
+			},
+			{
+				Name:     "satellite_address",
+				Help:     "Satellite Address. Custom satellite address should match the format: `<nodeid>@<address>:<port>`.",
+				Required: false,
+				Provider: newProvider,
+				Default:  "us-central-1.tardigrade.io",
+				Examples: []fs.OptionExample{{
+					Value: "us-central-1.tardigrade.io",
+					Help:  "US Central 1",
+				}, {
+					Value: "europe-west-1.tardigrade.io",
+					Help:  "Europe West 1",
+				}, {
+					Value: "asia-east-1.tardigrade.io",
+					Help:  "Asia East 1",
+				},
+				},
+			},
+			{
+				Name:     "api_key",
+				Help:     "API Key.",
+				Required: false,
+				Provider: newProvider,
+			},
+			{
+				Name:     "passphrase",
+				Help:     "Encryption Passphrase. To access existing objects enter passphrase used for uploading.",
+				Required: false,
+				Provider: newProvider,
+			},
+		},
+	})
+}
+
+// Options defines the configuration for this backend
+type Options struct {
+	Access string `config:"access_grant"`
+
+	SatelliteAddress string `config:"satellite_address"`
+	APIKey           string `config:"api_key"`
+	Passphrase       string `config:"passphrase"`
+}
+
+// Fs represents a remote to Tardigrade
+type Fs struct {
+	name string // the name of the remote
+	root string // root of the filesystem
+
+	opts     Options      // parsed options
+	features *fs.Features // optional features
+
+	access *uplink.Access // parsed scope
+
+	project *uplink.Project // project client
+}
+
+// Check the interfaces are satisfied.
+var (
+	_ fs.Fs          = &Fs{}
+	_ fs.ListRer     = &Fs{}
+	_ fs.PutStreamer = &Fs{}
+)
+
+// NewFs creates a filesystem backed by Tardigrade.
+func NewFs(name, root string, m configmap.Mapper) (_ fs.Fs, err error) {
+	ctx := context.Background()
+
+	// Setup filesystem and connection to Tardigrade
+	root = norm.NFC.String(root)
+	root = strings.Trim(root, "/")
+
+	f := &Fs{
+		name: name,
+		root: root,
+	}
+
+	// Parse config into Options struct
+	err = configstruct.Set(m, &f.opts)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse access
+	var access *uplink.Access
+
+	if f.opts.Access != "" {
+		access, err = uplink.ParseAccess(f.opts.Access)
+		if err != nil {
+			return nil, errors.Wrap(err, "tardigrade: access")
+		}
+	}
+
+	if access == nil && f.opts.SatelliteAddress != "" && f.opts.APIKey != "" && f.opts.Passphrase != "" {
+		access, err = uplink.RequestAccessWithPassphrase(ctx, f.opts.SatelliteAddress, f.opts.APIKey, f.opts.Passphrase)
+		if err != nil {
+			return nil, errors.Wrap(err, "tardigrade: access")
+		}
+
+		serializedAccess, err := access.Serialize()
+		if err != nil {
+			return nil, errors.Wrap(err, "tardigrade: access")
+		}
+
+		err = config.SetValueAndSave(f.name, "access_grant", serializedAccess)
+		if err != nil {
+			return nil, errors.Wrap(err, "tardigrade: access")
+		}
+	}
+
+	if access == nil {
+		return nil, errors.New("access not found")
+	}
+
+	f.access = access
+
+	f.features = (&fs.Features{
+		BucketBased:       true,
+		BucketBasedRootOK: true,
+	}).Fill(f)
+
+	project, err := f.connect(ctx)
+	if err != nil {
+		return nil, err
+	}
+	f.project = project
+
+	// Root validation needs to check the following: If a bucket path is
+	// specified and exists, then the object must be a directory.
+	//
+	// NOTE: At this point this must return the filesystem object we've
+	// created so far even if there is an error.
+	if root != "" {
+		bucketName, bucketPath := bucket.Split(root)
+
+		if bucketName != "" && bucketPath != "" {
+			_, err = project.StatBucket(ctx, bucketName)
+			if err != nil {
+				return f, errors.Wrap(err, "tardigrade: bucket")
+			}
+
+			object, err := project.StatObject(ctx, bucketName, bucketPath)
+			if err == nil {
+				if !object.IsPrefix {
+					// If the root is actually a file we
+					// need to return the *parent*
+					// directory of the root instead and an
+					// error that the original root
+					// requested is a file.
+					newRoot := path.Dir(f.root)
+					if newRoot == "." {
+						newRoot = ""
+					}
+					f.root = newRoot
+
+					return f, fs.ErrorIsFile
+				}
+			}
+		}
+	}
+
+	return f, nil
+}
+
+// connect opens a connection to Tardigrade.
+func (f *Fs) connect(ctx context.Context) (project *uplink.Project, err error) {
+	fs.Debugf(f, "connecting...")
+	defer fs.Debugf(f, "connected: %+v", err)
+
+	cfg := uplink.Config{
+		UserAgent: "rclone",
+	}
+
+	project, err = cfg.OpenProject(ctx, f.access)
+	if err != nil {
+		return nil, errors.Wrap(err, "tardigrade: project")
+	}
+
+	return
+}
+
+// absolute computes the absolute bucket name and path from the filesystem root
+// and the relative path provided.
+func (f *Fs) absolute(relative string) (bucketName, bucketPath string) {
+	bn, bp := bucket.Split(path.Join(f.root, relative))
+
+	// NOTE: Technically libuplink does not care about the encoding. It is
+	// happy to work with them as opaque byte sequences. However, rclone
+	// has a test that requires two paths with the same normalized form
+	// (but different un-normalized forms) to point to the same file. This
+	// means we have to normalize before we interact with libuplink.
+	return norm.NFC.String(bn), norm.NFC.String(bp)
+}
+
+// Name of the remote (as passed into NewFs)
+func (f *Fs) Name() string {
+	return f.name
+}
+
+// Root of the remote (as passed into NewFs)
+func (f *Fs) Root() string {
+	return f.root
+}
+
+// String returns a description of the FS
+func (f *Fs) String() string {
+	return fmt.Sprintf("FS sj://%s", f.root)
+}
+
+// Precision of the ModTimes in this Fs
+func (f *Fs) Precision() time.Duration {
+	return time.Nanosecond
+}
+
+// Hashes returns the supported hash types of the filesystem.
+func (f *Fs) Hashes() hash.Set {
+	return hash.NewHashSet()
+}
+
+// Features returns the optional features of this Fs
+func (f *Fs) Features() *fs.Features {
+	return f.features
+}
+
+// List the objects and directories in relative into entries. The entries can
+// be returned in any order but should be for a complete directory.
+//
+// relative should be "" to list the root, and should not have trailing
+// slashes.
+//
+// This should return fs.ErrDirNotFound if the directory isn't found.
+func (f *Fs) List(ctx context.Context, relative string) (entries fs.DirEntries, err error) {
+	fs.Debugf(f, "ls ./%s", relative)
+
+	bucketName, bucketPath := f.absolute(relative)
+
+	defer func() {
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			err = fs.ErrorDirNotFound
+		}
+	}()
+
+	if bucketName == "" {
+		if bucketPath != "" {
+			return nil, fs.ErrorListBucketRequired
+		}
+
+		return f.listBuckets(ctx)
+	}
+
+	return f.listObjects(ctx, relative, bucketName, bucketPath)
+}
+
+func (f *Fs) listBuckets(ctx context.Context) (entries fs.DirEntries, err error) {
+	fs.Debugf(f, "BKT ls")
+
+	buckets := f.project.ListBuckets(ctx, nil)
+
+	for buckets.Next() {
+		bucket := buckets.Item()
+
+		entries = append(entries, fs.NewDir(bucket.Name, bucket.Created))
+	}
+
+	return entries, buckets.Err()
+}
+
+// newDirEntry creates a directory entry from an uplink object.
+//
+// NOTE: Getting the exact behavior required by rclone is somewhat tricky. The
+// path manipulation here is necessary to cover all the different ways the
+// filesystem and object could be initialized and combined.
+func (f *Fs) newDirEntry(relative, prefix string, object *uplink.Object) fs.DirEntry {
+	if object.IsPrefix {
+		//                         . The entry must include the relative path as its prefix. Depending on
+		//                         | what is being listed and how the filesystem root was initialized the
+		//                         | relative path may be empty (and so we use path joining here to ensure
+		//                         | we don't end up with an empty path segment).
+		//                         |
+		//                         |                    . Remove the prefix used during listing.
+		//                         |                    |
+		//                         |                    |           . Remove the trailing slash.
+		//                         |                    |           |
+		//                         v                    v           v
+		return fs.NewDir(path.Join(relative, object.Key[len(prefix):len(object.Key)-1]), object.System.Created)
+	}
+
+	return newObjectFromUplink(f, relative, object)
+}
+
+func (f *Fs) listObjects(ctx context.Context, relative, bucketName, bucketPath string) (entries fs.DirEntries, err error) {
+	fs.Debugf(f, "OBJ ls ./%s (%q, %q)", relative, bucketName, bucketPath)
+
+	opts := &uplink.ListObjectsOptions{
+		Prefix: newPrefix(bucketPath),
+
+		System: true,
+		Custom: true,
+	}
+	fs.Debugf(f, "opts %+v", opts)
+
+	objects := f.project.ListObjects(ctx, bucketName, opts)
+
+	for objects.Next() {
+		entries = append(entries, f.newDirEntry(relative, opts.Prefix, objects.Item()))
+	}
+
+	err = objects.Err()
+	if err != nil {
+		return nil, err
+	}
+
+	return entries, nil
+}
+
+// ListR lists the objects and directories of the Fs starting from dir
+// recursively into out.
+//
+// relative should be "" to start from the root, and should not have trailing
+// slashes.
+//
+// This should return ErrDirNotFound if the directory isn't found.
+//
+// It should call callback for each tranche of entries read. These need not be
+// returned in any particular order. If callback returns an error then the
+// listing will stop immediately.
+//
+// Don't implement this unless you have a more efficient way of listing
+// recursively that doing a directory traversal.
+func (f *Fs) ListR(ctx context.Context, relative string, callback fs.ListRCallback) (err error) {
+	fs.Debugf(f, "ls -R ./%s", relative)
+
+	bucketName, bucketPath := f.absolute(relative)
+
+	defer func() {
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			err = fs.ErrorDirNotFound
+		}
+	}()
+
+	if bucketName == "" {
+		if bucketPath != "" {
+			return fs.ErrorListBucketRequired
+		}
+
+		return f.listBucketsR(ctx, callback)
+	}
+
+	return f.listObjectsR(ctx, relative, bucketName, bucketPath, callback)
+}
+
+func (f *Fs) listBucketsR(ctx context.Context, callback fs.ListRCallback) (err error) {
+	fs.Debugf(f, "BKT ls -R")
+
+	buckets := f.project.ListBuckets(ctx, nil)
+
+	for buckets.Next() {
+		bucket := buckets.Item()
+
+		err = f.listObjectsR(ctx, bucket.Name, bucket.Name, "", callback)
+		if err != nil {
+			return err
+		}
+	}
+
+	return buckets.Err()
+}
+
+func (f *Fs) listObjectsR(ctx context.Context, relative, bucketName, bucketPath string, callback fs.ListRCallback) (err error) {
+	fs.Debugf(f, "OBJ ls -R ./%s (%q, %q)", relative, bucketName, bucketPath)
+
+	opts := &uplink.ListObjectsOptions{
+		Prefix:    newPrefix(bucketPath),
+		Recursive: true,
+
+		System: true,
+		Custom: true,
+	}
+
+	objects := f.project.ListObjects(ctx, bucketName, opts)
+
+	for objects.Next() {
+		object := objects.Item()
+
+		err = callback(fs.DirEntries{f.newDirEntry(relative, opts.Prefix, object)})
+		if err != nil {
+			return err
+		}
+	}
+
+	err = objects.Err()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// NewObject finds the Object at relative. If it can't be found it returns the
+// error ErrorObjectNotFound.
+func (f *Fs) NewObject(ctx context.Context, relative string) (_ fs.Object, err error) {
+	fs.Debugf(f, "stat ./%s", relative)
+
+	bucketName, bucketPath := f.absolute(relative)
+
+	object, err := f.project.StatObject(ctx, bucketName, bucketPath)
+	if err != nil {
+		fs.Debugf(f, "err: %+v", err)
+
+		if errors.Is(err, uplink.ErrObjectNotFound) {
+			return nil, fs.ErrorObjectNotFound
+		}
+		return nil, err
+	}
+
+	return newObjectFromUplink(f, relative, object), nil
+}
+
+// Put in to the remote path with the modTime given of the given size
+//
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Put should
+// either return an error or upload it properly (rather than e.g. calling
+// panic).
+//
+// May create the object even if it returns an error - if so will return the
+// object and the error, otherwise will return nil and the error
+func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (_ fs.Object, err error) {
+	fs.Debugf(f, "cp input ./%s # %+v %d", src.Remote(), options, src.Size())
+
+	// Reject options we don't support.
+	for _, option := range options {
+		if option.Mandatory() {
+			fs.Errorf(f, "Unsupported mandatory option: %v", option)
+
+			return nil, errors.New("unsupported mandatory option")
+		}
+	}
+
+	bucketName, bucketPath := f.absolute(src.Remote())
+
+	upload, err := f.project.UploadObject(ctx, bucketName, bucketPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			aerr := upload.Abort()
+			if aerr != nil {
+				fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr)
+			}
+		}
+	}()
+
+	err = upload.SetCustomMetadata(ctx, uplink.CustomMetadata{
+		"rclone:mtime": src.ModTime(ctx).Format(time.RFC3339Nano),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = io.Copy(upload, in)
+	if err != nil {
+		err = fserrors.RetryError(err)
+		fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err)
+
+		return nil, err
+	}
+
+	err = upload.Commit()
+	if err != nil {
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			// Rclone assumes the backend will create the bucket if not existing yet.
+			// Here we create the bucket and return a retry error for rclone to retry the upload.
+			_, err = f.project.EnsureBucket(ctx, bucketName)
+			if err != nil {
+				return nil, err
+			}
+			err = fserrors.RetryError(errors.New("bucket was not available, now created, the upload must be retried"))
+		}
+		return nil, err
+	}
+
+	return newObjectFromUplink(f, "", upload.Info()), nil
+}
+
+// PutStream uploads to the remote path with the modTime given of indeterminate
+// size.
+//
+// May create the object even if it returns an error - if so will return the
+// object and the error, otherwise will return nil and the error.
+func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (_ fs.Object, err error) {
+	return f.Put(ctx, in, src, options...)
+}
+
+// Mkdir makes the directory (container, bucket)
+//
+// Shouldn't return an error if it already exists
+func (f *Fs) Mkdir(ctx context.Context, relative string) (err error) {
+	fs.Debugf(f, "mkdir -p ./%s", relative)
+
+	bucketName, _ := f.absolute(relative)
+
+	_, err = f.project.EnsureBucket(ctx, bucketName)
+
+	return err
+}
+
+// Rmdir removes the directory (container, bucket)
+//
+// NOTE: Despite code documentation to the contrary, this method should not
+// return an error if the directory does not exist.
+func (f *Fs) Rmdir(ctx context.Context, relative string) (err error) {
+	fs.Debugf(f, "rmdir ./%s", relative)
+
+	bucketName, bucketPath := f.absolute(relative)
+
+	if bucketPath != "" {
+		// If we can successfully stat it, then it is an object (and not a prefix).
+		_, err := f.project.StatObject(ctx, bucketName, bucketPath)
+		if err != nil {
+			if errors.Is(err, uplink.ErrObjectNotFound) {
+				// At this point we know it is not an object,
+				// but we don't know if it is a prefix for one.
+				//
+				// We check this by doing a listing and if we
+				// get any results back, then we know this is a
+				// valid prefix (which implies the directory is
+				// not empty).
+				opts := &uplink.ListObjectsOptions{
+					Prefix: newPrefix(bucketPath),
+
+					System: true,
+					Custom: true,
+				}
+
+				objects := f.project.ListObjects(ctx, bucketName, opts)
+
+				if objects.Next() {
+					return fs.ErrorDirectoryNotEmpty
+				}
+
+				return objects.Err()
+			}
+
+			return err
+		}
+
+		return fs.ErrorIsFile
+	}
+
+	_, err = f.project.DeleteBucket(ctx, bucketName)
+	if err != nil {
+		if errors.Is(err, uplink.ErrBucketNotFound) {
+			return fs.ErrorDirNotFound
+		}
+
+		if errors.Is(err, uplink.ErrBucketNotEmpty) {
+			return fs.ErrorDirectoryNotEmpty
+		}
+
+		return err
+	}
+
+	return nil
+}
+
+// newPrefix returns a new prefix for listing conforming to the libuplink
+// requirements. In particular, libuplink requires a trailing slash for
+// listings, but rclone does not always provide one. Further, depending on how
+// the path was initially path normalization may have removed it (e.g. a
+// trailing slash from the CLI is removed before it ever gets to the backend
+// code).
+func newPrefix(prefix string) string {
+	if prefix == "" {
+		return prefix
+	}
+
+	if prefix[len(prefix)-1] == '/' {
+		return prefix
+	}
+
+	return prefix + "/"
+}

backend/tardigrade/object.go

@@ -0,0 +1,204 @@
+// +build go1.13,!plan9
+
+package tardigrade
+
+import (
+	"context"
+	"io"
+	"path"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/bucket"
+	"golang.org/x/text/unicode/norm"
+
+	"storj.io/uplink"
+)
+
+// Object describes a Tardigrade object
+type Object struct {
+	fs *Fs
+
+	absolute string
+
+	size     int64
+	created  time.Time
+	modified time.Time
+}
+
+// Check the interfaces are satisfied.
+var _ fs.Object = &Object{}
+
+// newObjectFromUplink creates a new object from a Tardigrade uplink object.
+func newObjectFromUplink(f *Fs, relative string, object *uplink.Object) *Object {
+	// Attempt to use the modified time from the metadata. Otherwise
+	// fallback to the server time.
+	modified := object.System.Created
+
+	if modifiedStr, ok := object.Custom["rclone:mtime"]; ok {
+		var err error
+
+		modified, err = time.Parse(time.RFC3339Nano, modifiedStr)
+		if err != nil {
+			modified = object.System.Created
+		}
+	}
+
+	bucketName, _ := bucket.Split(path.Join(f.root, relative))
+
+	return &Object{
+		fs: f,
+
+		absolute: norm.NFC.String(bucketName + "/" + object.Key),
+
+		size:     object.System.ContentLength,
+		created:  object.System.Created,
+		modified: modified,
+	}
+}
+
+// String returns a description of the Object
+func (o *Object) String() string {
+	if o == nil {
+		return "<nil>"
+	}
+
+	return o.Remote()
+}
+
+// Remote returns the remote path
+func (o *Object) Remote() string {
+	// It is possible that we have an empty root (meaning the filesystem is
+	// rooted at the project level). In this case the relative path is just
+	// the full absolute path to the object (including the bucket name).
+	if o.fs.root == "" {
+		return o.absolute
+	}
+
+	// At this point we know that the filesystem itself is at least a
+	// bucket name (and possibly a prefix path).
+	//
+	//                               . This is necessary to remove the slash.
+	//                               |
+	//                               v
+	return o.absolute[len(o.fs.root)+1:]
+}
+
+// ModTime returns the modification date of the file
+// It should return a best guess if one isn't available
+func (o *Object) ModTime(ctx context.Context) time.Time {
+	return o.modified
+}
+
+// Size returns the size of the file
+func (o *Object) Size() int64 {
+	return o.size
+}
+
+// Fs returns read only access to the Fs that this object is part of
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+// Hash returns the selected checksum of the file
+// If no checksum is available it returns ""
+func (o *Object) Hash(ctx context.Context, ty hash.Type) (_ string, err error) {
+	fs.Debugf(o, "%s", ty)
+
+	return "", hash.ErrUnsupported
+}
+
+// Storable says whether this object can be stored
+func (o *Object) Storable() bool {
+	return true
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) (err error) {
+	fs.Debugf(o, "touch -d %q sj://%s", t, o.absolute)
+
+	return fs.ErrorCantSetModTime
+}
+
+// Open opens the file for read. Call Close() on the returned io.ReadCloser
+func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (_ io.ReadCloser, err error) {
+	fs.Debugf(o, "cat sj://%s # %+v", o.absolute, options)
+
+	bucketName, bucketPath := bucket.Split(o.absolute)
+
+	// Convert the semantics of HTTP range headers to an offset and length
+	// that libuplink can use.
+	var (
+		offset int64 = 0
+		length int64 = -1
+	)
+
+	for _, option := range options {
+		switch opt := option.(type) {
+		case *fs.RangeOption:
+			s := opt.Start >= 0
+			e := opt.End >= 0
+
+			switch {
+			case s && e:
+				offset = opt.Start
+				length = (opt.End + 1) - opt.Start
+			case s && !e:
+				offset = opt.Start
+			case !s && e:
+				object, err := o.fs.project.StatObject(ctx, bucketName, bucketPath)
+				if err != nil {
+					return nil, err
+				}
+
+				offset = object.System.ContentLength - opt.End
+				length = opt.End
+			}
+		case *fs.SeekOption:
+			offset = opt.Offset
+		default:
+			if option.Mandatory() {
+				fs.Errorf(o, "Unsupported mandatory option: %v", option)
+
+				return nil, errors.New("unsupported mandatory option")
+			}
+		}
+	}
+
+	fs.Debugf(o, "range %d + %d", offset, length)
+
+	return o.fs.project.DownloadObject(ctx, bucketName, bucketPath, &uplink.DownloadOptions{
+		Offset: offset,
+		Length: length,
+	})
+}
+
+// Update in to the object with the modTime given of the given size
+//
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
+// return an error or update the object properly (rather than e.g. calling panic).
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
+	fs.Debugf(o, "cp input ./%s %+v", src.Remote(), options)
+
+	oNew, err := o.fs.Put(ctx, in, src, options...)
+
+	if err == nil {
+		*o = *(oNew.(*Object))
+	}
+
+	return err
+}
+
+// Remove this object.
+func (o *Object) Remove(ctx context.Context) (err error) {
+	fs.Debugf(o, "rm sj://%s", o.absolute)
+
+	bucketName, bucketPath := bucket.Split(o.absolute)
+
+	_, err = o.fs.project.DeleteObject(ctx, bucketName, bucketPath)
+
+	return err
+}

backend/tardigrade/tardigrade_test.go

@@ -0,0 +1,19 @@
+// +build go1.13,!plan9
+
+// Test Tardigrade filesystem interface
+package tardigrade_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/backend/tardigrade"
+	"github.com/rclone/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestTardigrade:",
+		NilObject:  (*tardigrade.Object)(nil),
+	})
+}

backend/tardigrade/tardigrade_unsupported.go

@@ -0,0 +1,3 @@
+// +build !go1.13 plan9
+
+package tardigrade

backend/union/entry.go

@@ -0,0 +1,167 @@
+package union
+
+import (
+	"bufio"
+	"context"
+	"io"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+// Object describes a union Object
+//
+// This is a wrapped object which returns the Union Fs as its parent
+type Object struct {
+	*upstream.Object
+	fs *Fs // what this object is part of
+	co []upstream.Entry
+}
+
+// Directory describes a union Directory
+//
+// This is a wrapped object contains all candidates
+type Directory struct {
+	*upstream.Directory
+	cd []upstream.Entry
+}
+
+type entry interface {
+	upstream.Entry
+	candidates() []upstream.Entry
+}
+
+// UnWrap returns the Object that this Object is wrapping or
+// nil if it isn't wrapping anything
+func (o *Object) UnWrap() *upstream.Object {
+	return o.Object
+}
+
+// Fs returns the union Fs as the parent
+func (o *Object) Fs() fs.Info {
+	return o.fs
+}
+
+func (o *Object) candidates() []upstream.Entry {
+	return o.co
+}
+
+func (d *Directory) candidates() []upstream.Entry {
+	return d.cd
+}
+
+// Update in to the object with the modTime given of the given size
+//
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
+// But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
+// return an error or update the object properly (rather than e.g. calling panic).
+func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	entries, err := o.fs.actionEntries(o.candidates()...)
+	if err != nil {
+		return err
+	}
+	if len(entries) == 1 {
+		obj := entries[0].(*upstream.Object)
+		return obj.Update(ctx, in, src, options...)
+	}
+	// Get multiple reader
+	readers := make([]io.Reader, len(entries))
+	writers := make([]io.Writer, len(entries))
+	errs := Errors(make([]error, len(entries)+1))
+	for i := range entries {
+		r, w := io.Pipe()
+		bw := bufio.NewWriter(w)
+		readers[i], writers[i] = r, bw
+		defer func() {
+			err := w.Close()
+			if err != nil {
+				panic(err)
+			}
+		}()
+	}
+	go func() {
+		mw := io.MultiWriter(writers...)
+		es := make([]error, len(writers)+1)
+		_, es[len(es)-1] = io.Copy(mw, in)
+		for i, bw := range writers {
+			es[i] = bw.(*bufio.Writer).Flush()
+		}
+		errs[len(entries)] = Errors(es).Err()
+	}()
+	// Multi-threading
+	multithread(len(entries), func(i int) {
+		if o, ok := entries[i].(*upstream.Object); ok {
+			err := o.Update(ctx, readers[i], src, options...)
+			errs[i] = errors.Wrap(err, o.UpstreamFs().Name())
+		} else {
+			errs[i] = fs.ErrorNotAFile
+		}
+	})
+	return errs.Err()
+}
+
+// Remove candidate objects selected by ACTION policy
+func (o *Object) Remove(ctx context.Context) error {
+	entries, err := o.fs.actionEntries(o.candidates()...)
+	if err != nil {
+		return err
+	}
+	errs := Errors(make([]error, len(entries)))
+	multithread(len(entries), func(i int) {
+		if o, ok := entries[i].(*upstream.Object); ok {
+			err := o.Remove(ctx)
+			errs[i] = errors.Wrap(err, o.UpstreamFs().Name())
+		} else {
+			errs[i] = fs.ErrorNotAFile
+		}
+	})
+	return errs.Err()
+}
+
+// SetModTime sets the metadata on the object to set the modification date
+func (o *Object) SetModTime(ctx context.Context, t time.Time) error {
+	entries, err := o.fs.actionEntries(o.candidates()...)
+	if err != nil {
+		return err
+	}
+	var wg sync.WaitGroup
+	errs := Errors(make([]error, len(entries)))
+	multithread(len(entries), func(i int) {
+		if o, ok := entries[i].(*upstream.Object); ok {
+			err := o.SetModTime(ctx, t)
+			errs[i] = errors.Wrap(err, o.UpstreamFs().Name())
+		} else {
+			errs[i] = fs.ErrorNotAFile
+		}
+	})
+	wg.Wait()
+	return errs.Err()
+}
+
+// ModTime returns the modification date of the directory
+// It returns the latest ModTime of all candidates
+func (d *Directory) ModTime(ctx context.Context) (t time.Time) {
+	entries := d.candidates()
+	times := make([]time.Time, len(entries))
+	multithread(len(entries), func(i int) {
+		times[i] = entries[i].ModTime(ctx)
+	})
+	for _, ti := range times {
+		if t.Before(ti) {
+			t = ti
+		}
+	}
+	return t
+}
+
+// Size returns the size of the directory
+// It returns the sum of all candidates
+func (d *Directory) Size() (s int64) {
+	for _, e := range d.candidates() {
+		s += e.Size()
+	}
+	return s
+}

backend/union/errors.go

@@ -0,0 +1,68 @@
+package union
+
+import (
+	"bytes"
+	"fmt"
+)
+
+// The Errors type wraps a slice of errors
+type Errors []error
+
+// Map returns a copy of the error slice with all its errors modified
+// according to the mapping function. If mapping returns nil,
+// the error is dropped from the error slice with no replacement.
+func (e Errors) Map(mapping func(error) error) Errors {
+	s := make([]error, len(e))
+	i := 0
+	for _, err := range e {
+		nerr := mapping(err)
+		if nerr == nil {
+			continue
+		}
+		s[i] = nerr
+		i++
+	}
+	return Errors(s[:i])
+}
+
+// FilterNil returns the Errors without nil
+func (e Errors) FilterNil() Errors {
+	ne := e.Map(func(err error) error {
+		return err
+	})
+	return ne
+}
+
+// Err returns an error interface that filtered nil,
+// or nil if no non-nil Error is presented.
+func (e Errors) Err() error {
+	ne := e.FilterNil()
+	if len(ne) == 0 {
+		return nil
+	}
+	return ne
+}
+
+// Error returns a concatenated string of the contained errors
+func (e Errors) Error() string {
+	var buf bytes.Buffer
+
+	if len(e) == 0 {
+		buf.WriteString("no error")
+	}
+	if len(e) == 1 {
+		buf.WriteString("1 error: ")
+	} else {
+		fmt.Fprintf(&buf, "%d errors: ", len(e))
+	}
+
+	for i, err := range e {
+		if i != 0 {
+			buf.WriteString("; ")
+		}
+
+		buf.WriteString(err.Error())
+	}
+
+	return buf.String()
+}

backend/union/policy/all.go

@@ -0,0 +1,44 @@
+package policy
+
+import (
+	"context"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("all", &All{})
+}
+
+// All policy behaves the same as EpAll except for the CREATE category
+// Action category: same as epall.
+// Create category: apply to all branches.
+// Search category: same as epall.
+type All struct {
+	EpAll
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *All) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterNC(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return upstreams, nil
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *All) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	entries = filterNCEntries(entries)
+	if len(entries) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return entries, nil
+}

backend/union/policy/epall.go

@@ -0,0 +1,99 @@
+package policy
+
+import (
+	"context"
+	"path"
+	"sync"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("epall", &EpAll{})
+}
+
+// EpAll stands for existing path, all
+// Action category: apply to all found.
+// Create category: apply to all found.
+// Search category: same as epff.
+type EpAll struct {
+	EpFF
+}
+
+func (p *EpAll) epall(ctx context.Context, upstreams []*upstream.Fs, filePath string) ([]*upstream.Fs, error) {
+	var wg sync.WaitGroup
+	ufs := make([]*upstream.Fs, len(upstreams))
+	for i, u := range upstreams {
+		wg.Add(1)
+		i, u := i, u // Closure
+		go func() {
+			rfs := u.RootFs
+			remote := path.Join(u.RootPath, filePath)
+			if findEntry(ctx, rfs, remote) != nil {
+				ufs[i] = u
+			}
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+	var results []*upstream.Fs
+	for _, f := range ufs {
+		if f != nil {
+			results = append(results, f)
+		}
+	}
+	if len(results) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return results, nil
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpAll) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterRO(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return p.epall(ctx, upstreams, path)
+}
+
+// ActionEntries is ACTION category policy but receivng a set of candidate entries
+func (p *EpAll) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	entries = filterROEntries(entries)
+	if len(entries) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return entries, nil
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpAll) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterNC(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	upstreams, err := p.epall(ctx, upstreams, path+"/..")
+	return upstreams, err
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpAll) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	entries = filterNCEntries(entries)
+	if len(entries) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return entries, nil
+}

backend/union/policy/epff.go

@@ -0,0 +1,115 @@
+package policy
+
+import (
+	"context"
+	"path"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("epff", &EpFF{})
+}
+
+// EpFF stands for existing path, first found
+// Given the order of the candidates, act on the first one found where the relative path exists.
+type EpFF struct{}
+
+func (p *EpFF) epff(ctx context.Context, upstreams []*upstream.Fs, filePath string) (*upstream.Fs, error) {
+	ch := make(chan *upstream.Fs)
+	for _, u := range upstreams {
+		u := u // Closure
+		go func() {
+			rfs := u.RootFs
+			remote := path.Join(u.RootPath, filePath)
+			if findEntry(ctx, rfs, remote) == nil {
+				u = nil
+			}
+			ch <- u
+		}()
+	}
+	var u *upstream.Fs
+	for i := 0; i < len(upstreams); i++ {
+		u = <-ch
+		if u != nil {
+			// close remaining goroutines
+			go func(num int) {
+				defer close(ch)
+				for i := 0; i < num; i++ {
+					<-ch
+				}
+			}(len(upstreams) - 1 - i)
+		}
+	}
+	if u == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return u, nil
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpFF) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterRO(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	u, err := p.epff(ctx, upstreams, path)
+	return []*upstream.Fs{u}, err
+}
+
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
+func (p *EpFF) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	entries = filterROEntries(entries)
+	if len(entries) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return entries[:1], nil
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpFF) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterNC(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	u, err := p.epff(ctx, upstreams, path+"/..")
+	return []*upstream.Fs{u}, err
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpFF) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	entries = filterNCEntries(entries)
+	if len(entries) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	return entries[:1], nil
+}
+
+// Search category policy, governing the access to files and directories
+func (p *EpFF) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return p.epff(ctx, upstreams, path)
+}
+
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
+func (p *EpFF) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return entries[0], nil
+}

backend/union/policy/eplfs.go

@@ -0,0 +1,116 @@
+package policy
+
+import (
+	"context"
+	"math"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("eplfs", &EpLfs{})
+}
+
+// EpLfs stands for existing path, least free space
+// Of all the candidates on which the path exists choose the one with the least free space.
+type EpLfs struct {
+	EpAll
+}
+
+func (p *EpLfs) lfs(upstreams []*upstream.Fs) (*upstream.Fs, error) {
+	var minFreeSpace int64 = math.MaxInt64
+	var lfsupstream *upstream.Fs
+	for _, u := range upstreams {
+		space, err := u.GetFreeSpace()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Free Space is not supported for upstream %s, treating as infinite", u.Name())
+		}
+		if space < minFreeSpace {
+			minFreeSpace = space
+			lfsupstream = u
+		}
+	}
+	if lfsupstream == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return lfsupstream, nil
+}
+
+func (p *EpLfs) lfsEntries(entries []upstream.Entry) (upstream.Entry, error) {
+	var minFreeSpace int64
+	var lfsEntry upstream.Entry
+	for _, e := range entries {
+		space, err := e.UpstreamFs().GetFreeSpace()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Free Space is not supported for upstream %s, treating as infinite", e.UpstreamFs().Name())
+		}
+		if space < minFreeSpace {
+			minFreeSpace = space
+			lfsEntry = e
+		}
+	}
+	return lfsEntry, nil
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpLfs) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Action(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.lfs(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
+func (p *EpLfs) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.ActionEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.lfsEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpLfs) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Create(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.lfs(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpLfs) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.CreateEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.lfsEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Search category policy, governing the access to files and directories
+func (p *EpLfs) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams, err := p.epall(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return p.lfs(upstreams)
+}
+
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
+func (p *EpLfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return p.lfsEntries(entries)
+}

backend/union/policy/eplno.go

@@ -0,0 +1,116 @@
+package policy
+
+import (
+	"context"
+	"math"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("eplno", &EpLno{})
+}
+
+// EpLno stands for existing path, least number of objects
+// Of all the candidates on which the path exists choose the one with the least number of objects
+type EpLno struct {
+	EpAll
+}
+
+func (p *EpLno) lno(upstreams []*upstream.Fs) (*upstream.Fs, error) {
+	var minNumObj int64 = math.MaxInt64
+	var lnoUpstream *upstream.Fs
+	for _, u := range upstreams {
+		numObj, err := u.GetNumObjects()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Number of Objects is not supported for upstream %s, treating as 0", u.Name())
+		}
+		if minNumObj > numObj {
+			minNumObj = numObj
+			lnoUpstream = u
+		}
+	}
+	if lnoUpstream == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return lnoUpstream, nil
+}
+
+func (p *EpLno) lnoEntries(entries []upstream.Entry) (upstream.Entry, error) {
+	var minNumObj int64 = math.MaxInt64
+	var lnoEntry upstream.Entry
+	for _, e := range entries {
+		numObj, err := e.UpstreamFs().GetNumObjects()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Number of Objects is not supported for upstream %s, treating as 0", e.UpstreamFs().Name())
+		}
+		if minNumObj > numObj {
+			minNumObj = numObj
+			lnoEntry = e
+		}
+	}
+	return lnoEntry, nil
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpLno) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Action(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.lno(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
+func (p *EpLno) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.ActionEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.lnoEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpLno) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Create(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.lno(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpLno) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.CreateEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.lnoEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Search category policy, governing the access to files and directories
+func (p *EpLno) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams, err := p.epall(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return p.lno(upstreams)
+}
+
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
+func (p *EpLno) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return p.lnoEntries(entries)
+}

backend/union/policy/eplus.go

@@ -0,0 +1,116 @@
+package policy
+
+import (
+	"context"
+	"math"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("eplus", &EpLus{})
+}
+
+// EpLus stands for existing path, least used space
+// Of all the candidates on which the path exists choose the one with the least used space.
+type EpLus struct {
+	EpAll
+}
+
+func (p *EpLus) lus(upstreams []*upstream.Fs) (*upstream.Fs, error) {
+	var minUsedSpace int64 = math.MaxInt64
+	var lusupstream *upstream.Fs
+	for _, u := range upstreams {
+		space, err := u.GetUsedSpace()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Used Space is not supported for upstream %s, treating as 0", u.Name())
+		}
+		if space < minUsedSpace {
+			minUsedSpace = space
+			lusupstream = u
+		}
+	}
+	if lusupstream == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return lusupstream, nil
+}
+
+func (p *EpLus) lusEntries(entries []upstream.Entry) (upstream.Entry, error) {
+	var minUsedSpace int64
+	var lusEntry upstream.Entry
+	for _, e := range entries {
+		space, err := e.UpstreamFs().GetFreeSpace()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Used Space is not supported for upstream %s, treating as 0", e.UpstreamFs().Name())
+		}
+		if space < minUsedSpace {
+			minUsedSpace = space
+			lusEntry = e
+		}
+	}
+	return lusEntry, nil
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpLus) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Action(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.lus(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
+func (p *EpLus) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.ActionEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.lusEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpLus) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Create(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.lus(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpLus) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.CreateEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.lusEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Search category policy, governing the access to files and directories
+func (p *EpLus) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams, err := p.epall(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return p.lus(upstreams)
+}
+
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
+func (p *EpLus) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return p.lusEntries(entries)
+}

backend/union/policy/epmfs.go

@@ -0,0 +1,115 @@
+package policy
+
+import (
+	"context"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("epmfs", &EpMfs{})
+}
+
+// EpMfs stands for existing path, most free space
+// Of all the candidates on which the path exists choose the one with the most free space.
+type EpMfs struct {
+	EpAll
+}
+
+func (p *EpMfs) mfs(upstreams []*upstream.Fs) (*upstream.Fs, error) {
+	var maxFreeSpace int64
+	var mfsupstream *upstream.Fs
+	for _, u := range upstreams {
+		space, err := u.GetFreeSpace()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Free Space is not supported for upstream %s, treating as infinite", u.Name())
+		}
+		if maxFreeSpace < space {
+			maxFreeSpace = space
+			mfsupstream = u
+		}
+	}
+	if mfsupstream == nil {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return mfsupstream, nil
+}
+
+func (p *EpMfs) mfsEntries(entries []upstream.Entry) (upstream.Entry, error) {
+	var maxFreeSpace int64
+	var mfsEntry upstream.Entry
+	for _, e := range entries {
+		space, err := e.UpstreamFs().GetFreeSpace()
+		if err != nil {
+			fs.LogPrintf(fs.LogLevelNotice, nil,
+				"Free Space is not supported for upstream %s, treating as infinite", e.UpstreamFs().Name())
+		}
+		if maxFreeSpace < space {
+			maxFreeSpace = space
+			mfsEntry = e
+		}
+	}
+	return mfsEntry, nil
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpMfs) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Action(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.mfs(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
+func (p *EpMfs) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.ActionEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.mfsEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpMfs) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Create(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	u, err := p.mfs(upstreams)
+	return []*upstream.Fs{u}, err
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpMfs) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.CreateEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	e, err := p.mfsEntries(entries)
+	return []upstream.Entry{e}, err
+}
+
+// Search category policy, governing the access to files and directories
+func (p *EpMfs) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams, err := p.epall(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return p.mfs(upstreams)
+}
+
+// SearchEntries is SEARCH category policy but receivng a set of candidate entries
+func (p *EpMfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return p.mfsEntries(entries)
+}

backend/union/policy/eprand.go

@@ -0,0 +1,86 @@
+package policy
+
+import (
+	"context"
+	"math/rand"
+	"time"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("eprand", &EpRand{})
+}
+
+// EpRand stands for existing path, random
+// Calls epall and then randomizes. Returns one candidate.
+type EpRand struct {
+	EpAll
+}
+
+func (p *EpRand) rand(upstreams []*upstream.Fs) *upstream.Fs {
+	rand.Seed(time.Now().Unix())
+	return upstreams[rand.Intn(len(upstreams))]
+}
+
+func (p *EpRand) randEntries(entries []upstream.Entry) upstream.Entry {
+	rand.Seed(time.Now().Unix())
+	return entries[rand.Intn(len(entries))]
+}
+
+// Action category policy, governing the modification of files and directories
+func (p *EpRand) Action(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Action(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return []*upstream.Fs{p.rand(upstreams)}, nil
+}
+
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
+func (p *EpRand) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.ActionEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	return []upstream.Entry{p.randEntries(entries)}, nil
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *EpRand) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	upstreams, err := p.EpAll.Create(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return []*upstream.Fs{p.rand(upstreams)}, nil
+}
+
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
+func (p *EpRand) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
+	entries, err := p.EpAll.CreateEntries(entries...)
+	if err != nil {
+		return nil, err
+	}
+	return []upstream.Entry{p.randEntries(entries)}, nil
+}
+
+// Search category policy, governing the access to files and directories
+func (p *EpRand) Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams, err := p.epall(ctx, upstreams, path)
+	if err != nil {
+		return nil, err
+	}
+	return p.rand(upstreams), nil
+}
+
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
+func (p *EpRand) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
+	if len(entries) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	return p.randEntries(entries), nil
+}

backend/union/policy/ff.go

@@ -0,0 +1,32 @@
+package policy
+
+import (
+	"context"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("ff", &FF{})
+}
+
+// FF stands for first found
+// Search category: same as epff.
+// Action category: same as epff.
+// Create category: Given the order of the candiates, act on the first one found.
+type FF struct {
+	EpFF
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *FF) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterNC(upstreams)
+	if len(upstreams) == 0 {
+		return upstreams, fs.ErrorPermissionDenied
+	}
+	return upstreams[:1], nil
+}

backend/union/policy/lfs.go

@@ -0,0 +1,33 @@
+package policy
+
+import (
+	"context"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("lfs", &Lfs{})
+}
+
+// Lfs stands for least free space
+// Search category: same as eplfs.
+// Action category: same as eplfs.
+// Create category: Pick the drive with the least free space.
+type Lfs struct {
+	EpLfs
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *Lfs) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterNC(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	u, err := p.lfs(upstreams)
+	return []*upstream.Fs{u}, err
+}

backend/union/policy/lno.go

@@ -0,0 +1,33 @@
+package policy
+
+import (
+	"context"
+
+	"github.com/rclone/rclone/backend/union/upstream"
+	"github.com/rclone/rclone/fs"
+)
+
+func init() {
+	registerPolicy("lno", &Lno{})
+}
+
+// Lno stands for least number of objects
+// Search category: same as eplno.
+// Action category: same as eplno.
+// Create category: Pick the drive with the least number of objects.
+type Lno struct {
+	EpLno
+}
+
+// Create category policy, governing the creation of files and directories
+func (p *Lno) Create(ctx context.Context, upstreams []*upstream.Fs, path string) ([]*upstream.Fs, error) {
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorObjectNotFound
+	}
+	upstreams = filterNC(upstreams)
+	if len(upstreams) == 0 {
+		return nil, fs.ErrorPermissionDenied
+	}
+	u, err := p.lno(upstreams)
+	return []*upstream.Fs{u}, err
+}