fs: Debug the names of env vars we read values from as part of the config

See: https://forum.rclone.org/t/how-can-i-check-if-environment-variables-are-being-used/17490

.github/FUNDING.yml

@@ -0,0 +1,4 @@
+github: [ncw]
+patreon: njcw
+liberapay: ncw
+custom: ["https://rclone.org/donate/"]

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Rclone Forum Community Support
+    url: https://forum.rclone.org/
+    about: Please ask and answer questions here.

.github/workflows/build.yml

@@ -197,8 +197,9 @@ jobs:
       - name: Deploy built binaries
         shell: bash
         run: |
-          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep ; fi
-          make travis_beta
+          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
+          if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
+          make ci_beta
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # working-directory: '$(modulePath)'
@@ -227,7 +228,7 @@ jobs:
       - name: Cross-compile rclone
         run: |
           docker pull billziss/xgo-cgofuse
-          go get -v github.com/karalabe/xgo
+          GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
           xgo \
               -image=billziss/xgo-cgofuse \
               -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
@@ -247,7 +248,7 @@ jobs:
 
       - name: Upload artifacts
         run: |
-          make circleci_upload
+          make ci_upload
         env:
           RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
         # Upload artifacts if not a PR && not a fork

.github/workflows/build_publish_docker_image.yml

@@ -0,0 +1,25 @@
+name: Docker beta build
+
+on:
+    push:
+      branches:
+        - master
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                fetch-depth: 0
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              with:
+                tag: beta
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

.github/workflows/build_publish_release_docker_image.yml

@@ -0,0 +1,33 @@
+name: Docker release build
+
+on:
+  release:
+    types: [published]
+
+jobs:
+    build:
+        runs-on: ubuntu-latest
+        name: Build image job
+        steps:
+            - name: Checkout master
+              uses: actions/checkout@v2
+              with:
+                fetch-depth: 0
+            - name: Get actual patch version
+              id: actual_patch_version
+              run: echo ::set-output name=ACTUAL_PATCH_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g')
+            - name: Get actual minor version
+              id: actual_minor_version
+              run: echo ::set-output name=ACTUAL_MINOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1,2)
+            - name: Get actual major version
+              id: actual_major_version
+              run: echo ::set-output name=ACTUAL_MAJOR_VERSION::$(echo $GITHUB_REF | cut -d / -f 3 | sed 's/v//g' | cut -d "." -f 1)
+            - name: Build and publish image
+              uses: ilteoood/docker_buildx@439099796bfc03dd9cedeb72a0c7cb92be5cc92c
+              with:
+                tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
+                imageName: rclone/rclone
+                platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
+                publish: true
+                dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
+                dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}

CONTRIBUTING.md

@@ -79,8 +79,15 @@ request](https://help.github.com/articles/creating-a-pull-request/).
 
 You patch will get reviewed and you might get asked to fix some stuff.
 
-If so, then make the changes in the same branch, squash the commits,
-rebase it to master then push it to GitHub with `--force`.
+If so, then make the changes in the same branch, squash the commits (make multiple commits one commit) by running:
+```
+git log # See how many commits you want to squash
+git reset --soft HEAD~2 # This squashes the 2 latest commits together.
+git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
+git commit # Add a new commit message.
+git push --force # Push the squashed commit to your GitHub repo.
+# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also reccommends wizardzines.com
+```
 
 ## CI for your fork ##
 
@@ -158,7 +165,7 @@ with modules beneath.
     * fserrors - rclone specific error handling
     * fshttp - http handling for rclone
     * fspath - path handling for rclone
-    * hash - defines rclones hash types and functions
+    * hash - defines rclone's hash types and functions
     * list - list a remote
     * log - logging facilities
     * march - iterates directories in lock step
@@ -295,7 +302,7 @@ If you need to update a dependency then run
     GO111MODULE=on go get -u github.com/pkg/errors
     GO111MODULE=on go mod vendor
 
-Check in in a single commit as above.
+Check in a single commit as above.
 
 ## Updating all the dependencies ##
 
@@ -383,7 +390,7 @@ alphabetical order of full name of remote (eg `drive` is ordered as
     * update them with `make backenddocs` - revert any changes in other backends
   * `docs/content/overview.md` - overview docs
   * `docs/content/docs.md` - list of remotes in config section
-  * `docs/content/about.md` - front page of rclone.org
+  * `docs/content/_index.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
 

Dockerfile

@@ -3,7 +3,6 @@ FROM golang AS builder
 COPY . /go/src/github.com/rclone/rclone/
 WORKDIR /go/src/github.com/rclone/rclone/
 
-RUN make quicktest
 RUN \
   CGO_ENABLED=0 \
   make
@@ -12,7 +11,8 @@ RUN ./rclone version
 # Begin final image
 FROM alpine:latest
 
-RUN apk --no-cache add ca-certificates fuse
+RUN apk --no-cache add ca-certificates fuse tzdata && \
+  echo "user_allow_other" >> /etc/fuse.conf
 
 COPY --from=builder /go/src/github.com/rclone/rclone/rclone /usr/local/bin/
 

MAINTAINERS.md

@@ -33,7 +33,7 @@ Rclone uses the labels like this:
 * `duplicate` - normally close these and ask the user to subscribe to the original
 * `enhancement: new remote` - a new rclone backend
 * `enhancement` - a new feature
-* `FUSE` - do do with `rclone mount` command
+* `FUSE` - to do with `rclone mount` command
 * `good first issue` - mark these if you find a small self contained issue - these get shown to new visitors to the project
 * `help` wanted - mark these if you find a self contained issue - these get shown to new visitors to the project
 * `IMPORTANT` - note to maintainers not to forget to fix this for the release

Makefile

@@ -1,6 +1,6 @@
 SHELL = bash
 # Branch we are working on
-BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
+BRANCH := $(or $(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
 # Tag of the current commit, if any.  If this is not "" then we are building a release
 RELEASE_TAG := $(shell git tag -l --points-at HEAD)
 # Version of last release (may not be on this branch)
@@ -61,6 +61,10 @@ vars:
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
+btest:
+	@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
+	@echo "Copied markdown of beta release to clip board"
+
 version:
 	@echo '$(TAG)'
 
@@ -86,11 +90,15 @@ check:	rclone
 build_dep:
 	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 
-# Get the release dependencies
-release_dep:
+# Get the release dependencies we only install on linux
+release_dep_linux:
 	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
 	go run bin/get-github-release.go -extract github-release aktau/github-release 'linux-amd64-github-release.tar.bz2'
 
+# Get the release dependencies we only install on Windows
+release_dep_windows:
+	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
+
 # Update dependencies
 update:
 	GO111MODULE=on go get -u ./...
@@ -105,16 +113,16 @@ tidy:
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
 rclone.1:	MANUAL.md
-	pandoc -s --from markdown --to man MANUAL.md -o rclone.1
+	pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
 
 MANUAL.md:	bin/make_manual.py docs/content/*.md commanddocs backenddocs
 	./bin/make_manual.py
 
 MANUAL.html:	MANUAL.md
-	pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
+	pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
 
 MANUAL.txt:	MANUAL.md
-	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
+	pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
 	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
@@ -136,11 +144,19 @@ clean:
 	rm -f rclone fs/operations/operations.test fs/sync/sync.test fs/test_all.log test.log
 
 website:
+	rm -rf docs/public
 	cd docs && hugo
+	@if grep -R "raw HTML omitted" docs/public ; then echo "ERROR: found unescaped HTML - fix the markdown source" ; fi
 
 upload_website:	website
 	rclone -v sync docs/public memstore:www-rclone-org
 
+upload_test_website:	website
+	rclone -P sync docs/public test-rclone-org:
+
+validate_website: website
+	find docs/public -type f -name "*.html" | xargs tidy --mute-id yes -errors --gnu-emacs yes --drop-empty-elements no --warn-proprietary-attributes no --mute MISMATCHED_ATTRIBUTE_WARN
+
 tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
@@ -175,14 +191,7 @@ log_since_last_release:
 compile_all:
 	go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
 
-appveyor_upload:
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
-ifndef BRANCH_PATH
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
-endif
-	@echo Beta release ready at $(BETA_URL)
-
-circleci_upload:
+ci_upload:
 	sudo chown -R $$USER build
 	find build -type l -delete
 	gzip -r9v build
@@ -192,10 +201,7 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)/testbuilds
 
-travis_beta:
-ifeq (linux,$(filter linux,$(subst Linux,linux,$(TRAVIS_OS_NAME) $(AGENT_OS))))
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
-endif
+ci_beta:
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
 	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
@@ -204,12 +210,12 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)
 
-# Fetch the binary builds from travis and appveyor
+# Fetch the binary builds from GitHub actions
 fetch_binaries:
 	rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
 
 serve:	website
-	cd docs && hugo server -v -w
+	cd docs && hugo server -v -w --disableFastRender
 
 tag:	doc
 	@echo "Old tag is $(VERSION)"

RELEASE.md

@@ -31,7 +31,7 @@ This file describes how to make the various kinds of releases
   * make upload_website
   * make upload_github
   * make startdev
-  * # announce with forum post, twitter post, G+ post
+  * # announce with forum post, twitter post, patreon post
 
 Early in the next release cycle update the vendored dependencies
 
@@ -63,14 +63,14 @@ If rclone needs a point release due to some horrendous bug:
 First make the release branch.  If this is a second point release then
 this will be done already.
 
-  * BASE_TAG=v1.XX          # eg v1.49
-  * NEW_TAG=${BASE_TAG}.Y   # eg v1.49.1
-  * echo $BASE_TAG $NEW_TAG # v1.49 v1.49.1
-  * git branch ${BASE_TAG} ${BASE_TAG}-fixes
+  * BASE_TAG=v1.XX          # eg v1.52
+  * NEW_TAG=${BASE_TAG}.Y   # eg v1.52.1
+  * echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
+  * git branch ${BASE_TAG} ${BASE_TAG}-stable
 
 Now
 
-  * git co ${BASE_TAG}-fixes
+  * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
   * Test (see above)
   * make NEXT_VERSION=${NEW_TAG} tag
@@ -79,7 +79,7 @@ Now
   * git commit -a -v -m "Version ${NEW_TAG}"
   * git tag -d ${NEW_TAG}
   * git tag -s -m "Version ${NEW_TAG}" ${NEW_TAG}
-  * git push --tags -u origin ${BASE_TAG}-fixes
+  * git push --tags -u origin ${BASE_TAG}-stable
   * Wait for builds to complete
   * make BRANCH_PATH= TAG=${NEW_TAG} fetch_binaries
   * make TAG=${NEW_TAG} tarball
@@ -92,20 +92,21 @@ Now
   * git co master
   * make VERSION=${NEW_TAG} startdev
   * # cherry pick the changes to the changelog and VERSION
-  * git checkout ${BASE_TAG}-fixes VERSION docs/content/changelog.md
+  * git checkout ${BASE_TAG}-stable VERSION docs/content/changelog.md
   * git commit --amend
   * git push
   * Announce!
 
 ## Making a manual build of docker
 
-The rclone docker image should autobuild on docker hub.  If it doesn't
+The rclone docker image should autobuild on via GitHub actions.  If it doesn't
 or needs to be updated then rebuild like this.
 
 ```
-docker build -t rclone/rclone:1.49.1 -t rclone/rclone:1.49 -t rclone/rclone:1 -t rclone/rclone:latest .
-docker push rclone/rclone:1.49.1
-docker push rclone/rclone:1.49
+docker pull golang
+docker build --rm --ulimit memlock=67108864  -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
+docker push rclone/rclone:1.52.0
+docker push rclone/rclone:1.52
 docker push rclone/rclone:1
 docker push rclone/rclone:latest
 ```

VERSION

@@ -1 +1 @@
-v1.51.0
+v1.52.2

backend/amazonclouddrive/amazonclouddrive.go

@@ -71,7 +71,7 @@ func init() {
 		Description: "Amazon Drive",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("amazon cloud drive", name, m, acdConfig)
+			err := oauthutil.Config("amazon cloud drive", name, m, acdConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -169,7 +169,7 @@ type Fs struct {
 	tokenRenewer *oauthutil.Renew   // renew the token on expiry
 }
 
-// Object describes a acd object
+// Object describes an acd object
 //
 // Will definitely have info but maybe not meta
 type Object struct {
@@ -229,7 +229,7 @@ func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 		}
 		// Work around receiving this error sporadically on authentication
 		//
-		// HTTP code 403: "403 Forbidden", reponse body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
+		// HTTP code 403: "403 Forbidden", response body: {"message":"Authorization header requires 'Credential' parameter. Authorization header requires 'Signature' parameter. Authorization header requires 'SignedHeaders' parameter. Authorization header requires existence of either a 'X-Amz-Date' or a 'Date' header. Authorization=Bearer"}
 		if resp.StatusCode == 403 && strings.Contains(err.Error(), "Authorization header requires") {
 			fs.Debugf(f, "403 \"Authorization header requires...\" error received - retry")
 			return true, err
@@ -514,10 +514,6 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -665,7 +661,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 		return nil, err
 	}
 	// If not create it
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -696,13 +692,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -724,10 +714,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 
 	// create the destination directory if necessary
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return nil, err
-	}
 	srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
 	if err != nil {
 		return nil, err
@@ -797,54 +783,24 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return errors.New("can't move root directory")
 	}
 
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
 	// Find ID of dst parent, creating subdirs if necessary
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, findPath, true)
+	dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, dstRemote, true)
 	if err != nil {
 		return err
 	}
 
 	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
+	_, err = f.dirCache.FindDir(ctx, dstRemote, false)
+	if err == fs.ErrorDirNotFound {
+		// OK
+	} else if err != nil {
+		return err
+	} else {
+		return fs.ErrorDirExists
 	}
 
 	// Find ID of src parent
-	findPath = srcRemote
-	var srcDirectoryID string
-	if srcRemote == "" {
-		srcDirectoryID, err = srcFs.dirCache.RootParentID()
-	} else {
-		_, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
-	}
+	_, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcRemote, false)
 	if err != nil {
 		return err
 	}
@@ -890,10 +846,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1037,7 +989,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if o.info != nil {
 		return nil
 	}
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return fs.ErrorObjectNotFound

backend/azureblob/azureblob.go

@@ -201,7 +201,7 @@ type Fs struct {
 	pool          *pool.Pool                      // memory pool
 }
 
-// Object describes a azure object
+// Object describes an azure object
 type Object struct {
 	fs         *Fs                   // what this object is part of
 	remote     string                // The remote path
@@ -338,7 +338,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 }
 
 // httpClientFactory creates a Factory object that sends HTTP requests
-// to a rclone's http.Client.
+// to an rclone's http.Client.
 //
 // copied from azblob.newDefaultHTTPClientFactory
 func httpClientFactory(client *http.Client) pipeline.Factory {
@@ -1455,7 +1455,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	httpHeaders.ContentType = fs.MimeType(ctx, o)
 	// Compute the Content-MD5 of the file, for multiparts uploads it
 	// will be set in PutBlockList API call using the 'x-ms-blob-content-md5' header
-	// Note: If multipart, a MD5 checksum will also be computed for each uploaded block
+	// Note: If multipart, an MD5 checksum will also be computed for each uploaded block
 	// 		 in order to validate its integrity during transport
 	if !o.fs.opt.DisableCheckSum {
 		if sourceMD5, _ := src.Hash(ctx, hash.MD5); sourceMD5 != "" {

backend/b2/api/types.go

@@ -337,3 +337,11 @@ type CopyFileRequest struct {
 	Info              map[string]string `json:"fileInfo,omitempty"`            // This field stores the metadata that will be stored with the file. (REPLACE only)
 	DestBucketID      string            `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
 }
+
+// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
+type CopyPartRequest struct {
+	SourceID    string `json:"sourceFileId"`    // The ID of the source file being copied.
+	LargeFileID string `json:"largeFileId"`     // The ID of the large file the part will belong to, as returned by b2_start_large_file.
+	PartNumber  int64  `json:"partNumber"`      // Which part this is (starting from 1)
+	Range       string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
+}

backend/b2/b2.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
+	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -54,6 +55,9 @@ const (
 	minChunkSize        = 5 * fs.MebiByte
 	defaultChunkSize    = 96 * fs.MebiByte
 	defaultUploadCutoff = 200 * fs.MebiByte
+	largeFileCopyCutoff = 4 * fs.GibiByte          // 5E9 is the max
+	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
+	memoryPoolUseMmap   = false
 )
 
 // Globals
@@ -113,6 +117,16 @@ Files above this size will be uploaded in chunks of "--b2-chunk-size".
 This value should be set no larger than 4.657GiB (== 5GB).`,
 			Default:  defaultUploadCutoff,
 			Advanced: true,
+		}, {
+			Name: "copy_cutoff",
+			Help: `Cutoff for switching to multipart copy
+
+Any files larger than this that need to be server side copied will be
+copied in chunks of this size.
+
+The minimum is 0 and the maximum is 4.6GB.`,
+			Default:  largeFileCopyCutoff,
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Upload chunk size. Must fit in memory.
@@ -150,6 +164,18 @@ The duration before the download authorization token will expire.
 The minimum value is 1 second. The maximum value is one week.`,
 			Default:  fs.Duration(7 * 24 * time.Hour),
 			Advanced: true,
+		}, {
+			Name:     "memory_pool_flush_time",
+			Default:  memoryPoolFlushTime,
+			Advanced: true,
+			Help: `How often internal memory buffer pools will be flushed.
+Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
+This option controls how often unused buffers will be removed from the pool.`,
+		}, {
+			Name:     "memory_pool_use_mmap",
+			Default:  memoryPoolUseMmap,
+			Advanced: true,
+			Help:     `Whether to use mmap buffers in internal memory pool.`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -173,10 +199,13 @@ type Options struct {
 	Versions                      bool                 `config:"versions"`
 	HardDelete                    bool                 `config:"hard_delete"`
 	UploadCutoff                  fs.SizeSuffix        `config:"upload_cutoff"`
+	CopyCutoff                    fs.SizeSuffix        `config:"copy_cutoff"`
 	ChunkSize                     fs.SizeSuffix        `config:"chunk_size"`
 	DisableCheckSum               bool                 `config:"disable_checksum"`
 	DownloadURL                   string               `config:"download_url"`
 	DownloadAuthorizationDuration fs.Duration          `config:"download_auth_duration"`
+	MemoryPoolFlushTime           fs.Duration          `config:"memory_pool_flush_time"`
+	MemoryPoolUseMmap             bool                 `config:"memory_pool_use_mmap"`
 	Enc                           encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -199,7 +228,8 @@ type Fs struct {
 	uploads         map[string][]*api.GetUploadURLResponse // Upload URLs by buckedID
 	authMu          sync.Mutex                             // lock for authorizing the account
 	pacer           *fs.Pacer                              // To pace and retry the API calls
-	bufferTokens    chan []byte                            // control concurrency of multipart uploads
+	uploadToken     *pacer.TokenDispenser                  // control concurrency
+	pool            *pool.Pool                             // memory pool
 }
 
 // Object describes a b2 object
@@ -335,7 +365,6 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	err = checkUploadChunkSize(cs)
 	if err == nil {
 		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
-		f.fillBufferTokens() // reset the buffer tokens
 	}
 	return
 }
@@ -396,6 +425,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		_bucketType: make(map[string]string, 1),
 		uploads:     make(map[string][]*api.GetUploadURLResponse),
 		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+		pool: pool.New(
+			time.Duration(opt.MemoryPoolFlushTime),
+			int(opt.ChunkSize),
+			fs.Config.Transfers,
+			opt.MemoryPoolUseMmap,
+		),
 	}
 	f.setRoot(root)
 	f.features = (&fs.Features{
@@ -410,7 +446,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		f.srv.SetHeader(testModeHeader, testMode)
 		fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
 	}
-	f.fillBufferTokens()
 	err = f.authorizeAccount(ctx)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to authorize account")
@@ -533,32 +568,25 @@ func (f *Fs) clearUploadURL(bucketID string) {
 	f.uploadMu.Unlock()
 }
 
-// Fill up (or reset) the buffer tokens
-func (f *Fs) fillBufferTokens() {
-	f.bufferTokens = make(chan []byte, fs.Config.Transfers)
-	for i := 0; i < fs.Config.Transfers; i++ {
-		f.bufferTokens <- nil
+// getBuf gets a buffer of f.opt.ChunkSize and an upload token
+//
+// If noBuf is set then it just gets an upload token
+func (f *Fs) getBuf(noBuf bool) (buf []byte) {
+	f.uploadToken.Get()
+	if !noBuf {
+		buf = f.pool.Get()
 	}
-}
-
-// getUploadBlock gets a block from the pool of size chunkSize
-func (f *Fs) getUploadBlock() []byte {
-	buf := <-f.bufferTokens
-	if buf == nil {
-		buf = make([]byte, f.opt.ChunkSize)
-	}
-	// fs.Debugf(f, "Getting upload block %p", buf)
 	return buf
 }
 
-// putUploadBlock returns a block to the pool of size chunkSize
-func (f *Fs) putUploadBlock(buf []byte) {
-	buf = buf[:cap(buf)]
-	if len(buf) != int(f.opt.ChunkSize) {
-		panic("bad blocksize returned to pool")
+// putBuf returns a buffer to the memory pool and an upload token
+//
+// If noBuf is set then it just returns the upload token
+func (f *Fs) putBuf(buf []byte, noBuf bool) {
+	if !noBuf {
+		f.pool.Put(buf)
 	}
-	// fs.Debugf(f, "Returning upload block %p", buf)
-	f.bufferTokens <- buf
+	f.uploadToken.Put()
 }
 
 // Return an Object from a path
@@ -1205,6 +1233,63 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 	return f.purge(ctx, f.rootBucket, f.rootDirectory, true)
 }
 
+// copy does a server side copy from dstObj <- srcObj
+//
+// If newInfo is nil then the metadata will be copied otherwise it
+// will be replaced with newInfo
+func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *api.File) (err error) {
+	if srcObj.size >= int64(f.opt.CopyCutoff) {
+		if newInfo == nil {
+			newInfo, err = srcObj.getMetaData(ctx)
+			if err != nil {
+				return err
+			}
+		}
+		up, err := f.newLargeUpload(ctx, dstObj, nil, srcObj, f.opt.CopyCutoff, true, newInfo)
+		if err != nil {
+			return err
+		}
+		return up.Upload(ctx)
+	}
+
+	dstBucket, dstPath := dstObj.split()
+	err = f.makeBucket(ctx, dstBucket)
+	if err != nil {
+		return err
+	}
+
+	destBucketID, err := f.getBucketID(ctx, dstBucket)
+	if err != nil {
+		return err
+	}
+
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/b2_copy_file",
+	}
+	var request = api.CopyFileRequest{
+		SourceID:     srcObj.id,
+		Name:         f.opt.Enc.FromStandardPath(dstPath),
+		DestBucketID: destBucketID,
+	}
+	if newInfo == nil {
+		request.MetadataDirective = "COPY"
+	} else {
+		request.MetadataDirective = "REPLACE"
+		request.ContentType = newInfo.ContentType
+		request.Info = newInfo.Info
+	}
+	var response api.FileInfo
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
+		return f.shouldRetry(ctx, resp, err)
+	})
+	if err != nil {
+		return err
+	}
+	return dstObj.decodeMetaDataFileInfo(&response)
+}
+
 // Copy src to this remote using server side copy operations.
 //
 // This is stored with the remote path given
@@ -1215,47 +1300,21 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 //
 // If it isn't possible then return fs.ErrorCantCopy
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
-	dstBucket, dstPath := f.split(remote)
-	err := f.makeBucket(ctx, dstBucket)
-	if err != nil {
-		return nil, err
-	}
 	srcObj, ok := src.(*Object)
 	if !ok {
 		fs.Debugf(src, "Can't copy - not same remote type")
 		return nil, fs.ErrorCantCopy
 	}
-	destBucketID, err := f.getBucketID(ctx, dstBucket)
-	if err != nil {
-		return nil, err
-	}
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          srcObj.id,
-		Name:              f.opt.Enc.FromStandardPath(dstPath),
-		MetadataDirective: "COPY",
-		DestBucketID:      destBucketID,
-	}
-	var response api.FileInfo
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
-		return f.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return nil, err
-	}
-	o := &Object{
+	// Temporary Object under construction
+	dstObj := &Object{
 		fs:     f,
 		remote: remote,
 	}
-	err = o.decodeMetaDataFileInfo(&response)
+	err := f.copy(ctx, dstObj, srcObj, nil)
 	if err != nil {
 		return nil, err
 	}
-	return o, nil
+	return dstObj, nil
 }
 
 // Hashes returns the supported hash sets.
@@ -1298,7 +1357,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
 }
 
 // PublicLink returns a link for downloading without account
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	bucket, bucketPath := f.split(remote)
 	var RootURL string
 	if f.opt.DownloadURL == "" {
@@ -1526,28 +1585,10 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 	if err != nil {
 		return err
 	}
-	_, bucketPath := o.split()
 	info.Info[timeKey] = timeString(modTime)
-	opts := rest.Opts{
-		Method: "POST",
-		Path:   "/b2_copy_file",
-	}
-	var request = api.CopyFileRequest{
-		SourceID:          o.id,
-		Name:              o.fs.opt.Enc.FromStandardPath(bucketPath), // copy to same name
-		MetadataDirective: "REPLACE",
-		ContentType:       info.ContentType,
-		Info:              info.Info,
-	}
-	var response api.FileInfo
-	err = o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(ctx, &opts, &request, &response)
-		return o.fs.shouldRetry(ctx, resp, err)
-	})
-	if err != nil {
-		return err
-	}
-	return o.decodeMetaDataFileInfo(&response)
+
+	// Copy to the same name, overwriting the metadata only
+	return o.fs.copy(ctx, o, o, info)
 }
 
 // Storable returns if this object is storable
@@ -1723,7 +1764,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	if size == -1 {
 		// Check if the file is large enough for a chunked upload (needs to be at least two chunks)
-		buf := o.fs.getUploadBlock()
+		buf := o.fs.getBuf(false)
+
 		n, err := io.ReadFull(in, buf)
 		if err == nil {
 			bufReader := bufio.NewReader(in)
@@ -1733,22 +1775,24 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 		if err == nil {
 			fs.Debugf(o, "File is big enough for chunked streaming")
-			up, err := o.fs.newLargeUpload(ctx, o, in, src)
+			up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 			if err != nil {
-				o.fs.putUploadBlock(buf)
+				o.fs.putBuf(buf, false)
 				return err
 			}
+			// NB Stream returns the buffer and token
 			return up.Stream(ctx, buf)
 		} else if err == io.EOF || err == io.ErrUnexpectedEOF {
 			fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
-			defer o.fs.putUploadBlock(buf)
+			defer o.fs.putBuf(buf, false)
 			size = int64(n)
 			in = bytes.NewReader(buf[:n])
 		} else {
+			o.fs.putBuf(buf, false)
 			return err
 		}
 	} else if size > int64(o.fs.opt.UploadCutoff) {
-		up, err := o.fs.newLargeUpload(ctx, o, in, src)
+		up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
 		if err != nil {
 			return err
 		}

backend/b2/upload.go

@@ -20,7 +20,9 @@ import (
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
+	"golang.org/x/sync/errgroup"
 )
 
 type hashAppendingReader struct {
@@ -68,20 +70,26 @@ func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader {
 
 // largeUpload is used to control the upload of large files which need chunking
 type largeUpload struct {
-	f        *Fs                             // parent Fs
-	o        *Object                         // object being uploaded
-	in       io.Reader                       // read the data from here
-	wrap     accounting.WrapFn               // account parts being transferred
-	id       string                          // ID of the file being uploaded
-	size     int64                           // total size
-	parts    int64                           // calculated number of parts, if known
-	sha1s    []string                        // slice of SHA1s for each part
-	uploadMu sync.Mutex                      // lock for upload variable
-	uploads  []*api.GetUploadPartURLResponse // result of get upload URL calls
+	f         *Fs                             // parent Fs
+	o         *Object                         // object being uploaded
+	doCopy    bool                            // doing copy rather than upload
+	what      string                          // text name of operation for logs
+	in        io.Reader                       // read the data from here
+	wrap      accounting.WrapFn               // account parts being transferred
+	id        string                          // ID of the file being uploaded
+	size      int64                           // total size
+	parts     int64                           // calculated number of parts, if known
+	sha1s     []string                        // slice of SHA1s for each part
+	uploadMu  sync.Mutex                      // lock for upload variable
+	uploads   []*api.GetUploadPartURLResponse // result of get upload URL calls
+	chunkSize int64                           // chunk size to use
+	src       *Object                         // if copying, object we are reading from
 }
 
 // newLargeUpload starts an upload of object o from in with metadata in src
-func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) {
+//
+// If newInfo is set then metadata from that will be used instead of reading it from src
+func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
 	remote := o.remote
 	size := src.Size()
 	parts := int64(0)
@@ -89,8 +97,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if size == -1 {
 		fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
 	} else {
-		parts = size / int64(o.fs.opt.ChunkSize)
-		if size%int64(o.fs.opt.ChunkSize) != 0 {
+		parts = size / int64(chunkSize)
+		if size%int64(chunkSize) != 0 {
 			parts++
 		}
 		if parts > maxParts {
@@ -99,7 +107,6 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		sha1SliceSize = parts
 	}
 
-	modTime := src.ModTime(ctx)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_start_large_file",
@@ -110,18 +117,24 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 		return nil, err
 	}
 	var request = api.StartLargeFileRequest{
-		BucketID:    bucketID,
-		Name:        f.opt.Enc.FromStandardPath(bucketPath),
-		ContentType: fs.MimeType(ctx, src),
-		Info: map[string]string{
-			timeKey: timeString(modTime),
-		},
+		BucketID: bucketID,
+		Name:     f.opt.Enc.FromStandardPath(bucketPath),
 	}
-	// Set the SHA1 if known
-	if !o.fs.opt.DisableCheckSum {
-		if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
-			request.Info[sha1Key] = calculatedSha1
+	if newInfo == nil {
+		modTime := src.ModTime(ctx)
+		request.ContentType = fs.MimeType(ctx, src)
+		request.Info = map[string]string{
+			timeKey: timeString(modTime),
 		}
+		// Set the SHA1 if known
+		if !o.fs.opt.DisableCheckSum || doCopy {
+			if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
+				request.Info[sha1Key] = calculatedSha1
+			}
+		}
+	} else {
+		request.ContentType = newInfo.ContentType
+		request.Info = newInfo.Info
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {
@@ -131,18 +144,24 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
 	if err != nil {
 		return nil, err
 	}
+	up = &largeUpload{
+		f:         f,
+		o:         o,
+		doCopy:    doCopy,
+		what:      "upload",
+		id:        response.ID,
+		size:      size,
+		parts:     parts,
+		sha1s:     make([]string, sha1SliceSize),
+		chunkSize: int64(chunkSize),
+	}
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering
-	in, wrap := accounting.UnWrap(in)
-	up = &largeUpload{
-		f:     f,
-		o:     o,
-		in:    in,
-		wrap:  wrap,
-		id:    response.ID,
-		size:  size,
-		parts: parts,
-		sha1s: make([]string, sha1SliceSize),
+	if doCopy {
+		up.what = "copy"
+		up.src = src.(*Object)
+	} else {
+		up.in, up.wrap = accounting.UnWrap(in)
 	}
 	return up, nil
 }
@@ -256,9 +275,41 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
 	return err
 }
 
+// Copy a chunk
+func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
+	err := up.f.pacer.Call(func() (bool, error) {
+		fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
+		opts := rest.Opts{
+			Method: "POST",
+			Path:   "/b2_copy_part",
+		}
+		offset := (part - 1) * up.chunkSize // where we are in the source file
+		var request = api.CopyPartRequest{
+			SourceID:    up.src.id,
+			LargeFileID: up.id,
+			PartNumber:  part,
+			Range:       fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
+		}
+		var response api.UploadPartResponse
+		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
+		retry, err := up.f.shouldRetry(ctx, resp, err)
+		if err != nil {
+			fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
+		}
+		up.sha1s[part-1] = response.SHA1
+		return retry, err
+	})
+	if err != nil {
+		fs.Debugf(up.o, "Error copying chunk %d: %v", part, err)
+	} else {
+		fs.Debugf(up.o, "Done copying chunk %d", part)
+	}
+	return err
+}
+
 // finish closes off the large upload
 func (up *largeUpload) finish(ctx context.Context) error {
-	fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
+	fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_finish_large_file",
@@ -280,6 +331,7 @@ func (up *largeUpload) finish(ctx context.Context) error {
 
 // cancel aborts the large upload
 func (up *largeUpload) cancel(ctx context.Context) error {
+	fs.Debugf(up.o, "Cancelling large file %s", up.what)
 	opts := rest.Opts{
 		Method: "POST",
 		Path:   "/b2_cancel_large_file",
@@ -292,139 +344,139 @@ func (up *largeUpload) cancel(ctx context.Context) error {
 		resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
 		return up.f.shouldRetry(ctx, resp, err)
 	})
-	return err
-}
-
-func (up *largeUpload) managedTransferChunk(ctx context.Context, wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
-	wg.Add(1)
-	go func(part int64, buf []byte) {
-		defer wg.Done()
-		defer up.f.putUploadBlock(buf)
-		err := up.transferChunk(ctx, part, buf)
-		if err != nil {
-			select {
-			case errs <- err:
-			default:
-			}
-		}
-	}(part, buf)
-}
-
-func (up *largeUpload) finishOrCancelOnError(ctx context.Context, err error, errs chan error) error {
-	if err == nil {
-		select {
-		case err = <-errs:
-		default:
-		}
-	}
 	if err != nil {
-		fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
-		cancelErr := up.cancel(ctx)
-		if cancelErr != nil {
-			fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
-		}
-		return err
+		fs.Errorf(up.o, "Failed to cancel large file %s: %v", up.what, err)
 	}
-	return up.finish(ctx)
+	return err
 }
 
 // Stream uploads the chunks from the input, starting with a required initial
 // chunk. Assumes the file size is unknown and will upload until the input
 // reaches EOF.
+//
+// Note that initialUploadBlock must be returned to f.putBuf()
 func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
 	fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
-	errs := make(chan error, 1)
-	hasMoreParts := true
-	var wg sync.WaitGroup
-
-	// Transfer initial chunk
+	var (
+		g, gCtx      = errgroup.WithContext(ctx)
+		hasMoreParts = true
+	)
 	up.size = int64(len(initialUploadBlock))
-	up.managedTransferChunk(ctx, &wg, errs, 1, initialUploadBlock)
+	g.Go(func() error {
+		for part := int64(1); hasMoreParts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			var buf []byte
+			if part == 1 {
+				buf = initialUploadBlock
+			} else {
+				buf = up.f.getBuf(false)
+			}
 
-outer:
-	for part := int64(2); hasMoreParts; part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+			// Fail fast, in case an errgroup managed function returns an error
+			// gCtx is cancelled. There is no point in uploading all the other parts.
+			if gCtx.Err() != nil {
+				up.f.putBuf(buf, false)
+				return nil
+			}
+
+			// Read the chunk
+			var n int
+			if part == 1 {
+				n = len(buf)
+			} else {
+				n, err = io.ReadFull(up.in, buf)
+				if err == io.ErrUnexpectedEOF {
+					fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
+					buf = buf[:n]
+					hasMoreParts = false
+				} else if err == io.EOF {
+					fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
+					up.f.putBuf(buf, false)
+					return nil
+				} else if err != nil {
+					// other kinds of errors indicate failure
+					up.f.putBuf(buf, false)
+					return err
+				}
+			}
+
+			// Keep stats up to date
+			up.parts = part
+			up.size += int64(n)
+			if part > maxParts {
+				up.f.putBuf(buf, false)
+				return errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
+			}
+
+			part := part // for the closure
+			g.Go(func() (err error) {
+				defer up.f.putBuf(buf, false)
+				return up.transferChunk(gCtx, part, buf)
+			})
 		}
-
-		// Get a block of memory
-		buf := up.f.getUploadBlock()
-
-		// Read the chunk
-		var n int
-		n, err = io.ReadFull(up.in, buf)
-		if err == io.ErrUnexpectedEOF {
-			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
-			buf = buf[:n]
-			hasMoreParts = false
-			err = nil
-		} else if err == io.EOF {
-			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
-			up.f.putUploadBlock(buf)
-			err = nil
-			break outer
-		} else if err != nil {
-			// other kinds of errors indicate failure
-			up.f.putUploadBlock(buf)
-			break outer
-		}
-
-		// Keep stats up to date
-		up.parts = part
-		up.size += int64(n)
-		if part > maxParts {
-			err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
-			break outer
-		}
-
-		// Transfer the chunk
-		up.managedTransferChunk(ctx, &wg, errs, part, buf)
+		return nil
+	})
+	err = g.Wait()
+	if err != nil {
+		return err
 	}
-	wg.Wait()
 	up.sha1s = up.sha1s[:up.parts]
-
-	return up.finishOrCancelOnError(ctx, err, errs)
+	return up.finish(ctx)
 }
 
 // Upload uploads the chunks from the input
-func (up *largeUpload) Upload(ctx context.Context) error {
-	fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
-	remaining := up.size
-	errs := make(chan error, 1)
-	var wg sync.WaitGroup
-	var err error
-outer:
-	for part := int64(1); part <= up.parts; part++ {
-		// Check any errors
-		select {
-		case err = <-errs:
-			break outer
-		default:
+func (up *largeUpload) Upload(ctx context.Context) (err error) {
+	defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
+	fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
+	var (
+		g, gCtx   = errgroup.WithContext(ctx)
+		remaining = up.size
+	)
+	g.Go(func() error {
+		for part := int64(1); part <= up.parts; part++ {
+			// Get a block of memory from the pool and token which limits concurrency.
+			buf := up.f.getBuf(up.doCopy)
+
+			// Fail fast, in case an errgroup managed function returns an error
+			// gCtx is cancelled. There is no point in uploading all the other parts.
+			if gCtx.Err() != nil {
+				up.f.putBuf(buf, up.doCopy)
+				return nil
+			}
+
+			reqSize := remaining
+			if reqSize >= up.chunkSize {
+				reqSize = up.chunkSize
+			}
+
+			if !up.doCopy {
+				// Read the chunk
+				buf = buf[:reqSize]
+				_, err = io.ReadFull(up.in, buf)
+				if err != nil {
+					up.f.putBuf(buf, up.doCopy)
+					return err
+				}
+			}
+
+			part := part // for the closure
+			g.Go(func() (err error) {
+				defer up.f.putBuf(buf, up.doCopy)
+				if !up.doCopy {
+					err = up.transferChunk(gCtx, part, buf)
+				} else {
+					err = up.copyChunk(gCtx, part, reqSize)
+				}
+				return err
+			})
+			remaining -= reqSize
 		}
-
-		reqSize := remaining
-		if reqSize >= int64(up.f.opt.ChunkSize) {
-			reqSize = int64(up.f.opt.ChunkSize)
-		}
-
-		// Get a block of memory
-		buf := up.f.getUploadBlock()[:reqSize]
-
-		// Read the chunk
-		_, err = io.ReadFull(up.in, buf)
-		if err != nil {
-			up.f.putUploadBlock(buf)
-			break outer
-		}
-
-		// Transfer the chunk
-		up.managedTransferChunk(ctx, &wg, errs, part, buf)
-		remaining -= reqSize
+		return nil
+	})
+	err = g.Wait()
+	if err != nil {
+		return err
 	}
-	wg.Wait()
-
-	return up.finishOrCancelOnError(ctx, err, errs)
+	return up.finish(ctx)
 }

backend/box/api/types.go

@@ -222,3 +222,23 @@ type AppAuth struct {
 	PrivateKey  string `json:"privateKey"`
 	Passphrase  string `json:"passphrase"`
 }
+
+// User is returned from /users/me
+type User struct {
+	Type          string    `json:"type"`
+	ID            string    `json:"id"`
+	Name          string    `json:"name"`
+	Login         string    `json:"login"`
+	CreatedAt     time.Time `json:"created_at"`
+	ModifiedAt    time.Time `json:"modified_at"`
+	Language      string    `json:"language"`
+	Timezone      string    `json:"timezone"`
+	SpaceAmount   int64     `json:"space_amount"`
+	SpaceUsed     int64     `json:"space_used"`
+	MaxUploadSize int64     `json:"max_upload_size"`
+	Status        string    `json:"status"`
+	JobTitle      string    `json:"job_title"`
+	Phone         string    `json:"phone"`
+	Address       string    `json:"address"`
+	AvatarURL     string    `json:"avatar_url"`
+}

backend/box/box.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/jwtutil"
 
 	"github.com/youmark/pkcs8"
@@ -93,7 +94,7 @@ func init() {
 					log.Fatalf("Failed to configure token with jwt authentication: %v", err)
 				}
 			} else {
-				err = oauthutil.Config("box", name, m, oauthConfig)
+				err = oauthutil.Config("box", name, m, oauthConfig, nil)
 				if err != nil {
 					log.Fatalf("Failed to configure token with oauth authentication: %v", err)
 				}
@@ -112,7 +113,7 @@ func init() {
 			Advanced: true,
 		}, {
 			Name: "box_config_file",
-			Help: "Box App config.json location\nLeave blank normally.",
+			Help: "Box App config.json location\nLeave blank normally." + env.ShellExpandHelp,
 		}, {
 			Name:    "box_sub_type",
 			Default: "user",
@@ -153,6 +154,7 @@ func init() {
 }
 
 func refreshJWTToken(jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
+	jsonFile = env.ShellExpand(jsonFile)
 	boxConfig, err := getBoxConfig(jsonFile)
 	if err != nil {
 		log.Fatalf("Failed to configure token: %v", err)
@@ -296,7 +298,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an box 'url'
+// parsePath parses a box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -327,7 +329,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -615,10 +617,6 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -659,7 +657,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -715,13 +713,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -746,10 +738,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -883,6 +871,30 @@ func (f *Fs) move(ctx context.Context, endpoint, id, leaf, directoryID string) (
 	return info, nil
 }
 
+// About gets quota information
+func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/users/me",
+	}
+	var user api.User
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(ctx, &opts, nil, &user)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read user info")
+	}
+	// FIXME max upload size would be useful to use in Update
+	usage = &fs.Usage{
+		Used:  fs.NewUsageValue(user.SpaceUsed),                    // bytes in use
+		Total: fs.NewUsageValue(user.SpaceAmount),                  // bytes total
+		Free:  fs.NewUsageValue(user.SpaceAmount - user.SpaceUsed), // bytes free
+	}
+	return usage, nil
+}
+
 // Move src to this remote using server side move operations.
 //
 // This is stored with the remote path given
@@ -932,64 +944,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	_, err = f.move(ctx, "/folders/", srcID, leaf, directoryID)
+	_, err = f.move(ctx, "/folders/", srcID, dstLeaf, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -998,7 +960,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	id, err := f.dirCache.FindDir(ctx, remote, false)
 	var opts rest.Opts
 	if err == nil {
@@ -1037,6 +999,66 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
 	return info.SharedLink.URL, err
 }
 
+// deletePermanently permenently deletes a trashed file
+func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
+	opts := rest.Opts{
+		Method:     "DELETE",
+		NoResponse: true,
+	}
+	if itemType == api.ItemTypeFile {
+		opts.Path = "/files/" + id + "/trash"
+	} else {
+		opts.Path = "/folders/" + id + "/trash"
+	}
+	return f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.Call(ctx, &opts)
+		return shouldRetry(resp, err)
+	})
+}
+
+// CleanUp empties the trash
+func (f *Fs) CleanUp(ctx context.Context) (err error) {
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/folders/trash/items",
+		Parameters: url.Values{
+			"fields": []string{"type", "id"},
+		},
+	}
+	opts.Parameters.Set("limit", strconv.Itoa(listChunks))
+	offset := 0
+	for {
+		opts.Parameters.Set("offset", strconv.Itoa(offset))
+
+		var result api.FolderItems
+		var resp *http.Response
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
+			return shouldRetry(resp, err)
+		})
+		if err != nil {
+			return errors.Wrap(err, "couldn't list trash")
+		}
+		for i := range result.Entries {
+			item := &result.Entries[i]
+			if item.Type == api.ItemTypeFolder || item.Type == api.ItemTypeFile {
+				err := f.deletePermanently(ctx, item.Type, item.ID)
+				if err != nil {
+					return errors.Wrap(err, "failed to delete file")
+				}
+			} else {
+				fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)
+				continue
+			}
+		}
+		offset += result.Limit
+		if offset >= result.TotalCount {
+			break
+		}
+	}
+	return
+}
+
 // DirCacheFlush resets the directory cache - used in testing as an
 // optional interface
 func (f *Fs) DirCacheFlush() {
@@ -1244,7 +1266,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
@@ -1274,10 +1296,12 @@ var (
 	_ fs.Purger          = (*Fs)(nil)
 	_ fs.PutStreamer     = (*Fs)(nil)
 	_ fs.Copier          = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
+	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)
 )

backend/box/upload.go

@@ -19,6 +19,7 @@ import (
 	"github.com/rclone/rclone/backend/box/api"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/rest"
 )
 
@@ -182,15 +183,13 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
 	fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize))
 
 	// Cancel the session if something went wrong
-	defer func() {
-		if err != nil {
-			fs.Debugf(o, "Cancelling multipart upload: %v", err)
-			cancelErr := o.abortUpload(ctx, session.ID)
-			if cancelErr != nil {
-				fs.Logf(o, "Failed to cancel multipart upload: %v", err)
-			}
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(o, "Cancelling multipart upload: %v", err)
+		cancelErr := o.abortUpload(ctx, session.ID)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-	}()
+	})()
 
 	// unwrap the accounting from the input, we use wrap to put it
 	// back on after the buffering

backend/cache/cache.go

@@ -87,7 +87,7 @@ func init() {
 			Advanced: true,
 		}, {
 			Name:     "plex_insecure",
-			Help:     "Skip all certificate verifications when connecting to the Plex server",
+			Help:     "Skip all certificate verification when connecting to the Plex server",
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -339,7 +339,7 @@ func parseRootPath(path string) (string, error) {
 	return strings.Trim(path, "/"), nil
 }
 
-// NewFs constructs a Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -520,9 +520,6 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 	// override only those features that use a temp fs and it doesn't support them
 	//f.features.ChangeNotify = f.ChangeNotify
 	if f.opt.TempWritePath != "" {
-		if f.tempFs.Features().Copy == nil {
-			f.features.Copy = nil
-		}
 		if f.tempFs.Features().Move == nil {
 			f.features.Move = nil
 		}
@@ -1533,6 +1530,9 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		fs.Errorf(src, "source remote (%v) doesn't support Copy", src.Fs())
 		return nil, fs.ErrorCantCopy
 	}
+	if f.opt.TempWritePath != "" && src.Fs() == f.tempFs {
+		return nil, fs.ErrorCantCopy
+	}
 	// the source must be a cached object or we abort
 	srcObj, ok := src.(*Object)
 	if !ok {
@@ -1829,6 +1829,19 @@ func (f *Fs) isRootInPath(p string) bool {
 	return strings.HasPrefix(p, f.Root()+"/")
 }
 
+// MergeDirs merges the contents of all the directories passed
+// in into the first one and rmdirs the other directories.
+func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
+	do := f.Fs.Features().MergeDirs
+	if do == nil {
+		return errors.New("MergeDirs not supported")
+	}
+	for _, dir := range dirs {
+		_ = f.cache.RemoveDir(dir.Remote())
+	}
+	return do(ctx, dirs)
+}
+
 // DirCacheFlush flushes the dir cache
 func (f *Fs) DirCacheFlush() {
 	_ = f.cache.RemoveDir("")
@@ -1926,4 +1939,5 @@ var (
 	_ fs.UserInfoer     = (*Fs)(nil)
 	_ fs.Disconnecter   = (*Fs)(nil)
 	_ fs.Commander      = (*Fs)(nil)
+	_ fs.MergeDirser    = (*Fs)(nil)
 )

backend/cache/cache_internal_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/rclone/rclone/fs/object"
 	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fstest"
+	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
 	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
@@ -299,6 +300,7 @@ func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {
 }
 
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
+	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -342,6 +344,7 @@ func TestInternalDoubleWrittenContentMatches(t *testing.T) {
 }
 
 func TestInternalCachedUpdatedContentMatches(t *testing.T) {
+	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticucm%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
 	defer runInstance.cleanupFs(t, rootFs, boltDb)
@@ -1009,6 +1012,15 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		return nil, nil
 	}
 
+	// Config to pass to NewFs
+	m := configmap.Simple{}
+	for k, v := range r.runDefaultCfgMap {
+		m.Set(k, v)
+	}
+	for k, v := range flags {
+		m.Set(k, v)
+	}
+
 	// if the remote doesn't exist, create a new one with a local one for it
 	// identify which is the cache remote (it can be wrapped by a crypt too)
 	rootIsCrypt := false
@@ -1017,8 +1029,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		localRemote := remote + "-local"
 		config.FileSet(localRemote, "type", "local")
 		config.FileSet(localRemote, "nounc", "true")
-		config.FileSet(remote, "type", "cache")
-		config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
+		m.Set("type", "cache")
+		m.Set("remote", localRemote+":"+filepath.Join(os.TempDir(), localRemote))
 	} else {
 		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
@@ -1028,8 +1040,8 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		if remoteType != "cache" {
 			if remoteType == "crypt" {
 				rootIsCrypt = true
-				config.FileSet(remote, "password", cryptPassword1)
-				config.FileSet(remote, "password2", cryptPassword2)
+				m.Set("password", cryptPassword1)
+				m.Set("password2", cryptPassword2)
 			}
 			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
@@ -1055,14 +1067,6 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 
 	fs.Config.LowLevelRetries = 1
 
-	m := configmap.Simple{}
-	for k, v := range r.runDefaultCfgMap {
-		m.Set(k, v)
-	}
-	for k, v := range flags {
-		m.Set(k, v)
-	}
-
 	// Instantiate root
 	if purge {
 		boltDb.PurgeTempUploads()

backend/cache/cache_test.go

@@ -18,7 +18,7 @@ func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName:                   "TestCache:",
 		NilObject:                    (*cache.Object)(nil),
-		UnimplementableFsMethods:     []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
+		UnimplementableFsMethods:     []string{"PublicLink", "OpenWriterAt"},
 		UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
 		SkipInvalidUTF8:              true, // invalid UTF-8 confuses the cache
 	})

backend/crypt/cipher.go

@@ -217,7 +217,7 @@ func decodeFileName(in string) ([]byte, error) {
 // 2003 paper "A Parallelizable Enciphering Mode" by Halevi and
 // Rogaway.
 //
-// This makes for determinstic encryption which is what we want - the
+// This makes for deterministic encryption which is what we want - the
 // same filename must encrypt to the same thing.
 //
 // This means that
@@ -559,7 +559,7 @@ func (n *nonce) increment() {
 	n.carry(0)
 }
 
-// add an uint64 to the nonce
+// add a uint64 to the nonce
 func (n *nonce) add(x uint64) {
 	carry := uint16(0)
 	for i := 0; i < 8; i++ {

backend/crypt/cipher_test.go

@@ -929,7 +929,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 		assert.Equal(t, 0, n)
 	}
 
-	// Now try decoding it with a open/seek
+	// Now try decoding it with an open/seek
 	for _, offset := range trials {
 		for _, limit := range limits {
 			if offset+limit > len(plaintext) {

backend/crypt/crypt.go

@@ -72,6 +72,20 @@ NB If filename_encryption is "off" then this option will do nothing.`,
 			Name:       "password2",
 			Help:       "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
 			IsPassword: true,
+		}, {
+			Name:    "server_side_across_configs",
+			Default: false,
+			Help: `Allow server side operations (eg copy) to work across different crypt configs.
+
+Normally this option is not what you want, but if you have two crypts
+pointing to the same backend you can use it.
+
+This can be used, for example, to change file name encryption type
+without re-uploading all the data. Just make two crypt backends
+pointing to two different directories with the single changed
+parameter and use rclone move to move the files between the crypt
+remotes.`,
+			Advanced: true,
 		}, {
 			Name: "show_mapping",
 			Help: `For all files listed show how the names encrypt.
@@ -181,6 +195,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		CanHaveEmptyDirectories: true,
 		SetTier:                 true,
 		GetTier:                 true,
+		ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
 	}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
 
 	return f, err
@@ -193,6 +208,7 @@ type Options struct {
 	DirectoryNameEncryption bool   `config:"directory_name_encryption"`
 	Password                string `config:"password"`
 	Password2               string `config:"password2"`
+	ServerSideAcrossConfigs bool   `config:"server_side_across_configs"`
 	ShowMapping             bool   `config:"show_mapping"`
 }
 
@@ -241,7 +257,7 @@ func (f *Fs) add(entries *fs.DirEntries, obj fs.Object) {
 	*entries = append(*entries, f.newObject(obj))
 }
 
-// Encrypt an directory file name to entries.
+// Encrypt a directory file name to entries.
 func (f *Fs) addDir(ctx context.Context, entries *fs.DirEntries, dir fs.Directory) {
 	remote := dir.Remote()
 	decryptedRemote, err := f.cipher.DecryptDirName(remote)
@@ -656,7 +672,7 @@ func (f *Fs) DirCacheFlush() {
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	do := f.Fs.Features().PublicLink
 	if do == nil {
 		return "", errors.New("PublicLink not supported")
@@ -664,9 +680,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
 	o, err := f.NewObject(ctx, remote)
 	if err != nil {
 		// assume it is a directory
-		return do(ctx, f.cipher.EncryptDirName(remote))
+		return do(ctx, f.cipher.EncryptDirName(remote), expire, unlink)
 	}
-	return do(ctx, o.(*Object).Object.Remote())
+	return do(ctx, o.(*Object).Object.Remote(), expire, unlink)
 }
 
 // ChangeNotify calls the passed function with a path
@@ -943,7 +959,7 @@ func (o *ObjectInfo) Hash(ctx context.Context, hash hash.Type) (string, error) {
 	if srcObj, ok = o.ObjectInfo.(fs.Object); ok {
 		// Prefer direct interface assertion
 	} else if do, ok := o.ObjectInfo.(fs.ObjectUnWrapper); ok {
-		// Otherwise likely is a operations.OverrideRemote
+		// Otherwise likely is an operations.OverrideRemote
 		srcObj = do.UnWrap()
 	} else {
 		return "", nil

backend/crypt/crypt_internal_test.go

@@ -82,7 +82,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 
 	var oi fs.ObjectInfo = obj
 	if wrap {
-		// wrap the object in a fs.ObjectUnwrapper if required
+		// wrap the object in an fs.ObjectUnwrapper if required
 		oi = testWrapper{oi}
 	}
 

backend/drive/drive.go

@@ -18,12 +18,12 @@ import (
 	"mime"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"text/template"
 	"time"
 
@@ -40,6 +40,7 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
@@ -68,6 +69,8 @@ const (
 	minChunkSize     = 256 * fs.KibiByte
 	defaultChunkSize = 8 * fs.MebiByte
 	partialFields    = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails"
+	listRGrouping    = 50   // number of IDs to search at once when using ListR
+	listRInputBuffer = 1000 // size of input buffer when using ListR
 )
 
 // Globals
@@ -178,7 +181,7 @@ func init() {
 			}
 
 			if opt.ServiceAccountFile == "" {
-				err = oauthutil.Config("drive", name, m, driveConfig)
+				err = oauthutil.Config("drive", name, m, driveConfig, nil)
 				if err != nil {
 					log.Fatalf("Failed to configure token: %v", err)
 				}
@@ -226,7 +229,7 @@ in with the ID of the root folder.
 `,
 		}, {
 			Name: "service_account_file",
-			Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
 			Name:     "service_account_credentials",
 			Help:     "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
@@ -263,7 +266,7 @@ videos.
 Setting this flag will cause Google photos and videos to return a
 blank MD5 checksum.
 
-Google photos are identifed by being in the "photos" space.
+Google photos are identified by being in the "photos" space.
 
 Corrupted checksums are caused by Google modifying the image/video but
 not updating the checksum.`,
@@ -346,9 +349,12 @@ date is used.`,
 			Help:     "Size of listing chunk 100-1000. 0 to disable.",
 			Advanced: true,
 		}, {
-			Name:     "impersonate",
-			Default:  "",
-			Help:     "Impersonate this user when using a service account.",
+			Name:    "impersonate",
+			Default: "",
+			Help: `Impersonate this user when using a service account.
+
+Note that if this is used then "root_folder_id" will be ignored.
+`,
 			Advanced: true,
 		}, {
 			Name:    "alternate_export",
@@ -558,6 +564,9 @@ type Fs struct {
 	isTeamDrive      bool               // true if this is a team drive
 	fileFields       googleapi.Field    // fields to fetch file info with
 	m                configmap.Mapper
+	grouping         int32               // number of IDs to search at once in ListR - read with atomic
+	listRmu          *sync.Mutex         // protects listRempties
+	listRempties     map[string]struct{} // IDs of supposedly empty directories which triggered grouping disable
 }
 
 type baseObject struct {
@@ -999,7 +1008,7 @@ func createOAuthClient(opt *Options, name string, m configmap.Mapper) (*http.Cli
 
 	// try loading service account credentials from env variable, then from a file
 	if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, errors.Wrap(err, "error opening service account credentials file")
 		}
@@ -1079,11 +1088,14 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	f := &Fs{
-		name:  name,
-		root:  root,
-		opt:   *opt,
-		pacer: newPacer(opt),
-		m:     m,
+		name:         name,
+		root:         root,
+		opt:          *opt,
+		pacer:        newPacer(opt),
+		m:            m,
+		grouping:     listRGrouping,
+		listRmu:      new(sync.Mutex),
+		listRempties: make(map[string]struct{}),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.fileFields = f.getFileFields()
@@ -1109,9 +1121,20 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		}
 	}
 
+	// If impersonating warn about root_folder_id if set and unset it
+	//
+	// This is because rclone v1.51 and v1.52 cached root_folder_id when
+	// using impersonate which they shouldn't have done. It is possible
+	// someone is using impersonate and root_folder_id in which case this
+	// breaks their workflow. There isn't an easy way around that.
+	if opt.RootFolderID != "" && opt.RootFolderID != "appDataFolder" && opt.Impersonate != "" {
+		fs.Logf(f, "Ignoring cached root_folder_id when using --drive-impersonate")
+		opt.RootFolderID = ""
+	}
+
 	// set root folder for a team drive or query the user root folder
 	if opt.RootFolderID != "" {
-		// override root folder if set or cached in the config
+		// override root folder if set or cached in the config and not impersonating
 		f.rootFolderID = opt.RootFolderID
 	} else if f.isTeamDrive {
 		f.rootFolderID = f.opt.TeamDriveID
@@ -1128,7 +1151,10 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 			}
 		}
 		f.rootFolderID = rootID
-		m.Set("root_folder_id", rootID)
+		// Don't cache the root folder ID if impersonating
+		if opt.Impersonate == "" {
+			m.Set("root_folder_id", rootID)
+		}
 	}
 
 	f.dirCache = dircache.New(root, f.rootFolderID, f)
@@ -1220,7 +1246,7 @@ func (f *Fs) getFileFields() (fields googleapi.Field) {
 	return fields
 }
 
-// newRegularObject creates a fs.Object for a normal drive.File
+// newRegularObject creates an fs.Object for a normal drive.File
 func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
 	// wipe checksum if SkipChecksumGphotos and file is type Photo or Video
 	if f.opt.SkipChecksumGphotos {
@@ -1239,7 +1265,7 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
 	}
 }
 
-// newDocumentObject creates a fs.Object for a google docs drive.File
+// newDocumentObject creates an fs.Object for a google docs drive.File
 func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
 	mediaType, _, err := mime.ParseMediaType(exportMimeType)
 	if err != nil {
@@ -1270,7 +1296,7 @@ func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, expor
 	}, nil
 }
 
-// newLinkObject creates a fs.Object that represents a link a google docs drive.File
+// newLinkObject creates an fs.Object that represents a link a google docs drive.File
 func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMimeType string) (fs.Object, error) {
 	t := linkTemplate(exportMimeType)
 	if t == nil {
@@ -1296,9 +1322,9 @@ func (f *Fs) newLinkObject(remote string, info *drive.File, extension, exportMim
 	}, nil
 }
 
-// newObjectWithInfo creates a fs.Object for any drive.File
+// newObjectWithInfo creates an fs.Object for any drive.File
 //
-// When the drive.File cannot be represented as a fs.Object it will return (nil, nil).
+// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, error) {
 	// If item has MD5 sum or a length it is a file stored on drive
 	if info.Md5Checksum != "" || info.Size > 0 {
@@ -1309,9 +1335,9 @@ func (f *Fs) newObjectWithInfo(remote string, info *drive.File) (fs.Object, erro
 	return f.newObjectWithExportInfo(remote, info, extension, exportName, exportMimeType, isDocument)
 }
 
-// newObjectWithExportInfo creates a fs.Object for any drive.File and the result of findExportFormat
+// newObjectWithExportInfo creates an fs.Object for any drive.File and the result of findExportFormat
 //
-// When the drive.File cannot be represented as a fs.Object it will return (nil, nil).
+// When the drive.File cannot be represented as an fs.Object it will return (nil, nil).
 func (f *Fs) newObjectWithExportInfo(
 	remote string, info *drive.File,
 	extension, exportName, exportMimeType string, isDocument bool) (o fs.Object, err error) {
@@ -1402,6 +1428,7 @@ func (f *Fs) CreateDir(ctx context.Context, pathID, leaf string) (newID string,
 	leaf = f.opt.Enc.FromStandardName(leaf)
 	// fmt.Println("Making", path)
 	// Define the metadata for the directory we are going to create.
+	pathID = actualID(pathID)
 	createInfo := &drive.File{
 		Name:        leaf,
 		Description: leaf,
@@ -1562,10 +1589,6 @@ func (f *Fs) findImportFormat(mimeType string) string {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -1629,20 +1652,22 @@ func (s listRSlices) Less(i, j int) bool {
 	return s.dirs[i] < s.dirs[j]
 }
 
-// listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
+// listRRunner will read dirIDs from the in channel, perform the file listing and call cb with each DirEntry.
 //
 // In each cycle it will read up to grouping entries from the in channel without blocking.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
+func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error) {
 	var dirs []string
 	var paths []string
+	var grouping int32
 
 	for dir := range in {
 		dirs = append(dirs[:0], dir.id)
 		paths = append(paths[:0], dir.path)
+		grouping = atomic.LoadInt32(&f.grouping)
 	waitloop:
-		for i := 1; i < grouping; i++ {
+		for i := int32(1); i < grouping; i++ {
 			select {
 			case d, ok := <-in:
 				if !ok {
@@ -1655,6 +1680,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
 		}
 		listRSlices{dirs, paths}.Sort()
 		var iErr error
+		foundItems := false
 		_, err := f.list(ctx, dirs, "", false, false, false, func(item *drive.File) bool {
 			// shared with me items have no parents when at the root
 			if f.opt.SharedWithMe && len(item.Parents) == 0 && len(paths) == 1 && paths[0] == "" {
@@ -1662,6 +1688,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
 			}
 			for _, parent := range item.Parents {
 				var i int
+				foundItems = true
 				earlyExit := false
 				// If only one item in paths then no need to search for the ID
 				// assuming google drive is doing its job properly.
@@ -1702,6 +1729,53 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
 			}
 			return false
 		})
+		// Found no items in more than one directory. Retry these as
+		// individual directories This is to work around a bug in google
+		// drive where (A in parents) or (B in parents) returns nothing
+		// sometimes. See #3114, #4289 and
+		// https://issuetracker.google.com/issues/149522397
+		if len(dirs) > 1 && !foundItems {
+			if atomic.SwapInt32(&f.grouping, 1) != 1 {
+				fs.Logf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
+			}
+			var recycled = make([]listREntry, len(dirs))
+			f.listRmu.Lock()
+			for i := range dirs {
+				recycled[i] = listREntry{id: dirs[i], path: paths[i]}
+				// Make a note of these dirs - if they all turn
+				// out to be empty then we can re-enable grouping
+				f.listRempties[dirs[i]] = struct{}{}
+			}
+			f.listRmu.Unlock()
+			// recycle these in the background so we don't deadlock
+			// the listR runners if they all get here
+			wg.Add(len(recycled))
+			go func() {
+				for _, entry := range recycled {
+					in <- entry
+				}
+				fs.Debugf(f, "Recycled %d entries", len(recycled))
+			}()
+		}
+		// If using a grouping of 1 and dir was empty then check to see if it
+		// is part of the group that caused grouping to be disabled.
+		if grouping == 1 && len(dirs) == 1 && !foundItems {
+			f.listRmu.Lock()
+			if _, found := f.listRempties[dirs[0]]; found {
+				// Remove the ID
+				delete(f.listRempties, dirs[0])
+				// If no empties left => all the directories that
+				// triggered the grouping being set to 1 were actually
+				// empty so must have made a mistake
+				if len(f.listRempties) == 0 {
+					if atomic.SwapInt32(&f.grouping, listRGrouping) != listRGrouping {
+						fs.Logf(f, "Re-enabling ListR as previous detection was in error")
+					}
+				}
+			}
+			f.listRmu.Unlock()
+		}
+
 		for range dirs {
 			wg.Done()
 		}
@@ -1736,15 +1810,6 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in <-chan list
 // Don't implement this unless you have a more efficient way
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	const (
-		grouping    = 50
-		inputBuffer = 1000
-	)
-
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1753,7 +1818,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 
 	mu := sync.Mutex{} // protects in and overflow
 	wg := sync.WaitGroup{}
-	in := make(chan listREntry, inputBuffer)
+	in := make(chan listREntry, listRInputBuffer)
 	out := make(chan error, fs.Config.Checkers)
 	list := walk.NewListRHelper(callback)
 	overflow := []listREntry{}
@@ -1766,6 +1831,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 			job := listREntry{actualID(d.ID()), d.Remote()}
 			select {
 			case in <- job:
+				// Adding the wg after we've entered the item is
+				// safe here because we know when the callback
+				// is called we are holding a waitgroup.
 				wg.Add(1)
 			default:
 				overflow = append(overflow, job)
@@ -1779,7 +1847,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	in <- listREntry{directoryID, dir}
 
 	for i := 0; i < fs.Config.Checkers; i++ {
-		go f.listRRunner(ctx, &wg, in, out, cb, grouping)
+		go f.listRRunner(ctx, &wg, in, out, cb)
 	}
 	go func() {
 		// wait until the all directories are processed
@@ -1788,9 +1856,9 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		for len(overflow) > 0 {
 			mu.Lock()
 			l := len(overflow)
-			// only fill half of the channel to prevent entries beeing put into overflow again
-			if l > inputBuffer/2 {
-				l = inputBuffer / 2
+			// only fill half of the channel to prevent entries being put into overflow again
+			if l > listRInputBuffer/2 {
+				l = listRInputBuffer / 2
 			}
 			wg.Add(l)
 			for _, d := range overflow[:l] {
@@ -1914,16 +1982,17 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to resolve shortcut")
 	}
-	// make sure we use the Name and Parents from the original item
+	// make sure we use the Name, Parents and Trashed from the original item
 	newItem.Name = item.Name
 	newItem.Parents = item.Parents
+	newItem.Trashed = item.Trashed
 	// the new ID is a composite ID
 	newItem.Id = joinID(newItem.Id, item.Id)
 	return newItem, nil
 }
 
-// itemToDirEntry converts a drive.File to a fs.DirEntry.
-// When the drive.File cannot be represented as a fs.DirEntry
+// itemToDirEntry converts a drive.File to an fs.DirEntry.
+// When the drive.File cannot be represented as an fs.DirEntry
 // (nil, nil) is returned.
 func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry, err error) {
 	switch {
@@ -1949,7 +2018,7 @@ func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry,
 //
 // Used to create new objects
 func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Time) (*drive.File, error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -2074,7 +2143,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 	}
 	dstDir := dirs[0]
 	for _, srcDir := range dirs[1:] {
-		// list the the objects
+		// list the objects
 		infos := []*drive.File{}
 		_, err := f.list(ctx, []string{srcDir.ID()}, "", false, false, true, func(info *drive.File) bool {
 			infos = append(infos, info)
@@ -2112,13 +2181,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -2244,10 +2307,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read description for Google Doc")
 		}
-		// set the description if there is one, or use the default if not
-		if info.Description != "" {
-			createInfo.Description = info.Description
-		}
+		createInfo.Description = info.Description
 	} else {
 		// don't overwrite the description on copy for files
 		// this should work for docs but it doesn't - it is probably a bug in Google Drive
@@ -2294,11 +2354,11 @@ func (f *Fs) Purge(ctx context.Context) error {
 	if f.opt.TrashedOnly {
 		return errors.New("Can't purge with --drive-trashed-only. Use delete if you want to selectively delete files")
 	}
-	err := f.dirCache.FindRoot(ctx, false)
+	rootID, err := f.dirCache.RootID(ctx, false)
 	if err != nil {
 		return err
 	}
-	err = f.delete(ctx, shortcutID(f.dirCache.RootID()), f.opt.UseTrash)
+	err = f.delete(ctx, shortcutID(rootID), f.opt.UseTrash)
 	f.dirCache.ResetRoot()
 	if err != nil {
 		return err
@@ -2433,7 +2493,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	id, err := f.dirCache.FindDir(ctx, remote, false)
 	if err == nil {
 		fs.Debugf(f, "attempting to share directory '%s'", remote)
@@ -2482,77 +2542,19 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
+	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
+	_ = srcLeaf
 
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, dstDirectoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
 	dstDirectoryID = actualID(dstDirectoryID)
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src parent
-	var srcDirectoryID string
-	if srcRemote == "" {
-		srcDirectoryID, err = srcFs.dirCache.RootParentID()
-	} else {
-		_, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, srcRemote, false)
-	}
-	if err != nil {
-		return err
-	}
 	srcDirectoryID = actualID(srcDirectoryID)
 
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
 	// Do the move
 	patch := drive.File{
-		Name: leaf,
+		Name: dstLeaf,
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.svc.Files.Update(shortcutID(srcID), &patch).
@@ -2804,11 +2806,10 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 	isDir := false
 	if srcPath == "" {
 		// source is root directory
-		err = f.dirCache.FindRoot(ctx, false)
+		srcID, err = f.dirCache.RootID(ctx, false)
 		if err != nil {
 			return nil, err
 		}
-		srcID = f.dirCache.RootID()
 		isDir = true
 	} else if srcObj, err := srcFs.NewObject(ctx, srcPath); err != nil {
 		if err != fs.ErrorNotAFile {
@@ -3037,7 +3038,7 @@ func (f *Fs) getRemoteInfo(ctx context.Context, remote string) (info *drive.File
 // getRemoteInfoWithExport returns a drive.File and the export settings for the remote
 func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
 	info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, "", "", "", false, fs.ErrorObjectNotFound
@@ -3147,7 +3148,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
 	return req, res, nil
 }
 
-// openDocumentFile represents an documentObject open for reading.
+// openDocumentFile represents a documentObject open for reading.
 // Updates the object size after read successfully.
 type openDocumentFile struct {
 	o       *documentObject // Object we are reading for

backend/drive/drive_internal_test.go

@@ -274,7 +274,7 @@ func (f *Fs) InternalTestShortcuts(t *testing.T) {
 	const (
 		// from fstest/fstests/fstests.go
 		existingDir    = "hello? sausage"
-		existingFile   = "file name.txt"
+		existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
 		existingSubDir = "êé"
 	)
 	ctx := context.Background()

backend/dropbox/dropbox.go

@@ -117,7 +117,10 @@ func init() {
 		Description: "Dropbox",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.ConfigNoOffline("dropbox", name, m, dropboxConfig)
+			opt := oauthutil.Options{
+				NoOffline: true,
+			}
+			err := oauthutil.Config("dropbox", name, m, dropboxConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -779,11 +782,17 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	absPath := f.opt.Enc.FromStandardPath(path.Join(f.slashRoot, remote))
 	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
 	createArg := sharing.CreateSharedLinkWithSettingsArg{
 		Path: absPath,
+		// FIXME this gives settings_error/not_authorized/.. errors
+		// and the expires setting isn't in the documentation so remove
+		// for now.
+		// Settings: &sharing.SharedLinkSettings{
+		// 	Expires: time.Now().Add(time.Duration(expire)).UTC().Round(time.Second),
+		// },
 	}
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {

backend/fichier/api.go

@@ -148,11 +148,6 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
 }
 
 func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
-
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err

backend/fichier/fichier.go

@@ -264,7 +264,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -298,7 +298,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Put in to the remote path with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Put should either
 // return an error or upload it properly (rather than e.g. calling panic).
 //
@@ -334,7 +334,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
 		return nil, err
 	}
 
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -389,13 +389,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 //
 // Shouldn't return an error if it already exists
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -403,11 +397,6 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 //
 // Return an error if it doesn't exist or isn't empty
 func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return err

backend/fichier/object.go

@@ -101,7 +101,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/ftp/ftp.go

@@ -50,8 +50,19 @@ func init() {
 			IsPassword: true,
 			Required:   true,
 		}, {
-			Name:    "tls",
-			Help:    "Use FTP over TLS (Implicit)",
+			Name: "tls",
+			Help: `Use FTPS over TLS (Implicit)
+When using implicit FTP over TLS the client will connect using TLS
+right from the start, which in turn breaks the compatibility with
+non-TLS-aware servers. This is usually served over port 990 rather
+than port 21. Cannot be used in combination with explicit FTP.`,
+			Default: false,
+		}, {
+			Name: "explicit_tls",
+			Help: `Use FTP over TLS (Explicit)
+When using explicit FTP over TLS the client explicitly request
+security from the server in order to upgrade a plain text connection
+to an encrypted one. Cannot be used in combination with implicit FTP.`,
 			Default: false,
 		}, {
 			Name:     "concurrency",
@@ -72,7 +83,7 @@ func init() {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
-			// The FTP protocal can't handle trailing spaces (for instance
+			// The FTP protocol can't handle trailing spaces (for instance
 			// pureftpd turns them into _)
 			//
 			// proftpd can't handle '*' in file names
@@ -90,6 +101,7 @@ type Options struct {
 	Pass              string               `config:"pass"`
 	Port              string               `config:"port"`
 	TLS               bool                 `config:"tls"`
+	ExplicitTLS       bool                 `config:"explicit_tls"`
 	Concurrency       int                  `config:"concurrency"`
 	SkipVerifyTLSCert bool                 `config:"no_check_certificate"`
 	DisableEPSV       bool                 `config:"disable_epsv"`
@@ -152,12 +164,21 @@ func (f *Fs) Features() *fs.Features {
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
 	ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
-	if f.opt.TLS {
+	if f.opt.TLS && f.opt.ExplicitTLS {
+		fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+		return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
+	} else if f.opt.TLS {
 		tlsConfig := &tls.Config{
 			ServerName:         f.opt.Host,
 			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
 		}
 		ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
+	} else if f.opt.ExplicitTLS {
+		tlsConfig := &tls.Config{
+			ServerName:         f.opt.Host,
+			InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
+		}
+		ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
 	}
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))

backend/googlecloudstorage/googlecloudstorage.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
-	"os"
 	"path"
 	"strings"
 	"time"
@@ -38,6 +37,7 @@ import (
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
+	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
 	"golang.org/x/oauth2"
@@ -82,7 +82,7 @@ func init() {
 			if saFile != "" || saCreds != "" {
 				return
 			}
-			err := oauthutil.Config("google cloud storage", name, m, storageConfig)
+			err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -98,7 +98,7 @@ func init() {
 			Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",
 		}, {
 			Name: "service_account_file",
-			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
+			Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
 		}, {
 			Name: "service_account_credentials",
 			Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
@@ -405,7 +405,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// try loading service account credentials from env variable, then from a file
 	if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
-		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
+		loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
 		if err != nil {
 			return nil, errors.Wrap(err, "error opening service account credentials file")
 		}

backend/googlephotos/albums_test.go

@@ -30,7 +30,7 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a1)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
+		"one": {a1},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1": a1,
@@ -39,7 +39,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"one": a1,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one"},
+		"": {"one"},
 	}, albums.path)
 
 	a2 := &api.Album{
@@ -49,8 +49,8 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a2)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
-		"two": []*api.Album{a2},
+		"one": {a1},
+		"two": {a2},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1": a1,
@@ -61,7 +61,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"two": a2,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one", "two"},
+		"": {"one", "two"},
 	}, albums.path)
 
 	// Add a duplicate
@@ -72,8 +72,8 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a2a)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one": []*api.Album{a1},
-		"two": []*api.Album{a2, a2a},
+		"one": {a1},
+		"two": {a2, a2a},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":  a1,
@@ -86,7 +86,7 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"": []string{"one", "two {2}", "two {2a}"},
+		"": {"one", "two {2}", "two {2a}"},
 	}, albums.path)
 
 	// Add a sub directory
@@ -97,9 +97,9 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":    a1,
@@ -114,8 +114,8 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	// Add a weird path
@@ -126,10 +126,10 @@ func TestAlbumsAdd(t *testing.T) {
 	albums.add(a0)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"{0}":     []*api.Album{a0},
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"{0}":     {a0},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"0":    a0,
@@ -146,8 +146,8 @@ func TestAlbumsAdd(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}", "{0}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}", "{0}"},
+		"one": {"sub"},
 	}, albums.path)
 }
 
@@ -181,9 +181,9 @@ func TestAlbumsDel(t *testing.T) {
 	albums.add(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1":    a1,
@@ -198,16 +198,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a1)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"2":    a2,
@@ -220,16 +220,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2}", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2}", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a2)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"2a":   a2a,
@@ -240,16 +240,16 @@ func TestAlbumsDel(t *testing.T) {
 		"two {2a}": a2a,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one", "two {2a}"},
-		"one": []string{"sub"},
+		"":    {"one", "two {2a}"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a2a)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{
 		"1sub": a1sub,
@@ -258,16 +258,16 @@ func TestAlbumsDel(t *testing.T) {
 		"one/sub": a1sub,
 	}, albums.byTitle)
 	assert.Equal(t, map[string][]string{
-		"":    []string{"one"},
-		"one": []string{"sub"},
+		"":    {"one"},
+		"one": {"sub"},
 	}, albums.path)
 
 	albums.del(a1sub)
 
 	assert.Equal(t, map[string][]*api.Album{
-		"one":     []*api.Album{a1},
-		"two":     []*api.Album{a2, a2a},
-		"one/sub": []*api.Album{a1sub},
+		"one":     {a1},
+		"two":     {a2, a2a},
+		"one/sub": {a1sub},
 	}, albums.dupes)
 	assert.Equal(t, map[string]*api.Album{}, albums.byID)
 	assert.Equal(t, map[string]*api.Album{}, albums.byTitle)

backend/googlephotos/api/types.go

@@ -17,7 +17,7 @@ type Error struct {
 	Details ErrorDetails `json:"error"`
 }
 
-// Error statisfies error interface
+// Error satisfies error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Details.Message, e.Details.Code, e.Details.Status)
 }

backend/googlephotos/googlephotos.go

@@ -96,7 +96,7 @@ func init() {
 			}
 
 			// Do the oauth
-			err = oauthutil.Config("google photos", name, m, oauthConfig)
+			err = oauthutil.Config("google photos", name, m, oauthConfig, nil)
 			if err != nil {
 				golog.Fatalf("Failed to configure token: %v", err)
 			}

backend/googlephotos/pattern.go

@@ -53,6 +53,7 @@ var patterns = dirPatterns{
 				fs.NewDir(prefix+"album", f.dirTime()),
 				fs.NewDir(prefix+"shared-album", f.dirTime()),
 				fs.NewDir(prefix+"upload", f.dirTime()),
+				fs.NewDir(prefix+"feature", f.dirTime()),
 			}, nil
 		},
 	},
@@ -190,6 +191,28 @@ var patterns = dirPatterns{
 		re:     `^shared-album/(.+?)/([^/]+)$`,
 		isFile: true,
 	},
+	{
+		re: `^feature$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			return fs.DirEntries{
+				fs.NewDir(prefix+"favorites", f.dirTime()),
+			}, nil
+		},
+	},
+	{
+		re: `^feature/favorites$`,
+		toEntries: func(ctx context.Context, f lister, prefix string, match []string) (entries fs.DirEntries, err error) {
+			filter := featureFilter(ctx, f, match)
+			if err != nil {
+				return nil, err
+			}
+			return f.listDir(ctx, prefix, filter)
+		},
+	},
+	{
+		re:     `^feature/favorites/([^/]+)$`,
+		isFile: true,
+	},
 }.mustCompile()
 
 // mustCompile compiles the regexps in the dirPatterns
@@ -201,7 +224,7 @@ func (ds dirPatterns) mustCompile() dirPatterns {
 	return ds
 }
 
-// match finds the path passed in in the matching structure and
+// match finds the path passed in the matching structure and
 // returns the parameters and a pointer to the match, or nil.
 func (ds dirPatterns) match(root string, itemPath string, isFile bool) (match []string, prefix string, pattern *dirPattern) {
 	itemPath = strings.Trim(itemPath, "/")
@@ -290,6 +313,24 @@ func yearMonthDayFilter(ctx context.Context, f lister, match []string) (sf api.S
 	return sf, nil
 }
 
+// featureFilter creates a filter for the Feature enum
+//
+// The API only supports one feature, FAVORITES, so hardcode that feature
+//
+// https://developers.google.com/photos/library/reference/rest/v1/mediaItems/search#FeatureFilter
+func featureFilter(ctx context.Context, f lister, match []string) (sf api.SearchFilter) {
+	sf = api.SearchFilter{
+		Filters: &api.Filters{
+			FeatureFilter: &api.FeatureFilter{
+				IncludedFeatures: []string{
+					"FAVORITES",
+				},
+			},
+		},
+	}
+	return sf
+}
+
 // Turns an albumPath into entries
 //
 // These can either be synthetic directory entries if the album path

backend/googlephotos/pattern_test.go

@@ -155,6 +155,38 @@ func TestPatternMatch(t *testing.T) {
 			wantPrefix:  "file.jpg/",
 			wantPattern: &patterns[5],
 		},
+		{
+			root:        "",
+			itemPath:    "feature",
+			isFile:      false,
+			wantMatch:   []string{"feature"},
+			wantPrefix:  "feature/",
+			wantPattern: &patterns[23],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature",
+			itemPath:    "favorites",
+			isFile:      false,
+			wantMatch:   []string{"feature/favorites"},
+			wantPrefix:  "favorites/",
+			wantPattern: &patterns[24],
+		},
+		{
+			root:        "feature/favorites",
+			itemPath:    "file.jpg",
+			isFile:      true,
+			wantMatch:   []string{"feature/favorites/file.jpg", "file.jpg"},
+			wantPrefix:  "file.jpg/",
+			wantPattern: &patterns[25],
+		},
 	} {
 		t.Run(fmt.Sprintf("#%d,root=%q,itemPath=%q,isFile=%v", testNumber, test.root, test.itemPath, test.isFile), func(t *testing.T) {
 			gotMatch, gotPrefix, gotPattern := patterns.match(test.root, test.itemPath, test.isFile)

backend/hubic/auth.go

@@ -21,7 +21,7 @@ func newAuth(f *Fs) *auth {
 	}
 }
 
-// Request constructs a http.Request for authentication
+// Request constructs an http.Request for authentication
 //
 // returns nil for not needed
 func (a *auth) Request(*swift.Connection) (r *http.Request, err error) {

backend/hubic/hubic.go

@@ -39,7 +39,7 @@ var (
 	// Description of how to auth for this app
 	oauthConfig = &oauth2.Config{
 		Scopes: []string{
-			"credentials.r", // Read Openstack credentials
+			"credentials.r", // Read OpenStack credentials
 		},
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  "https://api.hubic.com/oauth/auth/",
@@ -58,7 +58,7 @@ func init() {
 		Description: "Hubic",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("hubic", name, m, oauthConfig)
+			err := oauthutil.Config("hubic", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -76,8 +76,8 @@ func init() {
 // credentials is the JSON returned from the Hubic API to read the
 // OpenStack credentials
 type credentials struct {
-	Token    string `json:"token"`    // Openstack token
-	Endpoint string `json:"endpoint"` // Openstack endpoint
+	Token    string `json:"token"`    // OpenStack token
+	Endpoint string `json:"endpoint"` // OpenStack endpoint
 	Expires  string `json:"expires"`  // Expires date - eg "2015-11-09T14:24:56+01:00"
 }
 

backend/jottacloud/jottacloud.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -26,6 +27,7 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
+	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
@@ -39,26 +41,33 @@ import (
 
 // Globals
 const (
-	minSleep          = 10 * time.Millisecond
-	maxSleep          = 2 * time.Second
-	decayConstant     = 2 // bigger for slower decay, exponential
-	defaultDevice     = "Jotta"
-	defaultMountpoint = "Archive"
-	rootURL           = "https://www.jottacloud.com/jfs/"
-	apiURL            = "https://api.jottacloud.com/"
-	baseURL           = "https://www.jottacloud.com/"
-	defaultTokenURL   = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
-	cachePrefix       = "rclone-jcmd5-"
-	configDevice      = "device"
-	configMountpoint  = "mountpoint"
-	configTokenURL    = "tokenURL"
-	configVersion     = 1
+	minSleep           = 10 * time.Millisecond
+	maxSleep           = 2 * time.Second
+	decayConstant      = 2 // bigger for slower decay, exponential
+	defaultDevice      = "Jotta"
+	defaultMountpoint  = "Archive"
+	rootURL            = "https://jfs.jottacloud.com/jfs/"
+	apiURL             = "https://api.jottacloud.com/"
+	baseURL            = "https://www.jottacloud.com/"
+	defaultTokenURL    = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
+	cachePrefix        = "rclone-jcmd5-"
+	configDevice       = "device"
+	configMountpoint   = "mountpoint"
+	configTokenURL     = "tokenURL"
+	configClientID     = "client_id"
+	configClientSecret = "client_secret"
+	configVersion      = 1
+
+	v1tokenURL              = "https://api.jottacloud.com/auth/v1/token"
+	v1registerURL           = "https://api.jottacloud.com/auth/v1/register"
+	v1ClientID              = "nibfk8biu12ju7hpqomr8b1e40"
+	v1EncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
+	v1configVersion         = 0
 )
 
 var (
 	// Description of how to auth for this app for a personal account
 	oauthConfig = &oauth2.Config{
-		ClientID: "jottacli",
 		Endpoint: oauth2.Endpoint{
 			AuthURL:  defaultTokenURL,
 			TokenURL: defaultTokenURL,
@@ -72,7 +81,7 @@ func init() {
 	// needs to be done early so we can use oauth during config
 	fs.Register(&fs.RegInfo{
 		Name:        "jottacloud",
-		Description: "JottaCloud",
+		Description: "Jottacloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
 			ctx := context.TODO()
@@ -83,7 +92,7 @@ func init() {
 				if err != nil {
 					log.Fatalf("Failed to parse config version - corrupted config")
 				}
-				refresh = ver != configVersion
+				refresh = (ver != configVersion) && (ver != v1configVersion)
 			}
 
 			if refresh {
@@ -98,42 +107,12 @@ func init() {
 				}
 			}
 
-			clientConfig := *fs.Config
-			clientConfig.UserAgent = "JottaCli 0.6.18626 windows-amd64"
-			srv := rest.NewClient(fshttp.NewClient(&clientConfig))
-
-			fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
-			fmt.Printf("Login Token> ")
-			loginToken := config.ReadLine()
-
-			token, err := doAuth(ctx, srv, loginToken, m)
-			if err != nil {
-				log.Fatalf("Failed to get oauth token: %s", err)
-			}
-			err = oauthutil.PutToken(name, m, &token, true)
-			if err != nil {
-				log.Fatalf("Error while saving token: %s", err)
-			}
-
-			fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+			fmt.Printf("Use legacy authentification?.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n")
 			if config.Confirm(false) {
-				oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
-				if err != nil {
-					log.Fatalf("Failed to load oAuthClient: %s", err)
-				}
-
-				srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
-				apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
-
-				device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
-				if err != nil {
-					log.Fatalf("Failed to setup mountpoint: %s", err)
-				}
-				m.Set(configDevice, device)
-				m.Set(configMountpoint, mountpoint)
+				v1config(ctx, name, m)
+			} else {
+				v2config(ctx, name, m)
 			}
-
-			m.Set("configVersion", strconv.Itoa(configVersion))
 		},
 		Options: []fs.Option{{
 			Name:     "md5_memory_limit",
@@ -150,11 +129,6 @@ func init() {
 			Help:     "Delete files permanently rather than putting them into the trash.",
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name:     "unlink",
-			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     "upload_resume_limit",
 			Help:     "Files bigger than this can be resumed if the upload fail's.",
@@ -181,7 +155,6 @@ type Options struct {
 	MD5MemoryThreshold fs.SizeSuffix        `config:"md5_memory_limit"`
 	TrashedOnly        bool                 `config:"trashed_only"`
 	HardDelete         bool                 `config:"hard_delete"`
-	Unlink             bool                 `config:"unlink"`
 	UploadThreshold    fs.SizeSuffix        `config:"upload_resume_limit"`
 	Enc                encoder.MultiEncoder `config:"encoding"`
 }
@@ -235,7 +208,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an box 'url'
+// parsePath parses a box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -257,8 +230,181 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// doAuth runs the actual token request
-func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
+// v1config configure a jottacloud backend using legacy authentification
+func v1config(ctx context.Context, name string, m configmap.Mapper) {
+	srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+	fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
+	if config.Confirm(false) {
+		deviceRegistration, err := registerDevice(ctx, srv)
+		if err != nil {
+			log.Fatalf("Failed to register device: %v", err)
+		}
+
+		m.Set(configClientID, deviceRegistration.ClientID)
+		m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
+		fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
+	}
+
+	clientID, ok := m.Get(configClientID)
+	if !ok {
+		clientID = v1ClientID
+	}
+	clientSecret, ok := m.Get(configClientSecret)
+	if !ok {
+		clientSecret = v1EncryptedClientSecret
+	}
+	oauthConfig.ClientID = clientID
+	oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
+	oauthConfig.Endpoint.AuthURL = v1tokenURL
+	oauthConfig.Endpoint.TokenURL = v1tokenURL
+
+	fmt.Printf("Username> ")
+	username := config.ReadLine()
+	password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
+
+	token, err := doAuthV1(ctx, srv, username, password)
+	if err != nil {
+		log.Fatalf("Failed to get oauth token: %s", err)
+	}
+	err = oauthutil.PutToken(name, m, &token, true)
+	if err != nil {
+		log.Fatalf("Error while saving token: %s", err)
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(v1configVersion))
+}
+
+// registerDevice register a new device for use with the jottacloud API
+func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
+	// random generator to generate random device names
+	seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
+	randonDeviceNamePartLength := 21
+	randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
+	charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	for i := range randomDeviceNamePart {
+		randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
+	}
+	randomDeviceName := "rclone-" + string(randomDeviceNamePart)
+	fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
+
+	values := url.Values{}
+	values.Set("device_id", randomDeviceName)
+
+	opts := rest.Opts{
+		Method:       "POST",
+		RootURL:      v1registerURL,
+		ContentType:  "application/x-www-form-urlencoded",
+		ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
+		Parameters:   values,
+	}
+
+	var deviceRegistration *api.DeviceRegistrationResponse
+	_, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
+	return deviceRegistration, err
+}
+
+// doAuthV1 runs the actual token request for V1 authentification
+func doAuthV1(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
+	// prepare out token request with username and password
+	values := url.Values{}
+	values.Set("grant_type", "PASSWORD")
+	values.Set("password", password)
+	values.Set("username", username)
+	values.Set("client_id", oauthConfig.ClientID)
+	values.Set("client_secret", oauthConfig.ClientSecret)
+	opts := rest.Opts{
+		Method:      "POST",
+		RootURL:     oauthConfig.Endpoint.AuthURL,
+		ContentType: "application/x-www-form-urlencoded",
+		Parameters:  values,
+	}
+
+	// do the first request
+	var jsonToken api.TokenJSON
+	resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
+	if err != nil {
+		// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
+		if resp != nil {
+			if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
+				fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
+				fmt.Printf("Enter verification code> ")
+				authCode := config.ReadLine()
+
+				authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
+				opts.ExtraHeaders = make(map[string]string)
+				opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
+				resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
+			}
+		}
+	}
+
+	token.AccessToken = jsonToken.AccessToken
+	token.RefreshToken = jsonToken.RefreshToken
+	token.TokenType = jsonToken.TokenType
+	token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
+	return token, err
+}
+
+// v2config configure a jottacloud backend using the modern JottaCli token based authentification
+func v2config(ctx context.Context, name string, m configmap.Mapper) {
+	srv := rest.NewClient(fshttp.NewClient(fs.Config))
+
+	fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
+	fmt.Printf("Login Token> ")
+	loginToken := config.ReadLine()
+
+	token, err := doAuthV2(ctx, srv, loginToken, m)
+	if err != nil {
+		log.Fatalf("Failed to get oauth token: %s", err)
+	}
+	err = oauthutil.PutToken(name, m, &token, true)
+	if err != nil {
+		log.Fatalf("Error while saving token: %s", err)
+	}
+
+	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
+	if config.Confirm(false) {
+		oauthConfig.ClientID = "jottacli"
+		oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
+		if err != nil {
+			log.Fatalf("Failed to load oAuthClient: %s", err)
+		}
+
+		srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
+		apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
+		device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
+		if err != nil {
+			log.Fatalf("Failed to setup mountpoint: %s", err)
+		}
+		m.Set(configDevice, device)
+		m.Set(configMountpoint, mountpoint)
+	}
+
+	m.Set("configVersion", strconv.Itoa(configVersion))
+}
+
+// doAuthV2 runs the actual token request for V2 authentification
+func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
 	loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
 	if err != nil {
 		return token, err
@@ -454,7 +600,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// Jottacloud want's '+' to be URL encoded even though the RFC states it's not reserved
+// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
 func urlPathEscape(in string) string {
 	return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
 }
@@ -464,11 +610,34 @@ func (f *Fs) filePathRaw(file string) string {
 	return path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(path.Join(f.root, file)))
 }
 
-// filePath returns a escaped file path (f.root, file)
+// filePath returns an escaped file path (f.root, file)
 func (f *Fs) filePath(file string) string {
 	return urlPathEscape(f.filePathRaw(file))
 }
 
+// Jottacloud requires the grant_type 'refresh_token' string
+// to be uppercase and throws a 400 Bad Request if we use the
+// lower case used by the oauth2 module
+//
+// This filter catches all refresh requests, reads the body,
+// changes the case and then sends it on
+func grantTypeFilter(req *http.Request) {
+	if v1tokenURL == req.URL.String() {
+		// read the entire body
+		refreshBody, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			return
+		}
+		_ = req.Body.Close()
+
+		// make the refresh token upper case
+		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
+
+		// set the new ReadCloser (with a dummy Close())
+		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+	}
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	ctx := context.TODO()
@@ -480,28 +649,55 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Check config version
-	var ok bool
-	var version string
-	if version, ok = m.Get("configVersion"); ok {
-		ver, err := strconv.Atoi(version)
+	var ver int
+	version, ok := m.Get("configVersion")
+	if ok {
+		ver, err = strconv.Atoi(version)
 		if err != nil {
 			return nil, errors.New("Failed to parse config version")
 		}
-		ok = ver == configVersion
+		ok = (ver == configVersion) || (ver == v1configVersion)
 	}
 	if !ok {
 		return nil, errors.New("Outdated config - please reconfigure this backend")
 	}
 
-	// if custome endpoints are set use them else stick with defaults
-	if tokenURL, ok := m.Get(configTokenURL); ok {
-		oauthConfig.Endpoint.TokenURL = tokenURL
-		// jottacloud is weird. we need to use the tokenURL as authURL
-		oauthConfig.Endpoint.AuthURL = tokenURL
+	baseClient := fshttp.NewClient(fs.Config)
+
+	if ver == configVersion {
+		oauthConfig.ClientID = "jottacli"
+		// if custom endpoints are set use them else stick with defaults
+		if tokenURL, ok := m.Get(configTokenURL); ok {
+			oauthConfig.Endpoint.TokenURL = tokenURL
+			// jottacloud is weird. we need to use the tokenURL as authURL
+			oauthConfig.Endpoint.AuthURL = tokenURL
+		}
+	} else if ver == v1configVersion {
+		clientID, ok := m.Get(configClientID)
+		if !ok {
+			clientID = v1ClientID
+		}
+		clientSecret, ok := m.Get(configClientSecret)
+		if !ok {
+			clientSecret = v1EncryptedClientSecret
+		}
+		oauthConfig.ClientID = clientID
+		oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
+
+		oauthConfig.Endpoint.TokenURL = v1tokenURL
+		oauthConfig.Endpoint.AuthURL = v1tokenURL
+
+		// add the request filter to fix token refresh
+		if do, ok := baseClient.Transport.(interface {
+			SetRequestFilter(f func(req *http.Request))
+		}); ok {
+			do.SetRequestFilter(grantTypeFilter)
+		} else {
+			fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
+		}
 	}
 
 	// Create OAuth Client
-	baseClient := fshttp.NewClient(fs.Config)
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
 	if err != nil {
 		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
@@ -891,7 +1087,8 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		retry, _ := shouldRetry(resp, err)
+		return (retry && resp.StatusCode != 500), err
 	})
 	if err != nil {
 		return nil, err
@@ -995,6 +1192,18 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
+	// surprise! jottacloud fucked up dirmove - the api spits out an error but
+	// dir gets moved regardless
+	if apiErr, ok := err.(*api.Error); ok {
+		if apiErr.StatusCode == 500 {
+			_, err := f.NewObject(ctx, dstRemote)
+			if err == fs.ErrorNotAFile {
+				log.Printf("FIXME: ignoring DirMove error - move succeeded anyway\n")
+				return nil
+			}
+			return err
+		}
+	}
 	if err != nil {
 		return errors.Wrap(err, "couldn't move directory")
 	}
@@ -1002,14 +1211,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	opts := rest.Opts{
 		Method:     "GET",
 		Path:       f.filePath(remote),
 		Parameters: url.Values{},
 	}
 
-	if f.opt.Unlink {
+	if unlink {
 		opts.Parameters.Set("mode", "disableShare")
 	} else {
 		opts.Parameters.Set("mode", "enableShare")
@@ -1029,12 +1238,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 		}
 	}
 	if err != nil {
-		if f.opt.Unlink {
+		if unlink {
 			return "", errors.Wrap(err, "couldn't remove public link")
 		}
 		return "", errors.Wrap(err, "couldn't create public link")
 	}
-	if f.opt.Unlink {
+	if unlink {
 		if result.PublicSharePath != "" {
 			return "", errors.Errorf("couldn't remove public link - %q", result.PublicSharePath)
 		}
@@ -1105,7 +1314,7 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// filePath returns a escaped file path (f.root, remote)
+// filePath returns an escaped file path (f.root, remote)
 func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
@@ -1210,7 +1419,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 // The cleanup function should be called when out is finished with
 // regardless of whether this function returned an error or not.
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
-	// we need a MD5
+	// we need an MD5
 	md5Hasher := md5.New()
 	// use the teeReader to write to the local file AND calculate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)

backend/koofr/koofr.go

@@ -79,7 +79,7 @@ type Options struct {
 	Enc      encoder.MultiEncoder `config:"encoding"`
 }
 
-// A Fs is a representation of a remote Koofr Fs
+// An Fs is a representation of a remote Koofr Fs
 type Fs struct {
 	name     string
 	mountID  string
@@ -250,7 +250,7 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5)
 }
 
-// fullPath constructs a full, absolute path from a Fs root relative path,
+// fullPath constructs a full, absolute path from an Fs root relative path,
 func (f *Fs) fullPath(part string) string {
 	return f.opt.Enc.FromStandardPath(path.Join("/", f.root, part))
 }
@@ -421,7 +421,7 @@ func translateErrorsObject(err error) error {
 }
 
 // mkdir creates a directory at the given remote path. Creates ancestors if
-// neccessary
+// necessary
 func (f *Fs) mkdir(fullPath string) error {
 	if fullPath == "/" {
 		return nil
@@ -603,7 +603,7 @@ func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link,
 }
 
 // PublicLink creates a public link to the remote path
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
 	if err != nil {
 		return "", translateErrorsDir(err)

backend/local/local.go

@@ -90,7 +90,24 @@ are being uploaded and aborts with a message which starts "can't copy
 
 However on some file systems this modification time check may fail (eg
 [Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
-check can be disabled with this flag.`,
+check can be disabled with this flag.
+
+If this flag is set, rclone will use its best efforts to transfer a
+file which is being updated. If the file is only having things
+appended to it (eg a log) then rclone will transfer the log file with
+the size it had the first time rclone saw it.
+
+If the file is being modified throughout (not just appended to) then
+the transfer may fail with a hash check failure.
+
+In detail, once the file has had stat() called on it for the first
+time we:
+
+- Only transfer the size that stat gave
+- Only checksum the size that stat gave
+- Don't update the stat info for the file
+
+`,
 			Default:  false,
 			Advanced: true,
 		}, {
@@ -118,6 +135,16 @@ Windows/macOS and case sensitive for everything else.  Use this flag
 to override the default choice.`,
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name: "no_sparse",
+			Help: `Disable sparse files for multi-thread downloads
+
+On Windows platforms rclone will make sparse files when doing
+multi-thread downloads. This avoids long pauses on large files where
+the OS zeros the file. However sparse files may be undesirable as they
+cause disk fragmentation and can be slow to work with.`,
+			Default:  false,
+			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -139,6 +166,7 @@ type Options struct {
 	OneFileSystem     bool                 `config:"one_file_system"`
 	CaseSensitive     bool                 `config:"case_sensitive"`
 	CaseInsensitive   bool                 `config:"case_insensitive"`
+	NoSparse          bool                 `config:"no_sparse"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -798,6 +826,10 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		} else {
 			in, err = o.openTranslatedLink(0, -1)
 		}
+		// If not checking for updates, only read size given
+		if o.fs.opt.NoCheckUpdated {
+			in = readers.NewLimitedReadCloser(in, o.size)
+		}
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -944,6 +976,13 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		}
 	}
 
+	// If not checking updated then limit to current size.  This means if
+	// file is being extended, readers will read a o.Size() bytes rather
+	// than the new size making for a consistent upload.
+	if limit < 0 && o.fs.opt.NoCheckUpdated {
+		limit = o.size
+	}
+
 	// Handle a translated link
 	if o.translatedLink {
 		return o.openTranslatedLink(offset, limit)
@@ -1095,6 +1134,8 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	return o.lstat()
 }
 
+var sparseWarning sync.Once
+
 // OpenWriterAt opens with a handle for random access writes
 //
 // Pass in the remote desired and the size if known.
@@ -1122,10 +1163,15 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 	if err != nil {
 		fs.Debugf(o, "Failed to pre-allocate: %v", err)
 	}
-	// Set the file to be a sparse file (important on Windows)
-	err = file.SetSparse(out)
-	if err != nil {
-		fs.Debugf(o, "Failed to set sparse: %v", err)
+	if !f.opt.NoSparse && file.SetSparseImplemented {
+		sparseWarning.Do(func() {
+			fs.Infof(nil, "Writing sparse files: use --local-no-sparse or --multi-thread-streams 0 to disable")
+		})
+		// Set the file to be a sparse file (important on Windows)
+		err = file.SetSparse(out)
+		if err != nil {
+			fs.Debugf(o, "Failed to set sparse: %v", err)
+		}
 	}
 
 	return out, nil
@@ -1133,6 +1179,10 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 
 // setMetadata sets the file info from the os.FileInfo passed in
 func (o *Object) setMetadata(info os.FileInfo) {
+	// if not checking updated then don't update the stat
+	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
+		return
+	}
 	// Don't overwrite the info if we don't need to
 	// this avoids upsetting the race detector
 	if o.size != info.Size() {
@@ -1146,7 +1196,7 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	}
 }
 
-// Stat a Object into info
+// Stat an Object into info
 func (o *Object) lstat() error {
 	info, err := o.fs.lstat(o.path)
 	if err == nil {

backend/mailru/mailru.go

@@ -402,7 +402,7 @@ func (q *quirks) parseQuirks(option string) {
 			// "Accept-Encoding: gzip" header. However, enabling compression
 			// might be good for performance.
 			// Use this quirk to investigate the performance impact.
-			// Remove this quirk if perfomance does not improve.
+			// Remove this quirk if performance does not improve.
 			q.gzip = true
 		case "insecure":
 			// The mailru disk-o protocol is not documented. To compare HTTP
@@ -1450,7 +1450,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	// fs.Debugf(f, ">>> PublicLink %q", remote)
 
 	token, err := f.accessToken()

backend/mega/mega.go

@@ -150,7 +150,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an mega 'url'
+// parsePath parses a mega 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -836,7 +836,7 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	root, err := f.findRoot(false)
 	if err != nil {
 		return "", errors.Wrap(err, "PublicLink failed to find root node")
@@ -871,7 +871,7 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
 			return errors.Errorf("MergeDirs failed to find node for: %v", srcDir)
 		}
 
-		// list the the objects
+		// list the objects
 		infos := []*mega.Node{}
 		_, err := f.list(ctx, srcDirNode, func(info *mega.Node) bool {
 			infos = append(infos, info)

backend/onedrive/api/types.go

@@ -272,19 +272,19 @@ type CreateShareLinkResponse struct {
 	} `json:"link"`
 }
 
-// AsyncOperationStatus provides information on the status of a asynchronous job progress.
+// AsyncOperationStatus provides information on the status of an asynchronous job progress.
 //
 // The following API calls return AsyncOperationStatus resources:
 //
 // Copy Item
 // Upload From URL
 type AsyncOperationStatus struct {
-	PercentageComplete float64 `json:"percentageComplete"` // An float value between 0 and 100 that indicates the percentage complete.
+	PercentageComplete float64 `json:"percentageComplete"` // A float value between 0 and 100 that indicates the percentage complete.
 	Status             string  `json:"status"`             // A string value that maps to an enumeration of possible values about the status of the job. "notStarted | inProgress | completed | updating | failed | deletePending | deleteFailed | waiting"
 }
 
 // GetID returns a normalized ID of the item
-// If DriveID is known it will be prefixed to the ID with # seperator
+// If DriveID is known it will be prefixed to the ID with # separator
 // Can be parsed using onedrive.parseNormalizedID(normalizedID)
 func (i *Item) GetID() string {
 	if i.IsRemote() && i.RemoteItem.ID != "" {

backend/onedrive/onedrive.go

@@ -79,7 +79,7 @@ func init() {
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
 			ctx := context.TODO()
-			err := oauthutil.Config("onedrive", name, m, oauthConfig)
+			err := oauthutil.Config("onedrive", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 				return
@@ -396,7 +396,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an one drive 'url'
+// parsePath parses a one drive 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -498,13 +498,13 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
 	var dirCacheFoundRoot bool
 	var rootNormalizedID string
 	if f.dirCache != nil {
-		var dirCacheRootIDExists bool
-		rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
+		rootNormalizedID, err = f.dirCache.RootID(ctx, false)
+		dirCacheRootIDExists := err == nil
 		if f.root == "" {
 			// if f.root == "", it means f.root is the absolute root of the drive
 			// and its ID should have been found in NewFs
 			dirCacheFoundRoot = dirCacheRootIDExists
-		} else if _, err := f.dirCache.RootParentID(); err == nil {
+		} else if _, err := f.dirCache.RootParentID(ctx, false); err == nil {
 			// if root is in a folder, it must have a parent folder, and
 			// if dirCache has found root in NewFs, the parent folder's ID
 			// should be present.
@@ -813,10 +813,6 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -864,7 +860,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, leaf, directoryID, err
 	}
@@ -895,13 +891,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -924,10 +914,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1173,50 +1159,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, dstDirectoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
 	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
 	_, srcDriveID, _ := parseNormalizedID(srcID)
 
 	if dstDriveID != srcDriveID {
@@ -1225,18 +1174,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		return fs.ErrorCantDirMove
 	}
 
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
 	// Get timestamps of src so they can be preserved
 	srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(ctx, srcID, "")
 	if err != nil {
@@ -1246,7 +1183,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	// Do the move
 	opts := newOptsCall(srcID, "PATCH", "")
 	move := api.MoveItemRequest{
-		Name: f.opt.Enc.FromStandardName(leaf),
+		Name: f.opt.Enc.FromStandardName(dstLeaf),
 		ParentReference: &api.ItemReference{
 			DriveID: dstDriveID,
 			ID:      parsedDstDirID,
@@ -1310,8 +1247,8 @@ func (f *Fs) Hashes() hash.Set {
 	return hash.Set(QuickXorHashType)
 }
 
-// PublicLink returns a link for downloading without accout.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+// PublicLink returns a link for downloading without account.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
 	if err != nil {
 		return "", err
@@ -1688,41 +1625,22 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
 		return nil, errors.New("unknown-sized upload not supported")
 	}
 
-	uploadURLChan := make(chan string, 1)
-	gracefulCancel := func() {
-		uploadURL, ok := <-uploadURLChan
-		// Reading from uploadURLChan blocks the atexit process until
-		// we are able to use uploadURL to cancel the upload
-		if !ok { // createUploadSession failed - no need to cancel upload
-			return
-		}
-
-		fs.Debugf(o, "Cancelling multipart upload")
-		cancelErr := o.cancelUploadSession(ctx, uploadURL)
-		if cancelErr != nil {
-			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
-		}
-	}
-	cancelFuncHandle := atexit.Register(gracefulCancel)
-
 	// Create upload session
 	fs.Debugf(o, "Starting multipart upload")
 	session, err := o.createUploadSession(ctx, modTime)
 	if err != nil {
-		close(uploadURLChan)
-		atexit.Unregister(cancelFuncHandle)
 		return nil, err
 	}
 	uploadURL := session.UploadURL
-	uploadURLChan <- uploadURL
 
-	defer func() {
-		if err != nil {
-			fs.Debugf(o, "Error encountered during upload: %v", err)
-			gracefulCancel()
+	// Cancel the session if something went wrong
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(o, "Cancelling multipart upload: %v", err)
+		cancelErr := o.cancelUploadSession(ctx, uploadURL)
+		if cancelErr != nil {
+			fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-		atexit.Unregister(cancelFuncHandle)
-	}()
+	})()
 
 	// Upload the chunks
 	remaining := size

backend/opendrive/opendrive.go

@@ -63,7 +63,7 @@ func init() {
 			//   ? (question mark) -> '？' // FULLWIDTH QUESTION MARK
 			//   * (asterisk)      -> '＊' // FULLWIDTH ASTERISK
 			//
-			// Additionally names can't begin or end with a ASCII whitespace.
+			// Additionally names can't begin or end with an ASCII whitespace.
 			// List of replaced characters:
 			//     (space)           -> '␠'  // SYMBOL FOR SPACE
 			//     (horizontal tab)  -> '␉'  // SYMBOL FOR HORIZONTAL TABULATION
@@ -280,13 +280,7 @@ func errorHandler(resp *http.Response) error {
 // Mkdir creates the folder if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	// fs.Debugf(nil, "Mkdir(\"%s\")", dir)
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -312,10 +306,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -483,58 +473,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -546,9 +486,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		moveFolderData := moveCopyFolder{
 			SessionID:     f.session.SessionID,
 			FolderID:      srcID,
-			DstFolderID:   directoryID,
+			DstFolderID:   dstDirectoryID,
 			Move:          "true",
-			NewFolderName: leaf,
+			NewFolderName: dstLeaf,
 		}
 		opts := rest.Opts{
 			Method: "POST",
@@ -620,7 +560,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, leaf, directoryID, err
 	}
@@ -677,7 +617,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 	}
 
 	if "" == o.id {
-		// We need to create a ID for this file
+		// We need to create an ID for this file
 		var resp *http.Response
 		response := createFileResponse{}
 		err := o.fs.pacer.Call(func() (bool, error) {
@@ -803,10 +743,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	// fs.Debugf(nil, "List(%v)", dir)
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -1079,7 +1015,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 }
 
 func (o *Object) readMetaData(ctx context.Context) (err error) {
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return fs.ErrorObjectNotFound

backend/opendrive/types.go

@@ -18,13 +18,13 @@ func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }
 
-// Account describes a OpenDRIVE account
+// Account describes an OpenDRIVE account
 type Account struct {
 	Username string `json:"username"`
 	Password string `json:"passwd"`
 }
 
-// UserSessionInfo describes a OpenDRIVE session
+// UserSessionInfo describes an OpenDRIVE session
 type UserSessionInfo struct {
 	Username string `json:"username"`
 	Password string `json:"passwd"`
@@ -45,7 +45,7 @@ type UserSessionInfo struct {
 	PartnerUsersDomain string          `json:"PartnerUsersDomain"`
 }
 
-// FolderList describes a OpenDRIVE listing
+// FolderList describes an OpenDRIVE listing
 type FolderList struct {
 	// DirUpdateTime    string   `json:"DirUpdateTime,string"`
 	Name             string   `json:"Name"`
@@ -56,7 +56,7 @@ type FolderList struct {
 	Files            []File   `json:"Files"`
 }
 
-// Folder describes a OpenDRIVE folder
+// Folder describes an OpenDRIVE folder
 type Folder struct {
 	FolderID      string `json:"FolderID"`
 	Name          string `json:"Name"`
@@ -109,7 +109,7 @@ type removeFolder struct {
 	FolderID  string `json:"folder_id"`
 }
 
-// File describes a OpenDRIVE file
+// File describes an OpenDRIVE file
 type File struct {
 	FileID            string `json:"FileId"`
 	FileHash          string `json:"FileHash"`

backend/pcloud/api/types.go

@@ -152,6 +152,14 @@ type ChecksumFileResult struct {
 	Metadata Item `json:"metadata"`
 }
 
+// PubLinkResult is returned from /getfilepublink and /getfolderpublink
+type PubLinkResult struct {
+	Error
+	LinkID   int    `json:"linkid"`
+	Link     string `json:"link"`
+	LinkCode string `json:"code"`
+}
+
 // UserInfo is returned from /userinfo
 type UserInfo struct {
 	Error

backend/pcloud/pcloud.go

@@ -67,7 +67,10 @@ func init() {
 		Description: "Pcloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("pcloud", name, m, oauthConfig)
+			opt := oauthutil.Options{
+				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
+			}
+			err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -152,7 +155,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an pcloud 'url'
+// parsePath parses a pcloud 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -195,7 +198,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -452,10 +455,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -496,7 +495,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -527,13 +526,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -545,10 +538,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -656,7 +645,7 @@ func (f *Fs) Purge(ctx context.Context) error {
 
 // CleanUp empties the trash
 func (f *Fs) CleanUp(ctx context.Context) error {
-	err := f.dirCache.FindRoot(ctx, false)
+	rootID, err := f.dirCache.RootID(ctx, false)
 	if err != nil {
 		return err
 	}
@@ -665,7 +654,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
 		Path:       "/trash_clear",
 		Parameters: url.Values{},
 	}
-	opts.Parameters.Set("folderid", dirIDtoNumber(f.dirCache.RootID()))
+	opts.Parameters.Set("folderid", dirIDtoNumber(rootID))
 	var resp *http.Response
 	var result api.Error
 	return f.pacer.Call(func() (bool, error) {
@@ -738,58 +727,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -801,8 +740,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		Parameters: url.Values{},
 	}
 	opts.Parameters.Set("folderid", dirIDtoNumber(srcID))
-	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(leaf))
-	opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID))
+	opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(dstLeaf))
+	opts.Parameters.Set("tofolderid", dirIDtoNumber(dstDirectoryID))
 	var resp *http.Response
 	var result api.ItemResult
 	err = f.pacer.Call(func() (bool, error) {
@@ -824,6 +763,61 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
+func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (string, error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/getfolderpublink",
+		Parameters: url.Values{},
+	}
+	var result api.PubLinkResult
+	opts.Parameters.Set("folderid", dirIDtoNumber(dirID))
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
+		err = result.Error.Update(err)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return result.Link, err
+}
+
+func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (string, error) {
+	opts := rest.Opts{
+		Method:     "POST",
+		Path:       "/getfilepublink",
+		Parameters: url.Values{},
+	}
+	var result api.PubLinkResult
+	opts.Parameters.Set("path", path)
+	err := f.pacer.Call(func() (bool, error) {
+		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
+		err = result.Error.Update(err)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return result.Link, nil
+}
+
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	err := f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return "", err
+	}
+
+	dirID, err := f.dirCache.FindDir(ctx, remote, false)
+	if err == fs.ErrorDirNotFound {
+		return f.linkFile(ctx, remote, expire)
+	}
+	if err != nil {
+		return "", err
+	}
+	return f.linkDir(ctx, dirID, expire)
+}
+
 // About gets quota information
 func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 	opts := rest.Opts{
@@ -1053,7 +1047,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}
@@ -1158,6 +1152,7 @@ var (
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.IDer            = (*Object)(nil)

backend/premiumizeme/api/types.go

@@ -10,7 +10,7 @@ type Response struct {
 	Status  string `json:"status"`
 }
 
-// Error statisfies the error interface
+// Error satisfies the error interface
 func (e *Response) Error() string {
 	return fmt.Sprintf("%s: %s", e.Status, e.Message)
 }

backend/premiumizeme/premiumizeme.go

@@ -79,7 +79,7 @@ func init() {
 		Description: "premiumize.me",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("premiumizeme", name, m, oauthConfig)
+			err := oauthutil.Config("premiumizeme", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -183,7 +183,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -450,10 +450,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -493,7 +489,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -544,13 +540,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -562,10 +552,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -745,75 +731,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
-
-	// Find ID of src parent, not creating subdirs
-	var srcLeaf, srcDirectoryID string
-	findPath = srcRemote
-	if srcRemote == "" {
-		findPath = srcFs.root
-	}
-	srcLeaf, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
+	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	err = f.move(ctx, false, srcID, srcLeaf, leaf, srcDirectoryID, directoryID)
+	err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -822,7 +747,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	_, err := f.dirCache.FindDir(ctx, remote, false)
 	if err == nil {
 		return "", fs.ErrorCantShareDirectories
@@ -990,7 +915,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	size := src.Size()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}

backend/putio/fs.go

@@ -197,10 +197,6 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
 	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -260,7 +256,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 	// defer log.Trace(f, "src=%+v", src)("o=%+v, err=%v", &o, &err)
 	size := src.Size()
 	remote := src.Remote()
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -458,13 +454,7 @@ func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
-	err = f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err = f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -528,12 +518,11 @@ func (f *Fs) Purge(ctx context.Context) (err error) {
 	if f.root == "" {
 		return errors.New("can't purge root directory")
 	}
-	err = f.dirCache.FindRoot(ctx, false)
+	rootIDs, err := f.dirCache.RootID(ctx, false)
 	if err != nil {
 		return err
 	}
-
-	rootID := atoi(f.dirCache.RootID())
+	rootID := atoi(rootIDs)
 	// Let putio delete the filesystem tree
 	err = f.pacer.Call(func() (bool, error) {
 		// fs.Debugf(f, "deleting file: %d", rootID)
@@ -559,7 +548,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 	if !ok {
 		return nil, fs.ErrorCantCopy
 	}
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -598,7 +587,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
 	if !ok {
 		return nil, fs.ErrorCantMove
 	}
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return nil, err
 	}
@@ -636,57 +625,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	if !ok {
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err = srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, dstDirectoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
@@ -695,7 +635,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		params := url.Values{}
 		params.Set("file_id", srcID)
 		params.Set("parent_id", dstDirectoryID)
-		params.Set("name", f.opt.Enc.FromStandardName(leaf))
+		params.Set("name", f.opt.Enc.FromStandardName(dstLeaf))
 		req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
 		if err != nil {
 			return false, err

backend/putio/object.go

@@ -125,7 +125,7 @@ func (o *Object) setMetadataFromEntry(info putio.File) error {
 // Reads the entry for a file from putio
 func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
 	// defer log.Trace(o, "")("f=%+v, err=%v", f, &err)
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -248,7 +248,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		_ = resp.Body.Close()
 		return nil, fserrors.NoRetryError(err)
 	}
-	return resp.Body, err
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body, nil
 }
 
 // Update the already existing object

backend/putio/putio.go

@@ -60,7 +60,10 @@ func init() {
 		Description: "Put.io",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.ConfigNoOffline("putio", name, m, putioConfig)
+			opt := oauthutil.Options{
+				NoOffline: true,
+			}
+			err := oauthutil.Config("putio", name, m, putioConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}

backend/qingstor/qingstor.go

@@ -56,7 +56,7 @@ func init() {
 			Help: "QingStor Secret Access Key (password)\nLeave blank for anonymous access or runtime credentials.",
 		}, {
 			Name: "endpoint",
-			Help: "Enter a endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
+			Help: "Enter an endpoint URL to connection QingStor API.\nLeave blank will use the default value \"https://qingstor.com:443\"",
 		}, {
 			Name: "zone",
 			Help: "Zone to connect to.\nDefault is \"pek3a\".",
@@ -203,10 +203,10 @@ func (o *Object) split() (bucket, bucketPath string) {
 	return o.fs.split(o.remote)
 }
 
-// Split an URL into three parts: protocol host and port
+// Split a URL into three parts: protocol host and port
 func qsParseEndpoint(endpoint string) (protocol, host, port string, err error) {
 	/*
-	  Pattern to match a endpoint,
+	  Pattern to match an endpoint,
 	  eg: "http(s)://qingstor.com:443" --> "http(s)", "qingstor.com", 443
 	      "http(s)//qingstor.com"      --> "http(s)", "qingstor.com", ""
 	      "qingstor.com"               --> "", "qingstor.com", ""

backend/qingstor/upload.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/lib/atexit"
 	qs "github.com/yunify/qingstor-sdk-go/v3/service"
 )
 
@@ -22,7 +23,7 @@ const (
 	// maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
 	// maxMultiPartSize = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
 	minMultiPartSize = 1024 * 1024 * 4 // The minimum allowed part size when uploading a part to QingStor
-	maxMultiParts    = 10000           // The maximum allowed number of parts in an multi-part upload
+	maxMultiParts    = 10000           // The maximum allowed number of parts in a multi-part upload
 )
 
 const (
@@ -168,7 +169,7 @@ func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
 	return err
 }
 
-// Upload upload a object into QingStor
+// Upload upload an object into QingStor
 func (u *uploader) upload() error {
 	u.init()
 
@@ -255,7 +256,7 @@ func (mu *multiUploader) readChunk(ch chan chunk) {
 	}
 }
 
-// initiate init an Multiple Object and obtain UploadID
+// initiate init a Multiple Object and obtain UploadID
 func (mu *multiUploader) initiate() error {
 	bucketInit, _ := mu.bucketInit()
 	req := qs.InitiateMultipartUploadInput{
@@ -297,7 +298,7 @@ func (mu *multiUploader) send(c chunk) error {
 	return err
 }
 
-// complete complete an multipart upload
+// complete complete a multipart upload
 func (mu *multiUploader) complete() error {
 	var err error
 	if err = mu.getErr(); err != nil {
@@ -324,7 +325,7 @@ func (mu *multiUploader) complete() error {
 	return err
 }
 
-// abort abort an multipart upload
+// abort abort a multipart upload
 func (mu *multiUploader) abort() error {
 	var err error
 	bucketInit, _ := mu.bucketInit()
@@ -342,16 +343,19 @@ func (mu *multiUploader) abort() error {
 
 // multiPartUpload upload a multiple object into QingStor
 func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
-	// Initiate an multi-part upload
+	// Initiate a multi-part upload
 	if err = mu.initiate(); err != nil {
 		return err
 	}
-	defer func() {
-		// Abort the transfer if returning an error
-		if err != nil {
-			_ = mu.abort()
+
+	// Cancel the session if something went wrong
+	defer atexit.OnError(&err, func() {
+		fs.Debugf(mu, "Cancelling multipart upload: %v", err)
+		cancelErr := mu.abort()
+		if cancelErr != nil {
+			fs.Logf(mu, "Failed to cancel multipart upload: %v", cancelErr)
 		}
-	}()
+	})()
 
 	ch := make(chan chunk, mu.cfg.concurrency)
 	for i := 0; i < mu.cfg.concurrency; i++ {

backend/s3/s3.go

@@ -40,6 +40,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
 	"github.com/aws/aws-sdk-go/aws/request"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -52,13 +53,16 @@ import (
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
 	"github.com/rclone/rclone/fs/hash"
+	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
+	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/bucket"
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/pool"
 	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
+	"github.com/rclone/rclone/lib/structs"
 	"golang.org/x/sync/errgroup"
 )
 
@@ -68,6 +72,7 @@ func init() {
 		Name:        "s3",
 		Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
 		NewFs:       NewFs,
+		CommandHelp: commandHelp,
 		Options: []fs.Option{{
 			Name: fs.ConfigProvider,
 			Help: "Choose your S3 provider.",
@@ -95,6 +100,9 @@ func init() {
 			}, {
 				Value: "Netease",
 				Help:  "Netease Object Storage (NOS)",
+			}, {
+				Value: "Scaleway",
+				Help:  "Scaleway Object Storage",
 			}, {
 				Value: "StackPath",
 				Help:  "StackPath Object Storage",
@@ -175,10 +183,21 @@ func init() {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
 			}},
+		}, {
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "Scaleway",
+			Examples: []fs.OptionExample{{
+				Value: "nl-ams",
+				Help:  "Amsterdam, The Netherlands",
+			}, {
+				Value: "fr-par",
+				Help:  "Paris, France",
+			}},
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS,Alibaba",
+			Provider: "!AWS,Alibaba,Scaleway",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -360,6 +379,17 @@ func init() {
 				Value: "oss-me-east-1.aliyuncs.com",
 				Help:  "Middle East 1 (Dubai)",
 			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for Scaleway Object Storage.",
+			Provider: "Scaleway",
+			Examples: []fs.OptionExample{{
+				Value: "s3.nl-ams.scw.cloud",
+				Help:  "Amsterdam Endpoint",
+			}, {
+				Value: "s3.fr-par.scw.cloud",
+				Help:  "Paris Endpoint",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for StackPath Object Storage.",
@@ -377,7 +407,7 @@ func init() {
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS,Alibaba,StackPath",
+			Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-east-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -564,7 +594,7 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS,Alibaba,StackPath",
+			Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -677,7 +707,7 @@ isn't set then "acl" is used instead.`,
 			}},
 		}, {
 			Name:     "sse_customer_key",
-			Help:     "If using SSE-C you must provide the secret encyption key used to encrypt/decrypt your data.",
+			Help:     "If using SSE-C you must provide the secret encryption key used to encrypt/decrypt your data.",
 			Provider: "AWS,Ceph,Minio",
 			Advanced: true,
 			Examples: []fs.OptionExample{{
@@ -740,6 +770,21 @@ isn't set then "acl" is used instead.`,
 				Value: "STANDARD_IA",
 				Help:  "Infrequent access storage mode.",
 			}},
+		}, {
+			// Mapping from here: https://www.scaleway.com/en/docs/object-storage-glacier/#-Scaleway-Storage-Classes
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing new objects in S3.",
+			Provider: "Scaleway",
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "Default",
+			}, {
+				Value: "STANDARD",
+				Help:  "The Standard class for any upload; suitable for on-demand content like streaming or CDN.",
+			}, {
+				Value: "GLACIER",
+				Help:  "Archived storage; prices are lower, but it needs to be restored first to be accessed.",
+			}},
 		}, {
 			Name: "upload_cutoff",
 			Help: `Cutoff for switching to chunked upload
@@ -773,6 +818,21 @@ file you can stream upload is 48GB.  If you wish to stream upload
 larger files then you will need to increase chunk_size.`,
 			Default:  minChunkSize,
 			Advanced: true,
+		}, {
+			Name: "max_upload_parts",
+			Help: `Maximum number of parts in a multipart upload.
+
+This option defines the maximum number of multipart chunks to use
+when doing a multipart upload.
+
+This can be useful if a service does not support the AWS S3
+specification of 10,000 chunks.
+
+Rclone will automatically increase the chunk size when uploading a
+large file of a known size to stay below this number of chunks limit.
+`,
+			Default:  maxUploadParts,
+			Advanced: true,
 		}, {
 			Name: "copy_cutoff",
 			Help: `Cutoff for switching to multipart copy
@@ -909,6 +969,7 @@ const (
 
 	memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
 	memoryPoolUseMmap   = false
+	maxExpireDuration   = fs.Duration(7 * 24 * time.Hour) // max expiry is 1 week
 )
 
 // Options defines the configuration for this backend
@@ -931,6 +992,7 @@ type Options struct {
 	UploadCutoff          fs.SizeSuffix        `config:"upload_cutoff"`
 	CopyCutoff            fs.SizeSuffix        `config:"copy_cutoff"`
 	ChunkSize             fs.SizeSuffix        `config:"chunk_size"`
+	MaxUploadParts        int64                `config:"max_upload_parts"`
 	DisableChecksum       bool                 `config:"disable_checksum"`
 	SessionToken          string               `config:"session_token"`
 	UploadConcurrency     int                  `config:"upload_concurrency"`
@@ -1132,15 +1194,20 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
-	if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
+	if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.UseAccelerateEndpoint {
 		opt.ForcePathStyle = false
 	}
+	if opt.Provider == "Scaleway" && opt.MaxUploadParts > 1000 {
+		opt.MaxUploadParts = 1000
+	}
 	awsConfig := aws.NewConfig().
 		WithMaxRetries(0). // Rely on rclone's retry logic
 		WithCredentials(cred).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle).
-		WithS3UseAccelerate(opt.UseAccelerateEndpoint)
+		WithS3UseAccelerate(opt.UseAccelerateEndpoint).
+		WithS3UsEast1RegionalEndpoint(endpoints.RegionalS3UsEast1Endpoint)
+
 	if opt.Region != "" {
 		awsConfig.WithRegion(opt.Region)
 	}
@@ -1839,21 +1906,19 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
 	}
 	uid := cout.UploadId
 
-	defer func() {
-		if err != nil {
-			// We can try to abort the upload, but ignore the error.
-			fs.Debugf(nil, "Cancelling multipart copy")
-			_ = f.pacer.Call(func() (bool, error) {
-				_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
-					Bucket:       &dstBucket,
-					Key:          &dstPath,
-					UploadId:     uid,
-					RequestPayer: req.RequestPayer,
-				})
-				return f.shouldRetry(err)
+	defer atexit.OnError(&err, func() {
+		// Try to abort the upload, but ignore the error.
+		fs.Debugf(nil, "Cancelling multipart copy")
+		_ = f.pacer.Call(func() (bool, error) {
+			_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
+				Bucket:       &dstBucket,
+				Key:          &dstPath,
+				UploadId:     uid,
+				RequestPayer: req.RequestPayer,
 			})
-		}
-	}()
+			return f.shouldRetry(err)
+		})
+	})()
 
 	partSize := int64(f.opt.CopyCutoff)
 	numParts := (srcSize-1)/partSize + 1
@@ -1959,6 +2024,147 @@ func (f *Fs) getMemoryPool(size int64) *pool.Pool {
 	)
 }
 
+// PublicLink generates a public link to the remote path (usually readable by anyone)
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
+	if strings.HasSuffix(remote, "/") {
+		return "", fs.ErrorCantShareDirectories
+	}
+	if _, err := f.NewObject(ctx, remote); err != nil {
+		return "", err
+	}
+	if expire > maxExpireDuration {
+		fs.Logf(f, "Public Link: Reducing expiry to %v as %v is greater than the max time allowed", maxExpireDuration, expire)
+		expire = maxExpireDuration
+	}
+	bucket, bucketPath := f.split(remote)
+	httpReq, _ := f.c.GetObjectRequest(&s3.GetObjectInput{
+		Bucket: &bucket,
+		Key:    &bucketPath,
+	})
+
+	return httpReq.Presign(time.Duration(expire))
+}
+
+var commandHelp = []fs.CommandHelp{{
+	Name:  "restore",
+	Short: "Restore objects from GLACIER to normal storage",
+	Long: `This command can be used to restore one or more objects from GLACIER
+to normal storage.
+
+Usage Examples:
+
+    rclone backend restore s3:bucket/path/to/object [-o priority=PRIORITY] [-o lifetime=DAYS]
+    rclone backend restore s3:bucket/path/to/directory [-o priority=PRIORITY] [-o lifetime=DAYS]
+    rclone backend restore s3:bucket [-o priority=PRIORITY] [-o lifetime=DAYS]
+
+This flag also obeys the filters. Test first with -i/--interactive or --dry-run flags
+
+    rclone -i backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+
+All the objects shown will be marked for restore, then
+
+    rclone backend restore --include "*.txt" s3:bucket/path -o priority=Standard
+
+It returns a list of status dictionaries with Remote and Status
+keys. The Status will be OK if it was successfull or an error message
+if not.
+
+    [
+        {
+            "Status": "OK",
+            "Path": "test.txt"
+        },
+        {
+            "Status": "OK",
+            "Path": "test/file4.txt"
+        }
+    ]
+
+`,
+	Opts: map[string]string{
+		"priority":    "Priority of restore: Standard|Expedited|Bulk",
+		"lifetime":    "Lifetime of the active copy in days",
+		"description": "The optional description for the job.",
+	},
+}}
+
+// Command the backend to run a named command
+//
+// The command run is name
+// args may be used to read arguments from
+// opts may be used to read optional arguments from
+//
+// The result should be capable of being JSON encoded
+// If it is a string or a []string it will be shown to the user
+// otherwise it will be JSON encoded and shown to the user like that
+func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
+	switch name {
+	case "restore":
+		req := s3.RestoreObjectInput{
+			//Bucket:         &f.rootBucket,
+			//Key:            &encodedDirectory,
+			RestoreRequest: &s3.RestoreRequest{},
+		}
+		if lifetime := opt["lifetime"]; lifetime != "" {
+			ilifetime, err := strconv.ParseInt(lifetime, 10, 64)
+			if err != nil {
+				return nil, errors.Wrap(err, "bad lifetime")
+			}
+			req.RestoreRequest.Days = &ilifetime
+		}
+		if priority := opt["priority"]; priority != "" {
+			req.RestoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{
+				Tier: &priority,
+			}
+		}
+		if description := opt["description"]; description != "" {
+			req.RestoreRequest.Description = &description
+		}
+		type status struct {
+			Status string
+			Remote string
+		}
+		var (
+			outMu sync.Mutex
+			out   = []status{}
+		)
+		err = operations.ListFn(ctx, f, func(obj fs.Object) {
+			// Remember this is run --checkers times concurrently
+			o, ok := obj.(*Object)
+			st := status{Status: "OK", Remote: obj.Remote()}
+			defer func() {
+				outMu.Lock()
+				out = append(out, st)
+				outMu.Unlock()
+			}()
+			if operations.SkipDestructive(ctx, obj, "restore") {
+				return
+			}
+			if !ok {
+				st.Status = "Not an S3 object"
+				return
+			}
+			bucket, bucketPath := o.split()
+			reqCopy := req
+			reqCopy.Bucket = &bucket
+			reqCopy.Key = &bucketPath
+			err = f.pacer.Call(func() (bool, error) {
+				_, err = f.c.RestoreObject(&reqCopy)
+				return f.shouldRetry(err)
+			})
+			if err != nil {
+				st.Status = err.Error()
+			}
+		})
+		if err != nil {
+			return out, err
+		}
+		return out, nil
+	default:
+		return nil, fs.ErrorCommandNotFound
+	}
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -2033,11 +2239,17 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
 			if awsErr.StatusCode() == http.StatusNotFound {
+				// NotFound indicates bucket was OK
+				// NoSuchBucket would be returned if bucket was bad
+				if awsErr.Code() == "NotFound" {
+					o.fs.cache.MarkOK(bucket)
+				}
 				return fs.ErrorObjectNotFound
 			}
 		}
 		return err
 	}
+	o.fs.cache.MarkOK(bucket)
 	var size int64
 	// Ignore missing Content-Length assuming it is 0
 	// Some versions of ceph do this due their apache proxies
@@ -2177,6 +2389,13 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	tokens := pacer.NewTokenDispenser(concurrency)
 
+	uploadParts := f.opt.MaxUploadParts
+	if uploadParts < 1 {
+		uploadParts = 1
+	} else if uploadParts > maxUploadParts {
+		uploadParts = maxUploadParts
+	}
+
 	// calculate size of parts
 	partSize := int(f.opt.ChunkSize)
 
@@ -2186,31 +2405,24 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	if size == -1 {
 		warnStreamUpload.Do(func() {
 			fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
-				f.opt.ChunkSize, fs.SizeSuffix(partSize*maxUploadParts))
+				f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
 		})
 	} else {
 		// Adjust partSize until the number of parts is small enough.
-		if size/int64(partSize) >= maxUploadParts {
+		if size/int64(partSize) >= uploadParts {
 			// Calculate partition size rounded up to the nearest MB
-			partSize = int((((size / maxUploadParts) >> 20) + 1) << 20)
+			partSize = int((((size / uploadParts) >> 20) + 1) << 20)
 		}
 	}
 
 	memPool := f.getMemoryPool(int64(partSize))
 
+	var mReq s3.CreateMultipartUploadInput
+	structs.SetFrom(&mReq, req)
 	var cout *s3.CreateMultipartUploadOutput
 	err = f.pacer.Call(func() (bool, error) {
 		var err error
-		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &s3.CreateMultipartUploadInput{
-			Bucket:               req.Bucket,
-			ACL:                  req.ACL,
-			Key:                  req.Key,
-			ContentType:          req.ContentType,
-			Metadata:             req.Metadata,
-			ServerSideEncryption: req.ServerSideEncryption,
-			SSEKMSKeyId:          req.SSEKMSKeyId,
-			StorageClass:         req.StorageClass,
-		})
+		cout, err = f.c.CreateMultipartUploadWithContext(ctx, &mReq)
 		return f.shouldRetry(err)
 	})
 	if err != nil {
@@ -2218,27 +2430,24 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
 	}
 	uid := cout.UploadId
 
-	defer func() {
+	defer atexit.OnError(&err, func() {
 		if o.fs.opt.LeavePartsOnError {
 			return
 		}
-		if err != nil {
-			// We can try to abort the upload, but ignore the error.
-			fs.Debugf(o, "Cancelling multipart upload")
-			errCancel := f.pacer.Call(func() (bool, error) {
-				_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
-					Bucket:       req.Bucket,
-					Key:          req.Key,
-					UploadId:     uid,
-					RequestPayer: req.RequestPayer,
-				})
-				return f.shouldRetry(err)
+		fs.Debugf(o, "Cancelling multipart upload")
+		errCancel := f.pacer.Call(func() (bool, error) {
+			_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
+				Bucket:       req.Bucket,
+				Key:          req.Key,
+				UploadId:     uid,
+				RequestPayer: req.RequestPayer,
 			})
-			if errCancel != nil {
-				fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
-			}
+			return f.shouldRetry(err)
+		})
+		if errCancel != nil {
+			fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
 		}
-	}()
+	})()
 
 	var (
 		g, gCtx  = errgroup.WithContext(ctx)
@@ -2423,6 +2632,35 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	if o.fs.opt.StorageClass != "" {
 		req.StorageClass = &o.fs.opt.StorageClass
 	}
+	// Apply upload options
+	for _, option := range options {
+		key, value := option.Header()
+		lowerKey := strings.ToLower(key)
+		switch lowerKey {
+		case "":
+			// ignore
+		case "cache-control":
+			req.CacheControl = aws.String(value)
+		case "content-disposition":
+			req.ContentDisposition = aws.String(value)
+		case "content-encoding":
+			req.ContentEncoding = aws.String(value)
+		case "content-language":
+			req.ContentLanguage = aws.String(value)
+		case "content-type":
+			req.ContentType = aws.String(value)
+		case "x-amz-tagging":
+			req.Tagging = aws.String(value)
+		default:
+			const amzMetaPrefix = "x-amz-meta-"
+			if strings.HasPrefix(lowerKey, amzMetaPrefix) {
+				metaKey := lowerKey[len(amzMetaPrefix):]
+				req.Metadata[metaKey] = aws.String(value)
+			} else {
+				fs.Errorf(o, "Don't know how to set key %q on upload", key)
+			}
+		}
+	}
 
 	if multipart {
 		err = o.uploadMultipart(ctx, &req, size, in)
@@ -2463,18 +2701,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		httpReq.Header = headers
 		httpReq.ContentLength = size
 
-		for _, option := range options {
-			switch option.(type) {
-			case *fs.HTTPOption:
-				key, value := option.Header()
-				httpReq.Header.Add(key, value)
-			default:
-				if option.Mandatory() {
-					fs.Logf(o, "Unsupported mandatory option: %v", option)
-				}
-			}
-		}
-
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
 			resp, err := o.fs.srv.Do(httpReq)
 			if err != nil {
@@ -2556,6 +2782,7 @@ var (
 	_ fs.Copier      = &Fs{}
 	_ fs.PutStreamer = &Fs{}
 	_ fs.ListRer     = &Fs{}
+	_ fs.Commander   = &Fs{}
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
 	_ fs.GetTierer   = &Object{}

backend/seafile/object.go

@@ -85,7 +85,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/seafile/seafile.go

@@ -32,8 +32,16 @@ import (
 )
 
 const (
-	librariesCacheKey = "all"
-	retryAfterHeader  = "Retry-After"
+	librariesCacheKey   = "all"
+	retryAfterHeader    = "Retry-After"
+	configURL           = "url"
+	configUser          = "user"
+	configPassword      = "pass"
+	config2FA           = "2fa"
+	configLibrary       = "library"
+	configLibraryKey    = "library_key"
+	configCreateLibrary = "create_library"
+	configAuthToken     = "auth_token"
 )
 
 // This is global to all instances of fs
@@ -49,8 +57,9 @@ func init() {
 		Name:        "seafile",
 		Description: "seafile",
 		NewFs:       NewFs,
+		Config:      Config,
 		Options: []fs.Option{{
-			Name:     "url",
+			Name:     configURL,
 			Help:     "URL of seafile host to connect to",
 			Required: true,
 			Examples: []fs.OptionExample{{
@@ -58,26 +67,35 @@ func init() {
 				Help:  "Connect to cloud.seafile.com",
 			}},
 		}, {
-			Name:     "user",
-			Help:     "User name",
+			Name:     configUser,
+			Help:     "User name (usually email address)",
 			Required: true,
 		}, {
-			Name:       "pass",
+			// Password is not required, it will be left blank for 2FA
+			Name:       configPassword,
 			Help:       "Password",
 			IsPassword: true,
-			Required:   true,
 		}, {
-			Name: "library",
+			Name:    config2FA,
+			Help:    "Two-factor authentication ('true' if the account has 2FA enabled)",
+			Default: false,
+		}, {
+			Name: configLibrary,
 			Help: "Name of the library. Leave blank to access all non-encrypted libraries.",
 		}, {
-			Name:       "library_key",
+			Name:       configLibraryKey,
 			Help:       "Library password (for encrypted libraries only). Leave blank if you pass it through the command line.",
 			IsPassword: true,
 		}, {
-			Name:     "create_library",
-			Help:     "Should create library if it doesn't exist",
+			Name:     configCreateLibrary,
+			Help:     "Should rclone create a library if it doesn't exist",
 			Advanced: true,
 			Default:  false,
+		}, {
+			// Keep the authentication token after entering the 2FA code
+			Name: configAuthToken,
+			Help: "Authentication token",
+			Hide: fs.OptionHideBoth,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -97,6 +115,8 @@ type Options struct {
 	URL           string               `config:"url"`
 	User          string               `config:"user"`
 	Password      string               `config:"pass"`
+	Is2FA         bool                 `config:"2fa"`
+	AuthToken     string               `config:"auth_token"`
 	LibraryName   string               `config:"library"`
 	LibraryKey    string               `config:"library_key"`
 	CreateLibrary bool                 `config:"create_library"`
@@ -192,7 +212,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 	fs.Debugf(nil, "Seafile server version %s", serverInfo.Version)
 
-	// We don't support bellow seafile v6.0 (version 6.0 is already more than 3 years old)
+	// We don't support lower than seafile v6.0 (version 6.0 is already more than 3 years old)
 	serverVersion := semver.New(serverInfo.Version)
 	if serverVersion.Major < 6 {
 		return nil, errors.New("unsupported Seafile server (version < 6.0)")
@@ -205,10 +225,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		f.moveDirNotAvailable = true
 	}
 
-	err = f.authorizeAccount(ctx)
-	if err != nil {
-		return nil, err
+	// Take the authentication token from the configuration first
+	token := f.opt.AuthToken
+	if token == "" {
+		// If not available, send the user/password instead
+		token, err = f.authorizeAccount(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
+	f.setAuthorizationToken(token)
 
 	if f.libraryName != "" {
 		// Check if the library exists
@@ -270,26 +296,108 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
+// Config callback for 2FA
+func Config(name string, m configmap.Mapper) {
+	serverURL, ok := m.Get(configURL)
+	if !ok || serverURL == "" {
+		// If there's no server URL, it means we're trying an operation at the backend level, like a "rclone authorize seafile"
+		fmt.Print("\nOperation not supported on this remote.\nIf you need a 2FA code on your account, use the command:\n\nrclone config reconnect <remote name>:\n\n")
+		return
+	}
+
+	// Stop if we are running non-interactive config
+	if fs.Config.AutoConfirm {
+		return
+	}
+
+	u, err := url.Parse(serverURL)
+	if err != nil {
+		fs.Errorf(nil, "Invalid server URL %s", serverURL)
+		return
+	}
+
+	is2faEnabled, _ := m.Get(config2FA)
+	if is2faEnabled != "true" {
+		fmt.Println("Two-factor authentication is not enabled on this account.")
+		return
+	}
+
+	username, _ := m.Get(configUser)
+	if username == "" {
+		fs.Errorf(nil, "A username is required")
+		return
+	}
+
+	password, _ := m.Get(configPassword)
+	if password != "" {
+		password, _ = obscure.Reveal(password)
+	}
+	// Just make sure we do have a password
+	for password == "" {
+		fmt.Print("Two-factor authentication: please enter your password (it won't be saved in the configuration)\npassword> ")
+		password = config.ReadPassword()
+	}
+
+	// Create rest client for getAuthorizationToken
+	url := u.String()
+	if !strings.HasPrefix(url, "/") {
+		url += "/"
+	}
+	srv := rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(url)
+
+	// We loop asking for a 2FA code
+	for {
+		code := ""
+		for code == "" {
+			fmt.Print("Two-factor authentication: please enter your 2FA code\n2fa code> ")
+			code = config.ReadLine()
+		}
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+
+		fmt.Println("Authenticating...")
+		token, err := getAuthorizationToken(ctx, srv, username, password, code)
+		if err != nil {
+			fmt.Printf("Authentication failed: %v\n", err)
+			tryAgain := strings.ToLower(config.ReadNonEmptyLine("Do you want to try again (y/n)?"))
+			if tryAgain != "y" && tryAgain != "yes" {
+				// The user is giving up, we're done here
+				break
+			}
+		}
+		if token != "" {
+			fmt.Println("Success!")
+			// Let's save the token into the configuration
+			m.Set(configAuthToken, token)
+			// And delete any previous entry for password
+			m.Set(configPassword, "")
+			config.SaveConfig()
+			// And we're done here
+			break
+		}
+	}
+}
+
 // sets the AuthorizationToken up
 func (f *Fs) setAuthorizationToken(token string) {
 	f.srv.SetHeader("Authorization", "Token "+token)
 }
 
 // authorizeAccount gets the auth token.
-func (f *Fs) authorizeAccount(ctx context.Context) error {
+func (f *Fs) authorizeAccount(ctx context.Context) (string, error) {
 	f.authMu.Lock()
 	defer f.authMu.Unlock()
+
 	token, err := f.getAuthorizationToken(ctx)
 	if err != nil {
-		return err
+		return "", err
 	}
-	f.setAuthorizationToken(token)
-	return nil
+	return token, nil
 }
 
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
-	401, // Unauthorized (eg "Token has expired")
 	408, // Request Timeout
 	429, // Rate exceeded.
 	500, // Get occasional 500 Internal Server Error
@@ -298,9 +406,9 @@ var retryErrorCodes = []int{
 	520, // Operation failed (We get them sometimes when running tests in parallel)
 }
 
-// shouldRetryNoAuth returns a boolean as to whether this resp and err
+// shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
+func (f *Fs) shouldRetry(resp *http.Response, err error) (bool, error) {
 	// For 429 errors look at the Retry-After: header and
 	// set the retry appropriately, starting with a minimum of 1
 	// second if it isn't set.
@@ -319,22 +427,6 @@ func (f *Fs) shouldRetryNoReauth(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// shouldRetry returns a boolean as to whether this resp and err
-// deserve to be retried.  It returns the err as a convenience
-func (f *Fs) shouldRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
-	// It looks like seafile is using the 403 error code instead of the standard 401.
-	if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 403) {
-		fs.Debugf(f, "Unauthorized: %v", err)
-		// Reauth
-		authErr := f.authorizeAccount(ctx)
-		if authErr != nil {
-			err = authErr
-		}
-		return true, err
-	}
-	return f.shouldRetryNoReauth(resp, err)
-}
-
 func (f *Fs) shouldRetryUpload(ctx context.Context, resp *http.Response, err error) (bool, error) {
 	if err != nil || (resp != nil && resp.StatusCode > 400) {
 		return true, err
@@ -436,7 +528,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 
 // Put in to the remote path with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Put should either
 // return an error or upload it properly (rather than e.g. calling panic).
 //
@@ -880,7 +972,7 @@ func (f *Fs) UserInfo(ctx context.Context) (map[string]string, error) {
 // ==================== Optional Interface fs.PublicLinker ====================
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	libraryName, filePath := f.splitPath(remote)
 	if libraryName == "" {
 		// We cannot share the whole seafile server, we need at least a library

backend/seafile/webapi.go

@@ -32,37 +32,44 @@ var (
 // ==================== Seafile API ====================
 
 func (f *Fs) getAuthorizationToken(ctx context.Context) (string, error) {
-	// API Socumentation
+	return getAuthorizationToken(ctx, f.srv, f.opt.User, f.opt.Password, "")
+}
+
+// getAuthorizationToken can be called outside of an fs (during configuration of the remote to get the authentication token)
+// it's doing a single call (no pacer involved)
+func getAuthorizationToken(ctx context.Context, srv *rest.Client, user, password, oneTimeCode string) (string, error) {
+	// API Documentation
 	// https://download.seafile.com/published/web-api/home.md#user-content-Quick%20Start
 	opts := rest.Opts{
 		Method:       "POST",
 		Path:         "api2/auth-token/",
 		ExtraHeaders: map[string]string{"Authorization": ""}, // unset the Authorization for this request
+		IgnoreStatus: true,                                   // so we can load the error messages back into result
+	}
+
+	// 2FA
+	if oneTimeCode != "" {
+		opts.ExtraHeaders["X-SEAFILE-OTP"] = oneTimeCode
 	}
 
 	request := api.AuthenticationRequest{
-		Username: f.opt.User,
-		Password: f.opt.Password,
+		Username: user,
+		Password: password,
 	}
 	result := api.AuthenticationResult{}
 
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetryNoReauth(resp, err)
-	})
+	_, err := srv.CallJSON(ctx, &opts, &request, &result)
 	if err != nil {
-		if resp != nil {
-			if resp.StatusCode == 403 {
-				return "", fs.ErrorPermissionDenied
-			}
-		}
+		// This is only going to be http errors here
 		return "", errors.Wrap(err, "failed to authenticate")
 	}
-	if result.Errors != nil && len(result.Errors) > 1 {
+	if result.Errors != nil && len(result.Errors) > 0 {
 		return "", errors.New(strings.Join(result.Errors, ", "))
 	}
+	if result.Token == "" {
+		// No error in "non_field_errors" field but still empty token
+		return "", errors.New("failed to authenticate")
+	}
 	return result.Token, nil
 }
 
@@ -79,11 +86,11 @@ func (f *Fs) getServerInfo(ctx context.Context) (account *api.ServerInfo, err er
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 		}
@@ -105,11 +112,11 @@ func (f *Fs) getUserAccountInfo(ctx context.Context) (account *api.AccountInfo,
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 		}
@@ -132,11 +139,11 @@ func (f *Fs) getLibraries(ctx context.Context) ([]api.Library, error) {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 		}
@@ -163,11 +170,11 @@ func (f *Fs) createLibrary(ctx context.Context, libraryName, password string) (l
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 		}
@@ -190,11 +197,11 @@ func (f *Fs) deleteLibrary(ctx context.Context, libraryID string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 		}
@@ -221,7 +228,7 @@ func (f *Fs) decryptLibrary(ctx context.Context, libraryID, password string) err
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -264,10 +271,13 @@ func (f *Fs) getDirectoryEntriesAPIv21(ctx context.Context, libraryID, dirPath s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
+				return nil, fs.ErrorPermissionDenied
+			}
 			if resp.StatusCode == 404 {
 				return nil, fs.ErrorDirNotFound
 			}
@@ -306,11 +316,11 @@ func (f *Fs) getDirectoryDetails(ctx context.Context, libraryID, dirPath string)
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -348,11 +358,11 @@ func (f *Fs) createDir(ctx context.Context, libraryID, dirPath string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 		}
@@ -388,11 +398,11 @@ func (f *Fs) renameDir(ctx context.Context, libraryID, dirPath, newName string)
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 		}
@@ -428,11 +438,11 @@ func (f *Fs) moveDir(ctx context.Context, srcLibraryID, srcDir, srcName, dstLibr
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, nil)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -464,11 +474,11 @@ func (f *Fs) deleteDir(ctx context.Context, libraryID, filePath string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 		}
@@ -495,14 +505,14 @@ func (f *Fs) getFileDetails(ctx context.Context, libraryID, filePath string) (*a
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
 			if resp.StatusCode == 404 {
 				return nil, fs.ErrorObjectNotFound
 			}
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 		}
@@ -529,7 +539,7 @@ func (f *Fs) deleteFile(ctx context.Context, libraryID, filePath string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		return errors.Wrap(err, "failed to delete file")
@@ -555,7 +565,7 @@ func (f *Fs) getDownloadLink(ctx context.Context, libraryID, filePath string) (s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -604,7 +614,7 @@ func (f *Fs) download(ctx context.Context, url string, size int64, options ...fs
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -649,11 +659,11 @@ func (f *Fs) getUploadLink(ctx context.Context, libraryID string) (string, error
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return "", fs.ErrorPermissionDenied
 			}
 		}
@@ -693,7 +703,7 @@ func (f *Fs) upload(ctx context.Context, in io.Reader, uploadLink, filePath stri
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 500 {
@@ -729,11 +739,11 @@ func (f *Fs) listShareLinks(ctx context.Context, libraryID, remote string) ([]ap
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -767,11 +777,11 @@ func (f *Fs) createShareLink(ctx context.Context, libraryID, remote string) (*ap
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -808,11 +818,11 @@ func (f *Fs) copyFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -850,11 +860,11 @@ func (f *Fs) moveFile(ctx context.Context, srcLibraryID, srcPath, dstLibraryID,
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -890,11 +900,11 @@ func (f *Fs) renameFile(ctx context.Context, libraryID, filePath, newname string
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, &request, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -928,11 +938,11 @@ func (f *Fs) emptyLibraryTrash(ctx context.Context, libraryID string) error {
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, nil)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {
@@ -966,10 +976,13 @@ func (f *Fs) getDirectoryEntriesAPIv2(ctx context.Context, libraryID, dirPath st
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
+				return nil, fs.ErrorPermissionDenied
+			}
 			if resp.StatusCode == 404 {
 				return nil, fs.ErrorDirNotFound
 			}
@@ -1017,11 +1030,11 @@ func (f *Fs) copyFileAPIv2(ctx context.Context, srcLibraryID, srcPath, dstLibrar
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return nil, fs.ErrorPermissionDenied
 			}
 		}
@@ -1045,7 +1058,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
 	// No luck with JSON input with the older api2
 	postParameters := url.Values{
 		"operation": {"rename"},
-		"reloaddir": {"true"}, // This is an undocumented trick to avoid a http code 301 response (found in https://github.com/haiwen/seahub/blob/master/seahub/api2/views.py)
+		"reloaddir": {"true"}, // This is an undocumented trick to avoid an http code 301 response (found in https://github.com/haiwen/seahub/blob/master/seahub/api2/views.py)
 		"newname":   {f.opt.Enc.FromStandardName(newname)},
 	}
 
@@ -1062,7 +1075,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.Call(ctx, &opts)
-		return f.shouldRetry(ctx, resp, err)
+		return f.shouldRetry(resp, err)
 	})
 	if err != nil {
 		if resp != nil {
@@ -1070,7 +1083,7 @@ func (f *Fs) renameFileAPIv2(ctx context.Context, libraryID, filePath, newname s
 				// This is the normal response from the server
 				return nil
 			}
-			if resp.StatusCode == 403 {
+			if resp.StatusCode == 401 || resp.StatusCode == 403 {
 				return fs.ErrorPermissionDenied
 			}
 			if resp.StatusCode == 404 {

backend/sftp/sftp.go

@@ -69,9 +69,12 @@ func init() {
 			Name:       "pass",
 			Help:       "SSH password, leave blank to use ssh-agent.",
 			IsPassword: true,
+		}, {
+			Name: "key_pem",
+			Help: "Raw PEM-encoded private key, If specified, will override key_file parameter.",
 		}, {
 			Name: "key_file",
-			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
+			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent." + env.ShellExpandHelp,
 		}, {
 			Name: "key_file_pass",
 			Help: `The passphrase to decrypt the PEM-encoded private key file.
@@ -91,7 +94,7 @@ when the ssh-agent contains many keys.`,
 			Name: "use_insecure_cipher",
 			Help: `Enable the use of insecure ciphers and key exchange methods. 
 
-This enables the use of the the following insecure ciphers and key exchange methods:
+This enables the use of the following insecure ciphers and key exchange methods:
 
 - aes128-cbc
 - aes192-cbc
@@ -172,6 +175,7 @@ type Options struct {
 	User              string `config:"user"`
 	Port              string `config:"port"`
 	Pass              string `config:"pass"`
+	KeyPem            string `config:"key_pem"`
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
 	KeyUseAgent       bool   `config:"key_use_agent"`
@@ -390,8 +394,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
-	// Add ssh agent-auth if no password or file specified
-	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword) || opt.KeyUseAgent {
+	//keyPem := env.ShellExpand(opt.KeyPem)
+	// Add ssh agent-auth if no password or file or key PEM specified
+	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword && opt.KeyPem == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -427,10 +432,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	// Load key file if specified
-	if keyFile != "" {
-		key, err := ioutil.ReadFile(keyFile)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to read private key file")
+	if keyFile != "" || opt.KeyPem != "" {
+		var key []byte
+		if opt.KeyPem == "" {
+			key, err = ioutil.ReadFile(keyFile)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to read private key file")
+			}
+		} else {
+			// wrap in quotes because the config is a coming as a literal without them.
+			opt.KeyPem, err = strconv.Unquote("\"" + opt.KeyPem + "\"")
+			if err != nil {
+				return nil, errors.Wrap(err, "pem key not formatted properly")
+			}
+			key = []byte(opt.KeyPem)
 		}
 		clearpass := ""
 		if opt.KeyFilePass != "" {
@@ -470,7 +485,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return NewFsWithConnection(ctx, name, root, m, opt, sshConfig)
 }
 
-// NewFsWithConnection creates a new Fs object from the name and root and a ssh.ClientConfig. It connects to
+// NewFsWithConnection creates a new Fs object from the name and root and an ssh.ClientConfig. It connects to
 // the host specified in the ssh.ClientConfig
 func NewFsWithConnection(ctx context.Context, name string, root string, m configmap.Mapper, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
 	f := &Fs{
@@ -1021,7 +1036,7 @@ func parseHash(bytes []byte) string {
 
 // Parses the byte array output from the SSH session
 // returned by an invocation of df into
-// the disk size, used space, and avaliable space on the disk, in that order.
+// the disk size, used space, and available space on the disk, in that order.
 // Only works when `df` has output info on only one disk
 func parseUsage(bytes []byte) (spaceTotal int64, spaceUsed int64, spaceAvail int64) {
 	spaceTotal, spaceUsed, spaceAvail = -1, -1, -1

backend/sharefile/api/types.go

@@ -102,7 +102,7 @@ type UploadSpecification struct {
 	MaxNumberOfThreads int    `json:"MaxNumberOfThreads"` // Specifies the max number of chunks that can be sent simultaneously for threaded uploads
 }
 
-// UploadFinishResponse is returnes from calling UploadSpecification.FinishURI
+// UploadFinishResponse is returns from calling UploadSpecification.FinishURI
 type UploadFinishResponse struct {
 	Error        bool   `json:"error"`
 	ErrorMessage string `json:"errorMessage"`

backend/sharefile/sharefile.go

@@ -152,7 +152,10 @@ func init() {
 				oauthConfig.Endpoint.TokenURL = endpoint + tokenPath
 				return nil
 			}
-			err := oauthutil.ConfigWithCallback("sharefile", name, m, oauthConfig, checkAuth)
+			opt := oauthutil.Options{
+				CheckAuth: checkAuth,
+			}
+			err := oauthutil.Config("sharefile", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
@@ -347,7 +350,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string, directoriesOnly b
 
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -697,10 +700,6 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -740,7 +739,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -796,13 +795,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -814,10 +807,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	rootID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1030,75 +1019,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
-	if err != nil {
-		return err
-	}
-
-	// Find ID of src parent, not creating subdirs
-	var srcLeaf, srcDirectoryID string
-	findPath = srcRemote
-	if srcRemote == "" {
-		findPath = srcFs.root
-	}
-	srcLeaf, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
+	srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	_, err = f.move(ctx, false, srcID, srcLeaf, leaf, srcDirectoryID, directoryID)
+	_, err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -1399,7 +1327,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	isLargeFile := size < 0 || size > int64(o.fs.opt.UploadCutoff)
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}

backend/sugarsync/sugarsync.go

@@ -230,7 +230,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// parsePath parses an sugarsync 'url'
+// parsePath parses a sugarsync 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
 	return
@@ -255,7 +255,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 // readMetaDataForPath reads the metadata from the path
 func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.File, err error) {
 	// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
-	leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
+	leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
 	if err != nil {
 		if err == fs.ErrorDirNotFound {
 			return nil, fs.ErrorObjectNotFound
@@ -663,10 +663,6 @@ OUTER:
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	err = f.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return nil, err
-	}
 	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
 	if err != nil {
 		return nil, err
@@ -709,7 +705,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Used to create new objects
 func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return
 	}
@@ -765,13 +761,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	err := f.dirCache.FindRoot(ctx, true)
-	if err != nil {
-		return err
-	}
-	if dir != "" {
-		_, err = f.dirCache.FindDir(ctx, dir, true)
-	}
+	_, err := f.dirCache.FindDir(ctx, dir, true)
 	return err
 }
 
@@ -807,10 +797,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return errors.New("can't purge root directory")
 	}
 	dc := f.dirCache
-	err := dc.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
 	directoryID, err := dc.FindDir(ctx, dir, false)
 	if err != nil {
 		return err
@@ -1033,64 +1019,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.root, srcRemote)
-	dstPath := path.Join(f.root, dstRemote)
 
-	// Refuse to move to or from the root
-	if srcPath == "" || dstPath == "" {
-		fs.Debugf(src, "DirMove error: Can't move root")
-		return errors.New("can't move root directory")
-	}
-
-	// find the root src directory
-	err := srcFs.dirCache.FindRoot(ctx, false)
-	if err != nil {
-		return err
-	}
-
-	// find the root dst directory
-	if dstRemote != "" {
-		err = f.dirCache.FindRoot(ctx, true)
-		if err != nil {
-			return err
-		}
-	} else {
-		if f.dirCache.FoundRoot() {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of dst parent, creating subdirs if necessary
-	var leaf, directoryID string
-	findPath := dstRemote
-	if dstRemote == "" {
-		findPath = f.root
-	}
-	leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
-	if err != nil {
-		return err
-	}
-
-	// Check destination does not exist
-	if dstRemote != "" {
-		_, err = f.dirCache.FindDir(ctx, dstRemote, false)
-		if err == fs.ErrorDirNotFound {
-			// OK
-		} else if err != nil {
-			return err
-		} else {
-			return fs.ErrorDirExists
-		}
-	}
-
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
+	srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
 	if err != nil {
 		return err
 	}
 
 	// Do the move
-	err = f.moveDir(ctx, srcID, leaf, directoryID)
+	err = f.moveDir(ctx, srcID, dstLeaf, dstDirectoryID)
 	if err != nil {
 		return err
 	}
@@ -1099,7 +1035,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
-func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
 	obj, err := f.NewObject(ctx, remote)
 	if err != nil {
 		return "", err
@@ -1289,7 +1225,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	remote := o.Remote()
 
 	// Create the directory for the object if it doesn't exist
-	leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
+	leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
 	if err != nil {
 		return err
 	}

backend/swift/swift.go

@@ -73,7 +73,7 @@ copy operations.`,
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "swift",
-		Description: "Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
+		Description: "OpenStack Swift (Rackspace Cloud Files, Memset Memstore, OVH)",
 		NewFs:       NewFs,
 		Options: append([]fs.Option{{
 			Name:    "env_auth",
@@ -284,7 +284,7 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this err deserves to be
 // retried.  It returns the err as a convenience
 func shouldRetry(err error) (bool, error) {
-	// If this is an swift.Error object extract the HTTP error code
+	// If this is a swift.Error object extract the HTTP error code
 	if swiftError, ok := err.(*swift.Error); ok {
 		for _, e := range retryErrorCodes {
 			if swiftError.StatusCode == e {
@@ -1124,6 +1124,9 @@ func (o *Object) removeSegments(except string) error {
 	// remove the segments container if empty, ignore errors
 	err = o.fs.pacer.Call(func() (bool, error) {
 		err = o.fs.c.ContainerDelete(segmentsContainer)
+		if err == swift.ContainerNotFound || err == swift.ContainerNotEmpty {
+			return false, err
+		}
 		return shouldRetry(err)
 	})
 	if err == nil {
@@ -1253,7 +1256,7 @@ func deleteChunks(o *Object, segmentsContainer string, segmentInfos []string) {
 			fs.Debugf(o, "Delete segment file %q on %q", v, segmentsContainer)
 			e := o.fs.c.ObjectDelete(segmentsContainer, v)
 			if e != nil {
-				fs.Errorf(o, "Error occured in delete segment file %q on %q , error: %q", v, segmentsContainer, e)
+				fs.Errorf(o, "Error occurred in delete segment file %q on %q, error: %q", v, segmentsContainer, e)
 			}
 		}
 	}

backend/tardigrade/fs.go

@@ -103,7 +103,7 @@ func init() {
 			},
 			{
 				Name:     "satellite_address",
-				Help:     "Satellite Address. Custom satellite address should match the format: <nodeid>@<address>:<port>.",
+				Help:     "Satellite Address. Custom satellite address should match the format: `<nodeid>@<address>:<port>`.",
 				Required: false,
 				Provider: newProvider,
 				Default:  "us-central-1.tardigrade.io",
@@ -269,7 +269,9 @@ func (f *Fs) connect(ctx context.Context) (project *uplink.Project, err error) {
 	fs.Debugf(f, "connecting...")
 	defer fs.Debugf(f, "connected: %+v", err)
 
-	cfg := uplink.Config{}
+	cfg := uplink.Config{
+		UserAgent: "rclone",
+	}
 
 	project, err = cfg.OpenProject(ctx, f.access)
 	if err != nil {
@@ -518,7 +520,7 @@ func (f *Fs) NewObject(ctx context.Context, relative string) (_ fs.Object, err e
 
 // Put in to the remote path with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Put should
 // either return an error or upload it properly (rather than e.g. calling
 // panic).
@@ -669,7 +671,7 @@ func (f *Fs) Rmdir(ctx context.Context, relative string) (err error) {
 // requirements. In particular, libuplink requires a trailing slash for
 // listings, but rclone does not always provide one. Further, depending on how
 // the path was initially path normalization may have removed it (e.g. a
-// trailing slash from the CLI is removed before it ever get's to the backend
+// trailing slash from the CLI is removed before it ever gets to the backend
 // code).
 func newPrefix(prefix string) string {
 	if prefix == "" {

backend/tardigrade/object.go

@@ -177,7 +177,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (_ io.ReadC
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {

backend/union/entry.go

@@ -55,7 +55,7 @@ func (d *Directory) candidates() []upstream.Entry {
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/union/errors.go

@@ -33,7 +33,7 @@ func (e Errors) FilterNil() Errors {
 	return ne
 }
 
-// Err returns a error interface that filtered nil,
+// Err returns an error interface that filtered nil,
 // or nil if no non-nil Error is presented.
 func (e Errors) Err() error {
 	ne := e.FilterNil()

backend/union/policy/all.go

@@ -31,7 +31,7 @@ func (p *All) Create(ctx context.Context, upstreams []*upstream.Fs, path string)
 	return upstreams, nil
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *All) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/epall.go

@@ -61,7 +61,7 @@ func (p *EpAll) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.epall(ctx, upstreams, path)
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receivng a set of candidate entries
 func (p *EpAll) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound
@@ -86,7 +86,7 @@ func (p *EpAll) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return upstreams, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpAll) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/epff.go

@@ -61,7 +61,7 @@ func (p *EpFF) Action(ctx context.Context, upstreams []*upstream.Fs, path string
 	return []*upstream.Fs{u}, err
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *EpFF) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound
@@ -86,7 +86,7 @@ func (p *EpFF) Create(ctx context.Context, upstreams []*upstream.Fs, path string
 	return []*upstream.Fs{u}, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpFF) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound
@@ -106,7 +106,7 @@ func (p *EpFF) Search(ctx context.Context, upstreams []*upstream.Fs, path string
 	return p.epff(ctx, upstreams, path)
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *EpFF) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/eplfs.go

@@ -65,7 +65,7 @@ func (p *EpLfs) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *EpLfs) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.ActionEntries(entries...)
 	if err != nil {
@@ -85,7 +85,7 @@ func (p *EpLfs) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpLfs) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.CreateEntries(entries...)
 	if err != nil {
@@ -107,7 +107,7 @@ func (p *EpLfs) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.lfs(upstreams)
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *EpLfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/eplno.go

@@ -65,7 +65,7 @@ func (p *EpLno) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *EpLno) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.ActionEntries(entries...)
 	if err != nil {
@@ -85,7 +85,7 @@ func (p *EpLno) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpLno) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.CreateEntries(entries...)
 	if err != nil {
@@ -107,7 +107,7 @@ func (p *EpLno) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.lno(upstreams)
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *EpLno) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/eplus.go

@@ -65,7 +65,7 @@ func (p *EpLus) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *EpLus) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.ActionEntries(entries...)
 	if err != nil {
@@ -85,7 +85,7 @@ func (p *EpLus) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpLus) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.CreateEntries(entries...)
 	if err != nil {
@@ -107,7 +107,7 @@ func (p *EpLus) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.lus(upstreams)
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *EpLus) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/epmfs.go

@@ -64,7 +64,7 @@ func (p *EpMfs) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *EpMfs) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.ActionEntries(entries...)
 	if err != nil {
@@ -84,7 +84,7 @@ func (p *EpMfs) Create(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return []*upstream.Fs{u}, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpMfs) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.CreateEntries(entries...)
 	if err != nil {
@@ -106,7 +106,7 @@ func (p *EpMfs) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.mfs(upstreams)
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receivng a set of candidate entries
 func (p *EpMfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/eprand.go

@@ -38,7 +38,7 @@ func (p *EpRand) Action(ctx context.Context, upstreams []*upstream.Fs, path stri
 	return []*upstream.Fs{p.rand(upstreams)}, nil
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *EpRand) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.ActionEntries(entries...)
 	if err != nil {
@@ -56,7 +56,7 @@ func (p *EpRand) Create(ctx context.Context, upstreams []*upstream.Fs, path stri
 	return []*upstream.Fs{p.rand(upstreams)}, nil
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *EpRand) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.EpAll.CreateEntries(entries...)
 	if err != nil {
@@ -77,7 +77,7 @@ func (p *EpRand) Search(ctx context.Context, upstreams []*upstream.Fs, path stri
 	return p.rand(upstreams), nil
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *EpRand) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/newest.go

@@ -15,7 +15,7 @@ func init() {
 }
 
 // Newest policy picks the file / directory with the largest mtime
-// It implies the existance of a path
+// It implies the existence of a path
 type Newest struct {
 	EpAll
 }
@@ -93,7 +93,7 @@ func (p *Newest) Action(ctx context.Context, upstreams []*upstream.Fs, path stri
 	return []*upstream.Fs{u}, err
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *Newest) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound
@@ -119,7 +119,7 @@ func (p *Newest) Create(ctx context.Context, upstreams []*upstream.Fs, path stri
 	return []*upstream.Fs{u}, err
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *Newest) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound
@@ -140,7 +140,7 @@ func (p *Newest) Search(ctx context.Context, upstreams []*upstream.Fs, path stri
 	return p.newest(ctx, upstreams, path)
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *Newest) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/policy.go

@@ -26,13 +26,13 @@ type Policy interface {
 	// Search category policy, governing the access to files and directories
 	Search(ctx context.Context, upstreams []*upstream.Fs, path string) (*upstream.Fs, error)
 
-	// ActionEntries is ACTION category policy but receving a set of candidate entries
+	// ActionEntries is ACTION category policy but receiving a set of candidate entries
 	ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error)
 
-	// CreateEntries is CREATE category policy but receving a set of candidate entries
+	// CreateEntries is CREATE category policy but receiving a set of candidate entries
 	CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error)
 
-	// SearchEntries is SEARCH category policy but receving a set of candidate entries
+	// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 	SearchEntries(entries ...upstream.Entry) (upstream.Entry, error)
 }
 

backend/union/policy/rand.go

@@ -35,7 +35,7 @@ func (p *Rand) Action(ctx context.Context, upstreams []*upstream.Fs, path string
 	return []*upstream.Fs{p.rand(upstreams)}, nil
 }
 
-// ActionEntries is ACTION category policy but receving a set of candidate entries
+// ActionEntries is ACTION category policy but receiving a set of candidate entries
 func (p *Rand) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.All.ActionEntries(entries...)
 	if err != nil {
@@ -53,7 +53,7 @@ func (p *Rand) Create(ctx context.Context, upstreams []*upstream.Fs, path string
 	return []*upstream.Fs{p.rand(upstreams)}, nil
 }
 
-// CreateEntries is CREATE category policy but receving a set of candidate entries
+// CreateEntries is CREATE category policy but receiving a set of candidate entries
 func (p *Rand) CreateEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	entries, err := p.All.CreateEntries(entries...)
 	if err != nil {
@@ -74,7 +74,7 @@ func (p *Rand) Search(ctx context.Context, upstreams []*upstream.Fs, path string
 	return p.rand(upstreams), nil
 }
 
-// SearchEntries is SEARCH category policy but receving a set of candidate entries
+// SearchEntries is SEARCH category policy but receiving a set of candidate entries
 func (p *Rand) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/union.go

@@ -80,7 +80,7 @@ type Fs struct {
 	searchPolicy policy.Policy  // policy for SEARCH
 }
 
-// Wrap candidate objects in to an union Object
+// Wrap candidate objects in to a union Object
 func (f *Fs) wrapEntries(entries ...upstream.Entry) (entry, error) {
 	e, err := f.searchEntries(entries...)
 	if err != nil {

backend/union/upstream/upstream.go

@@ -100,7 +100,7 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
 	return f, err
 }
 
-// WrapDirectory wraps a fs.Directory to include the info
+// WrapDirectory wraps an fs.Directory to include the info
 // of the upstream Fs
 func (f *Fs) WrapDirectory(e fs.Directory) *Directory {
 	if e == nil {
@@ -112,7 +112,7 @@ func (f *Fs) WrapDirectory(e fs.Directory) *Directory {
 	}
 }
 
-// WrapObject wraps a fs.Object to include the info
+// WrapObject wraps an fs.Object to include the info
 // of the upstream Fs
 func (f *Fs) WrapObject(o fs.Object) *Object {
 	if o == nil {
@@ -124,7 +124,7 @@ func (f *Fs) WrapObject(o fs.Object) *Object {
 	}
 }
 
-// WrapEntry wraps a fs.DirEntry to include the info
+// WrapEntry wraps an fs.DirEntry to include the info
 // of the upstream Fs
 func (f *Fs) WrapEntry(e fs.DirEntry) (Entry, error) {
 	switch e.(type) {
@@ -219,7 +219,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // Update in to the object with the modTime given of the given size
 //
-// When called from outside a Fs by rclone, src.Size() will always be >= 0.
+// When called from outside an Fs by rclone, src.Size() will always be >= 0.
 // But for unknown-sized objects (indicated by src.Size() == -1), Upload should either
 // return an error or update the object properly (rather than e.g. calling panic).
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {

backend/webdav/api/types.go

@@ -155,7 +155,7 @@ func (e *Error) Error() string {
 	return strings.Join(out, ": ")
 }
 
-// Time represents represents date and time information for the
+// Time represents date and time information for the
 // webdav API marshalling to and from timeFormat
 type Time time.Time
 
@@ -226,6 +226,6 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 //  </d:response>
 // </d:multistatus>
 type Quota struct {
-	Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
-	Used      int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
+	Available string `xml:"DAV: response>propstat>prop>quota-available-bytes"`
+	Used      string `xml:"DAV: response>propstat>prop>quota-used-bytes"`
 }

backend/webdav/odrvcookie/fetch.go

@@ -48,7 +48,7 @@ type SuccessResponseBody struct {
 	Token   string    `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"`
 }
 
-// SharepointError holds a error response microsoft login
+// SharepointError holds an error response microsoft login
 type SharepointError struct {
 	XMLName xml.Name          `xml:"Envelope"`
 	Body    ErrorResponseBody `xml:"Body"`
@@ -58,7 +58,7 @@ func (e *SharepointError) Error() string {
 	return fmt.Sprintf("%s: %s (%s)", e.Body.FaultCode, e.Body.Reason, e.Body.Detail)
 }
 
-// ErrorResponseBody contains the body of a erroneous repsonse
+// ErrorResponseBody contains the body of an erroneous response
 type ErrorResponseBody struct {
 	XMLName   xml.Name
 	FaultCode string `xml:"Fault>Code>Subcode>Value"`

backend/webdav/webdav.go

@@ -17,6 +17,7 @@ import (
 	"net/url"
 	"os/exec"
 	"path"
+	"strconv"
 	"strings"
 	"time"
 
@@ -975,10 +976,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
  </D:prop>
 </D:propfind>
 `))
-	var q = api.Quota{
-		Available: -1,
-		Used:      -1,
-	}
+	var q api.Quota
 	var resp *http.Response
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
@@ -989,14 +987,14 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 		return nil, errors.Wrap(err, "about call failed")
 	}
 	usage := &fs.Usage{}
-	if q.Used >= 0 {
-		usage.Used = fs.NewUsageValue(q.Used)
+	if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
+		usage.Used = fs.NewUsageValue(i)
 	}
-	if q.Available >= 0 {
-		usage.Free = fs.NewUsageValue(q.Available)
+	if i, err := strconv.ParseInt(q.Available, 10, 64); err == nil && i >= 0 {
+		usage.Free = fs.NewUsageValue(i)
 	}
-	if q.Available >= 0 && q.Used >= 0 {
-		usage.Total = fs.NewUsageValue(q.Available + q.Used)
+	if usage.Used != nil && usage.Free != nil {
+		usage.Total = fs.NewUsageValue(*usage.Used + *usage.Free)
 	}
 	return usage, nil
 }

backend/yandex/yandex.go

@@ -61,7 +61,7 @@ func init() {
 		Description: "Yandex Disk",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			err := oauthutil.Config("yandex", name, m, oauthConfig)
+			err := oauthutil.Config("yandex", name, m, oauthConfig, nil)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 				return
@@ -73,11 +73,6 @@ func init() {
 		}, {
 			Name: config.ConfigClientSecret,
 			Help: "Yandex Client Secret\nLeave blank normally.",
-		}, {
-			Name:     "unlink",
-			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -92,9 +87,8 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	Token  string               `config:"token"`
-	Unlink bool                 `config:"unlink"`
-	Enc    encoder.MultiEncoder `config:"encoding"`
+	Token string               `config:"token"`
+	Enc   encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote yandex
@@ -200,12 +194,12 @@ func (f *Fs) setRoot(root string) {
 	f.diskRoot = diskRoot
 }
 
-// filePath returns a escaped file path (f.root, file)
+// filePath returns an escaped file path (f.root, file)
 func (f *Fs) filePath(file string) string {
 	return path.Join(f.diskRoot, file)
 }
 
-// dirPath returns a escaped file path (f.root, file) ending with '/'
+// dirPath returns an escaped file path (f.root, file) ending with '/'
 func (f *Fs) dirPath(file string) string {
 	return path.Join(f.diskRoot, file) + "/"
 }
@@ -502,7 +496,7 @@ func (f *Fs) mkDirs(ctx context.Context, path string) (err error) {
 
 	if err = f.CreateDir(ctx, dirString); err != nil {
 		if apiErr, ok := err.(*api.ErrorResponse); ok {
-			// allready exists
+			// already exists
 			if apiErr.ErrorName != "DiskPathPointsToExistentDirectoryError" {
 				// 2 if it fails then create all directories in the path from root.
 				dirs := strings.Split(dirString, "/") //path separator
@@ -801,9 +795,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 }
 
 // PublicLink generates a public link to the remote path (usually readable by anyone)
-func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
+func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
 	var path string
-	if f.opt.Unlink {
+	if unlink {
 		path = "/resources/unpublish"
 	} else {
 		path = "/resources/publish"
@@ -830,7 +824,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err er
 		}
 	}
 	if err != nil {
-		if f.opt.Unlink {
+		if unlink {
 			return "", errors.Wrap(err, "couldn't remove public link")
 		}
 		return "", errors.Wrap(err, "couldn't create public link")

bin/check-merged.go

@@ -55,7 +55,7 @@ func gitDiffDiff(rev1, rev2 string) {
 		if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
 			// OK just different
 		} else {
-			log.Fatalf("git diff diff failed: %#v", err)
+			log.Fatalf("git diff failed: %#v", err)
 		}
 	}
 	_, _ = os.Stdout.Write(out)

bin/cross-compile.go

@@ -5,6 +5,7 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -20,6 +21,8 @@ import (
 	"sync"
 	"text/template"
 	"time"
+
+	"github.com/coreos/go-semver/semver"
 )
 
 var (
@@ -168,12 +171,109 @@ func buildDebAndRpm(dir, version, goarch string) []string {
 	return artifacts
 }
 
+// generate system object (syso) file to be picked up by a following go build for embedding icon and version info resources into windows executable
+func buildWindowsResourceSyso(goarch string, versionTag string) string {
+	type M map[string]interface{}
+	version := strings.TrimPrefix(versionTag, "v")
+	semanticVersion := semver.New(version)
+
+	// Build json input to goversioninfo utility
+	bs, err := json.Marshal(M{
+		"FixedFileInfo": M{
+			"FileVersion": M{
+				"Major": semanticVersion.Major,
+				"Minor": semanticVersion.Minor,
+				"Patch": semanticVersion.Patch,
+			},
+			"ProductVersion": M{
+				"Major": semanticVersion.Major,
+				"Minor": semanticVersion.Minor,
+				"Patch": semanticVersion.Patch,
+			},
+		},
+		"StringFileInfo": M{
+			"CompanyName":      "https://rclone.org",
+			"ProductName":      "Rclone",
+			"FileDescription":  "Rsync for cloud storage",
+			"InternalName":     "rclone",
+			"OriginalFilename": "rclone.exe",
+			"LegalCopyright":   "The Rclone Authors",
+			"FileVersion":      version,
+			"ProductVersion":   version,
+		},
+		"IconPath": "../graphics/logo/ico/logo_symbol_color.ico",
+	})
+	if err != nil {
+		log.Printf("Failed to build version info json: %v", err)
+		return ""
+	}
+
+	// Write json to temporary file that will only be used by the goversioninfo command executed below.
+	jsonPath, err := filepath.Abs("versioninfo_windows_" + goarch + ".json") // Appending goos and goarch as suffix to avoid any race conditions
+	if err != nil {
+		log.Printf("Failed to resolve path: %v", err)
+		return ""
+	}
+	err = ioutil.WriteFile(jsonPath, bs, 0644)
+	if err != nil {
+		log.Printf("Failed to write %s: %v", jsonPath, err)
+		return ""
+	}
+	defer func() {
+		if err := os.Remove(jsonPath); err != nil {
+			if !os.IsNotExist(err) {
+				log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", jsonPath, err)
+			}
+		}
+	}()
+
+	// Execute goversioninfo utility using the json file as input.
+	// It will produce a system object (syso) file that a following go build should pick up.
+	sysoPath, err := filepath.Abs("../resource_windows_" + goarch + ".syso") // Appending goos and goarch as suffix to avoid any race conditions, and also it is recognized by go build and avoids any builds for other systems considering it
+	if err != nil {
+		log.Printf("Failed to resolve path: %v", err)
+		return ""
+	}
+	args := []string{
+		"goversioninfo",
+		"-o",
+		sysoPath,
+	}
+	if goarch == "amd64" {
+		args = append(args, "-64") // Make the syso a 64-bit coff file
+	}
+	args = append(args, jsonPath)
+	err = runEnv(args, nil)
+	if err != nil {
+		return ""
+	}
+
+	return sysoPath
+}
+
+// delete generated system object (syso) resource file
+func cleanupResourceSyso(sysoFilePath string) {
+	if sysoFilePath == "" {
+		return
+	}
+	if err := os.Remove(sysoFilePath); err != nil {
+		if !os.IsNotExist(err) {
+			log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
+		}
+	}
+}
+
 // build the binary in dir returning success or failure
 func compileArch(version, goos, goarch, dir string) bool {
 	log.Printf("Compiling %s/%s", goos, goarch)
 	output := filepath.Join(dir, "rclone")
 	if goos == "windows" {
 		output += ".exe"
+		sysoPath := buildWindowsResourceSyso(goarch, version)
+		if sysoPath == "" {
+			log.Printf("Warning: Windows binaries will not have file information embedded")
+		}
+		defer cleanupResourceSyso(sysoPath)
 	}
 	err := os.MkdirAll(dir, 0777)
 	if err != nil {

bin/get-github-release.go

@@ -42,10 +42,10 @@ var (
 	// Globals
 	matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
 	osAliases    = map[string][]string{
-		"darwin": []string{"macos", "osx"},
+		"darwin": {"macos", "osx"},
 	}
 	archAliases = map[string][]string{
-		"amd64": []string{"x86_64"},
+		"amd64": {"x86_64"},
 	}
 )
 

bin/make_backend_docs.py

@@ -7,9 +7,10 @@ import os
 import io
 import subprocess
 
-marker = "<!--- autogenerated options"
+marker = "{{< rem autogenerated options"
 start = marker + " start"
 stop = marker + " stop"
+end = ">}}"
 
 def find_backends():
     """Return a list of all backends"""
@@ -38,11 +39,11 @@ def alter_doc(backend):
             if not in_docs:
                 if start in line:
                     in_docs = True
-                    start_full = start + " - DO NOT EDIT, instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs -->\n" % (backend, backend)
+                    start_full = (start + "\" - DO NOT EDIT - instead edit fs.RegInfo in backend/%s/%s.go then run make backenddocs\" " + end + "\n") % (backend, backend)
                     out_file.write(start_full)
                     output_docs(backend, out_file)
                     output_backend_tool_docs(backend, out_file)
-                    out_file.write(stop+" -->\n")
+                    out_file.write(stop+" "+end+"\n")
                     altered = True
             if not in_docs:
                 out_file.write(line)

bin/make_changelog.py

@@ -156,17 +156,23 @@ def main():
 
     # Output combined changelog with new part
     sys.stdout.write(old_head)
+    today = datetime.date.today()
+    new_features = "\n".join(new_features_lines)
+    bugfixes = "\n".join(bugfix_lines)
+    backend_changes = "\n".join(backend_lines)
     sys.stdout.write("""
 
-## %s - %s
+## %(next_version)s - %(today)s
+
+[See commits](https://github.com/rclone/rclone/compare/%(version)s...%(next_version)s)
 
 * New backends
 * New commands
 * New Features
-%s
+%(new_features)s
 * Bug Fixes
-%s
-%s""" % (next_version, datetime.date.today(), "\n".join(new_features_lines), "\n".join(bugfix_lines), "\n".join(backend_lines)))
+%(bugfixes)s
+%(backend_changes)s""" % locals())
     sys.stdout.write(old_tail)
                 
 

bin/make_manual.py

@@ -13,7 +13,7 @@ outfile = "MANUAL.md"
 
 # Order to add docs segments to make outfile
 docs = [
-    "about.md",
+    "_index.md",
     "install.md",
     "docs.md",
     "remote_setup.md",
@@ -111,11 +111,21 @@ def read_doc(doc):
     contents = parts[2].strip()+"\n\n"
     # Remove icons
     contents = re.sub(r'<i class="fa.*?</i>\s*', "", contents)
+    # Interpret img shortcodes
+    # {{< img ... >}}
+    contents = re.sub(r'\{\{<\s*img\s+(.*?)>\}\}', r"<img \1>", contents)
+    # Make any img tags absolute
+    contents = re.sub(r'(<img.*?src=")/', r"\1https://rclone.org/", contents)
     # Make [...](/links/) absolute
     contents = re.sub(r'\]\((\/.*?\/(#.*)?)\)', r"](https://rclone.org\1)", contents)
+    # Add additional links on the front page
+    contents = re.sub(r'\{\{< rem MAINPAGELINK >\}\}', "- [Donate.](https://rclone.org/donate/)", contents)
     # Interpret provider shortcode
     # {{< provider name="Amazon S3" home="https://aws.amazon.com/s3/" config="/s3/" >}}
-    contents = re.sub(r'\{\{<\s+provider.*?name="(.*?)".*?>\}\}', r"\1", contents)
+    contents = re.sub(r'\{\{<\s*provider.*?name="(.*?)".*?>\}\}', r"- \1", contents)
+    # Remove remaining shortcodes
+    contents = re.sub(r'\{\{<.*?>\}\}', r"", contents)
+    contents = re.sub(r'\{\{%.*?%\}\}', r"", contents)
     return contents
 
 def check_docs(docpath):

bin/make_rc_docs.sh

@@ -15,8 +15,8 @@ fusermount -u -z /tmp/rclone/rc_mount > /dev/null 2>&1 || umount /tmp/rclone/rc_
 
 awk '
     BEGIN       {p=1}
-    /^<!--- autogenerated start/  {print;system("cat /tmp/rclone/z.md");p=0}
-    /^<!--- autogenerated stop/   {p=1}
+    /^\{\{< rem autogenerated start/  {print;system("cat /tmp/rclone/z.md");p=0}
+    /^\{\{< rem autogenerated stop/   {p=1}
     p' docs/content/rc.md > /tmp/rclone/rc.md
 
 mv /tmp/rclone/rc.md docs/content/rc.md

bin/nfpm.yaml

@@ -15,7 +15,8 @@ description: |
 vendor: "rclone"
 homepage: "https://rclone.org"
 license: "MIT"
-bindir: "/usr/bin"
+# No longer supported? See https://github.com/goreleaser/nfpm/issues/144
+# bindir: "/usr/bin"
 files:
   ./rclone: "/usr/bin/rclone"
   ./README.html: "/usr/share/doc/rclone/README.html"