fs: Debug the names of env vars we read values from as part of the config

See: https://forum.rclone.org/t/how-can-i-check-if-environment-variables-are-being-used/17490

.github/workflows/build.yml

@@ -19,23 +19,24 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'go1.11', 'go1.12', 'go1.13', 'go1.14']
+        job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'modules_race', 'go1.11', 'go1.12', 'go1.13']
 
         include:
           - job_name: linux
             os: ubuntu-latest
-            go: '1.15.x'
+            go: '1.14.x'
+            modules: 'off'
             gotags: cmount
             build_flags: '-include "^linux/"'
             check: true
             quicktest: true
-            racequicktest: true
             deploy: true
 
           - job_name: mac
             os: macOS-latest
-            go: '1.15.x'
-            gotags: 'cmount'
+            go: '1.14.x'
+            modules: 'off'
+            gotags: ''  # cmount doesn't work on osx travis for some reason
             build_flags: '-include "^darwin/amd64" -cgo'
             quicktest: true
             racequicktest: true
@@ -43,7 +44,8 @@ jobs:
 
           - job_name: windows_amd64
             os: windows-latest
-            go: '1.15.x'
+            go: '1.14.x'
+            modules: 'off'
             gotags: cmount
             build_flags: '-include "^windows/amd64" -cgo'
             quicktest: true
@@ -52,7 +54,8 @@ jobs:
 
           - job_name: windows_386
             os: windows-latest
-            go: '1.15.x'
+            go: '1.14.x'
+            modules: 'off'
             gotags: cmount
             goarch: '386'
             cgo: '1'
@@ -62,51 +65,59 @@ jobs:
 
           - job_name: other_os
             os: ubuntu-latest
-            go: '1.15.x'
+            go: '1.14.x'
+            modules: 'off'
             build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
             compile_all: true
             deploy: true
 
+          - job_name: modules_race
+            os: ubuntu-latest
+            go: '1.14.x'
+            modules: 'on'
+            quicktest: true
+            racequicktest: true
+
           - job_name: go1.11
             os: ubuntu-latest
             go: '1.11.x'
+            modules: 'off'
             quicktest: true
 
           - job_name: go1.12
             os: ubuntu-latest
             go: '1.12.x'
+            modules: 'off'
             quicktest: true
 
           - job_name: go1.13
             os: ubuntu-latest
             go: '1.13.x'
+            modules: 'off'
             quicktest: true
 
-          - job_name: go1.14
-            os: ubuntu-latest
-            go: '1.14.x'
-            quicktest: true
-            racequicktest: true
-
     name: ${{ matrix.job_name }}
 
     runs-on: ${{ matrix.os }}
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v1
         with:
-          fetch-depth: 0
+          # Checkout into a fixed path to avoid import path problems on go < 1.11
+          path: ./src/github.com/rclone/rclone
 
       - name: Install Go
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v1
         with:
-          stable: 'false'
           go-version: ${{ matrix.go }}
 
       - name: Set environment variables
         shell: bash
         run: |
+          echo '::set-env name=GOPATH::${{ runner.workspace }}'
+          echo '::add-path::${{ runner.workspace }}/bin'
+          echo '::set-env name=GO111MODULE::${{ matrix.modules }}'
           echo '::set-env name=GOTAGS::${{ matrix.gotags }}'
           echo '::set-env name=BUILD_FLAGS::${{ matrix.build_flags }}'
           if [[ "${{ matrix.goarch }}" != "" ]]; then echo '::set-env name=GOARCH::${{ matrix.goarch }}' ; fi
@@ -124,8 +135,6 @@ jobs:
       - name: Install Libraries on macOS
         shell: bash
         run: |
-          brew untap local/homebrew-openssl  # workaround for https://github.com/actions/virtual-environments/issues/1811
-          brew untap local/homebrew-python2  # workaround for https://github.com/actions/virtual-environments/issues/1811
           brew update
           brew cask install osxfuse
         if: matrix.os == 'macOS-latest'
@@ -158,22 +167,10 @@ jobs:
           printf "\n\nSystem environment:\n\n"
           env
 
-      - name: Go module cache
-        uses: actions/cache@v2
-        with:
-          path: ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
-
-      - name: Build rclone
-        shell: bash
-        run: |
-          make
-
       - name: Run tests
         shell: bash
         run: |
+          make
           make quicktest
         if: matrix.quicktest
 
@@ -232,12 +229,12 @@ jobs:
         run: |
           docker pull billziss/xgo-cgofuse
           GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
-          # xgo \
-          #     -image=billziss/xgo-cgofuse \
-          #     -targets=darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
-          #     -tags cmount \
-          #     -dest build \
-          #     .
+          xgo \
+              -image=billziss/xgo-cgofuse \
+              -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
+              -tags cmount \
+              -dest build \
+              .
           xgo \
               -image=billziss/xgo-cgofuse \
               -targets=android/*,ios/* \
@@ -245,9 +242,9 @@ jobs:
               .
 
       - name: Build rclone
-        shell: bash
         run: |
-          make
+          docker pull golang
+          docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=vendor -v
 
       - name: Upload artifacts
         run: |

.gitignore

@@ -9,4 +9,3 @@ rclone.iml
 *.test
 *.log
 *.iml
-fuzz-build.zip

CONTRIBUTING.md

@@ -86,7 +86,7 @@ git reset --soft HEAD~2 # This squashes the 2 latest commits together.
 git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
 git commit # Add a new commit message.
 git push --force # Push the squashed commit to your GitHub repo.
-# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also recommends wizardzines.com
+# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also reccommends wizardzines.com
 ```
 
 ## CI for your fork ##
@@ -186,6 +186,7 @@ with modules beneath.
     * pacer - retries with backoff and paces operations
     * readers - a selection of useful io.Readers
     * rest - a thin abstraction over net/http for REST
+  * vendor - 3rd party code managed by `go mod`
   * vfs - Virtual FileSystem layer for implementing rclone mount and similar
 
 ## Writing Documentation ##
@@ -265,25 +266,41 @@ rclone uses the [go
 modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
 support in go1.11 and later to manage its dependencies.
 
-rclone can be built with modules outside of the GOPATH
+**NB** you must be using go1.11 or above to add a dependency to
+rclone.  Rclone will still build with older versions of go, but we use
+the `go mod` command for dependencies which is only in go1.11 and
+above.
+
+rclone can be built with modules outside of the GOPATH, but for
+backwards compatibility with older go versions, rclone also maintains
+a `vendor` directory with all the external code rclone needs for
+building.
+
+The `vendor` directory is entirely managed by the `go mod` tool, do
+not add things manually.
 
 To add a dependency `github.com/ncw/new_dependency` see the
-instructions below.  These will fetch the dependency and add it to
-`go.mod` and `go.sum`.
+instructions below.  These will fetch the dependency, add it to
+`go.mod` and `go.sum` and vendor it for older go versions.
 
     GO111MODULE=on go get github.com/ncw/new_dependency
+    GO111MODULE=on go mod vendor
 
 You can add constraints on that package when doing `go get` (see the
 go docs linked above), but don't unless you really need to.
 
-Please check in the changes generated by `go mod` including `go.mod`
-and `go.sum` in the same commit as your other changes.
+Please check in the changes generated by `go mod` including the
+`vendor` directory and `go.mod` and `go.sum` in a single commit
+separate from any other code changes with the title "vendor: add
+github.com/ncw/new_dependency".  Remember to `git add` any new files
+in `vendor`.
 
 ## Updating a dependency ##
 
 If you need to update a dependency then run
 
     GO111MODULE=on go get -u github.com/pkg/errors
+    GO111MODULE=on go mod vendor
 
 Check in a single commit as above.
 

Makefile

@@ -7,29 +7,27 @@ RELEASE_TAG := $(shell git tag -l --points-at HEAD)
 VERSION := $(shell cat VERSION)
 # Last tag on this branch
 LAST_TAG := $(shell git describe --tags --abbrev=0)
-# Next version
-NEXT_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2+1,0}')
-NEXT_PATCH_VERSION := $(shell echo $(VERSION) | awk -F. -v OFS=. '{print $$1,$$2,$$3+1}')
 # If we are working on a release, override branch to master
 ifdef RELEASE_TAG
 	BRANCH := master
-	LAST_TAG := $(shell git describe --abbrev=0 --tags $(VERSION)^)
 endif
-TAG_BRANCH := .$(BRANCH)
-BRANCH_PATH := branch/$(BRANCH)/
+TAG_BRANCH := -$(BRANCH)
+BRANCH_PATH := branch/
 # If building HEAD or master then unset TAG_BRANCH and BRANCH_PATH
 ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
 	TAG_BRANCH :=
 	BRANCH_PATH :=
 endif
-# Make version suffix -beta.NNNN.CCCCCCCC (N=Commit number, C=Commit)
-VERSION_SUFFIX := -beta.$(shell git rev-list --count HEAD).$(shell git show --no-patch --no-notes --pretty='%h' HEAD)
-# TAG is current version + commit number + commit + branch
+# Make version suffix -DDD-gCCCCCCCC (D=commits since last relase, C=Commit) or blank
+VERSION_SUFFIX := $(shell git describe --abbrev=8 --tags | perl -lpe 's/^v\d+\.\d+\.\d+//; s/^-(\d+)/"-".sprintf("%03d",$$1)/e;')
+# TAG is current version + number of commits since last release + branch
 TAG := $(VERSION)$(VERSION_SUFFIX)$(TAG_BRANCH)
-ifdef RELEASE_TAG
-	TAG := $(RELEASE_TAG)
+NEXT_VERSION := $(shell echo $(VERSION) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f.0", $$_)')
+ifndef RELEASE_TAG
+	TAG := $(TAG)-beta
 endif
 GO_VERSION := $(shell go version)
+GO_FILES := $(shell go list ./... | grep -v /vendor/ )
 ifdef BETA_SUBDIR
 	BETA_SUBDIR := /$(BETA_SUBDIR)
 endif
@@ -59,6 +57,7 @@ vars:
 	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
 	@echo VERSION="'$(VERSION)'"
+	@echo NEXT_VERSION="'$(NEXT_VERSION)'"
 	@echo GO_VERSION="'$(GO_VERSION)'"
 	@echo BETA_URL="'$(BETA_URL)'"
 
@@ -76,10 +75,10 @@ test:	rclone test_all
 
 # Quick test
 quicktest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) $(GO_FILES)
 
 racequicktest:
-	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race ./...
+	RCLONE_CONFIG="/notfound" go test $(BUILDTAGS) -cpu=2 -race $(GO_FILES)
 
 # Do source code quality checks
 check:	rclone
@@ -101,19 +100,15 @@ release_dep_windows:
 	GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
 
 # Update dependencies
-showupdates:
-	@echo "*** Direct dependencies that could be updated ***"
-	@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
-
-# Update direct and indirect dependencies and test dependencies
 update:
-	GO111MODULE=on go get -u -t ./...
-	-#GO111MODULE=on go get -d $(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
+	GO111MODULE=on go get -u ./...
 	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
 
 # Tidy the module dependencies
 tidy:
 	GO111MODULE=on go mod tidy
+	GO111MODULE=on go mod vendor
 
 doc:	rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
 
@@ -165,11 +160,6 @@ validate_website: website
 tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
-vendorball:
-	go mod vendor
-	tar -zcf build/rclone-$(TAG)-vendor.tar.gz vendor
-	rm -rf vendor
-
 sign_upload:
 	cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS
 	cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS
@@ -227,33 +217,26 @@ fetch_binaries:
 serve:	website
 	cd docs && hugo server -v -w --disableFastRender
 
-tag:	retag doc
-	bin/make_changelog.py $(LAST_TAG) $(VERSION) > docs/content/changelog.md.new
+tag:	doc
+	@echo "Old tag is $(VERSION)"
+	@echo "New tag is $(NEXT_VERSION)"
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)\"\n" | gofmt > fs/version.go
+	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
+	echo "$(NEXT_VERSION)" > VERSION
+	git tag -s -m "Version $(NEXT_VERSION)" $(NEXT_VERSION)
+	bin/make_changelog.py $(LAST_TAG) $(NEXT_VERSION) > docs/content/changelog.md.new
 	mv docs/content/changelog.md.new docs/content/changelog.md
 	@echo "Edit the new changelog in docs/content/changelog.md"
 	@echo "Then commit all the changes"
-	@echo git commit -m \"Version $(VERSION)\" -a -v
+	@echo git commit -m \"Version $(NEXT_VERSION)\" -a -v
 	@echo "And finally run make retag before make cross etc"
 
 retag:
-	@echo "Version is $(VERSION)"
 	git tag -f -s -m "Version $(VERSION)" $(VERSION)
 
 startdev:
-	@echo "Version is $(VERSION)"
-	@echo "Next version is $(NEXT_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
-	echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
-
-startstable:
-	@echo "Version is $(VERSION)"
-	@echo "Next stable version is $(NEXT_PATCH_VERSION)"
-	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
-	echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
-	echo "$(NEXT_PATCH_VERSION)" > VERSION
-	git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
+	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(VERSION)-DEV\"\n" | gofmt > fs/version.go
+	git commit -m "Start $(VERSION)-DEV development" fs/version.go
 
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe

README.md

@@ -64,7 +64,6 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * StackPath [:page_facing_up:](https://rclone.org/s3/#stackpath)
   * SugarSync [:page_facing_up:](https://rclone.org/sugarsync/)
   * Tardigrade [:page_facing_up:](https://rclone.org/tardigrade/)
-  * Tencent Cloud Object Storage (COS) [:page_facing_up:](https://rclone.org/s3/#tencent-cos)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
   * Yandex Disk [:page_facing_up:](https://rclone.org/yandex/)

RELEASE.md

@@ -9,7 +9,7 @@ This file describes how to make the various kinds of releases
 
 ## Making a release
 
-  * git checkout master # see below for stable branch
+  * git checkout master
   * git pull
   * git status - make sure everything is checked in
   * Check GitHub actions build for master is Green
@@ -21,20 +21,19 @@ This file describes how to make the various kinds of releases
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX.0"
   * make retag
-  * git push --follow-tags origin
+  * git push --tags origin master
   * # Wait for the GitHub builds to complete then...
   * make fetch_binaries
   * make tarball
-  * make vendorball
   * make sign_upload
   * make check_sign
   * make upload
   * make upload_website
   * make upload_github
-  * make startdev # make startstable for stable branch
+  * make startdev
   * # announce with forum post, twitter post, patreon post
 
-Early in the next release cycle update the dependencies
+Early in the next release cycle update the vendored dependencies
 
   * Review any pinned packages in go.mod and remove if possible
   * make update
@@ -42,35 +41,61 @@ Early in the next release cycle update the dependencies
   * git add new files
   * git commit -a -v
 
+If `make update` fails with errors like this:
+
+```
+# github.com/cpuguy83/go-md2man/md2man
+../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:11:16: undefined: blackfriday.EXTENSION_NO_INTRA_EMPHASIS
+../../../../pkg/mod/github.com/cpuguy83/go-md2man@v1.0.8/md2man/md2man.go:12:16: undefined: blackfriday.EXTENSION_TABLES
+```
+
+Can be fixed with
+
+    * GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
+    * GO111MODULE=on go mod tidy
+    * GO111MODULE=on go mod vendor
+ 
+
 ## Making a point release
 
 If rclone needs a point release due to some horrendous bug:
 
-Set vars
+First make the release branch.  If this is a second point release then
+this will be done already.
 
   * BASE_TAG=v1.XX          # eg v1.52
   * NEW_TAG=${BASE_TAG}.Y   # eg v1.52.1
   * echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
-
-First make the release branch.  If this is a second point release then
-this will be done already.
-
   * git branch ${BASE_TAG} ${BASE_TAG}-stable
-  * git co ${BASE_TAG}-stable
-  * make startstable
 
 Now
 
   * git co ${BASE_TAG}-stable
   * git cherry-pick any fixes
-  * Do the steps as above
-  * make startstable
+  * Test (see above)
+  * make NEXT_VERSION=${NEW_TAG} tag
+  * edit docs/content/changelog.md
+  * make TAG=${NEW_TAG} doc
+  * git commit -a -v -m "Version ${NEW_TAG}"
+  * git tag -d ${NEW_TAG}
+  * git tag -s -m "Version ${NEW_TAG}" ${NEW_TAG}
+  * git push --tags -u origin ${BASE_TAG}-stable
+  * Wait for builds to complete
+  * make BRANCH_PATH= TAG=${NEW_TAG} fetch_binaries
+  * make TAG=${NEW_TAG} tarball
+  * make TAG=${NEW_TAG} sign_upload
+  * make TAG=${NEW_TAG} check_sign
+  * make TAG=${NEW_TAG} upload
+  * make TAG=${NEW_TAG} upload_website
+  * make TAG=${NEW_TAG} upload_github
   * NB this overwrites the current beta so we need to do this
   * git co master
-  * # cherry pick the changes to the changelog
-  * git checkout ${BASE_TAG}-stable docs/content/changelog.md
-  * git commit -a -v -m "Changelog updates from Version ${NEW_TAG}"
+  * make VERSION=${NEW_TAG} startdev
+  * # cherry pick the changes to the changelog and VERSION
+  * git checkout ${BASE_TAG}-stable VERSION docs/content/changelog.md
+  * git commit --amend
   * git push
+  * Announce!
 
 ## Making a manual build of docker
 

VERSION

@@ -1 +1 @@
-v1.54.0
+v1.52.2

backend/alias/alias.go

@@ -5,7 +5,6 @@ import (
 	"strings"
 
 	"github.com/rclone/rclone/fs"
-	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
@@ -47,5 +46,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(opt.Remote, name+":") {
 		return nil, errors.New("can't point alias remote at itself - check the value of the remote setting")
 	}
-	return cache.Get(fspath.JoinRootPath(opt.Remote, root))
+	fsInfo, configName, fsPath, config, err := fs.ConfigFs(opt.Remote)
+	if err != nil {
+		return nil, err
+	}
+	return fsInfo.NewFs(configName, fspath.JoinRootPath(fsPath, root), config)
 }

backend/amazonclouddrive/amazonclouddrive.go

@@ -76,7 +76,23 @@ func init() {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name:     config.ConfigClientID,
+			Help:     "Amazon Application Client ID.",
+			Required: true,
+		}, {
+			Name:     config.ConfigClientSecret,
+			Help:     "Amazon Application Client Secret.",
+			Required: true,
+		}, {
+			Name:     config.ConfigAuthURL,
+			Help:     "Auth server URL.\nLeave blank to use Amazon's.",
+			Advanced: true,
+		}, {
+			Name:     config.ConfigTokenURL,
+			Help:     "Token server url.\nleave blank to use Amazon's.",
+			Advanced: true,
+		}, {
 			Name:     "checkpoint",
 			Help:     "Checkpoint for internal polling (debug).",
 			Hide:     fs.OptionHideBoth,
@@ -127,7 +143,7 @@ underlying S3 storage.`,
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			Default: (encoder.Base |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
@@ -921,8 +937,8 @@ func (f *Fs) Hashes() hash.Set {
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // ------------------------------------------------------------

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !plan9,!solaris,!js,go1.13
+// +build !plan9,!solaris
 
 package azureblob
 
@@ -967,7 +967,8 @@ func (f *Fs) Hashes() hash.Set {
 }
 
 // Purge deletes all the files and directories including the old versions.
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
+	dir := "" // forward compat!
 	container, directory := f.split(dir)
 	if container == "" || directory != "" {
 		// Delegate to caller if not root of a container
@@ -1245,15 +1246,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 	}
 	blob := o.getBlobReference()
 	ac := azblob.BlobAccessConditions{}
-	var downloadResponse *azblob.DownloadResponse
+	var dowloadResponse *azblob.DownloadResponse
 	err = o.fs.pacer.Call(func() (bool, error) {
-		downloadResponse, err = blob.Download(ctx, offset, count, ac, false)
+		dowloadResponse, err = blob.Download(ctx, offset, count, ac, false)
 		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open for download")
 	}
-	in = downloadResponse.Body(azblob.RetryReaderOptions{})
+	in = dowloadResponse.Body(azblob.RetryReaderOptions{})
 	return in, nil
 }
 
@@ -1475,7 +1476,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 	}
 	// FIXME Until https://github.com/Azure/azure-storage-blob-go/pull/75
 	// is merged the SDK can't upload a single blob of exactly the chunk
-	// size, so upload with a multipart upload to work around.
+	// size, so upload with a multpart upload to work around.
 	// See: https://github.com/rclone/rclone/issues/2653
 	multipartUpload := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
 	if size == int64(o.fs.opt.ChunkSize) {

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!solaris,!js,go1.13
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !plan9,!solaris,!js,go1.13
+// +build !plan9,!solaris
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 solaris js !go1.13
+// +build plan9 solaris
 
 package azureblob

backend/b2/b2.go

@@ -1143,8 +1143,7 @@ func (f *Fs) deleteByID(ctx context.Context, ID, Name string) error {
 // if oldOnly is true then it deletes only non current files.
 //
 // Implemented here so we can make sure we delete old versions.
-func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
-	bucket, directory := f.split(dir)
+func (f *Fs) purge(ctx context.Context, bucket, directory string, oldOnly bool) error {
 	if bucket == "" {
 		return errors.New("can't purge from root")
 	}
@@ -1219,19 +1218,19 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
 	wg.Wait()
 
 	if !oldOnly {
-		checkErr(f.Rmdir(ctx, dir))
+		checkErr(f.Rmdir(ctx, ""))
 	}
 	return errReturn
 }
 
 // Purge deletes all the files and directories including the old versions.
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purge(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purge(ctx, f.rootBucket, f.rootDirectory, false)
 }
 
 // CleanUp deletes all the hidden files.
 func (f *Fs) CleanUp(ctx context.Context) error {
-	return f.purge(ctx, "", true)
+	return f.purge(ctx, f.rootBucket, f.rootDirectory, true)
 }
 
 // copy does a server side copy from dstObj <- srcObj
@@ -1673,8 +1672,8 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		opts.RootURL = o.fs.opt.DownloadURL
 	}
 
-	// Download by id if set and not using DownloadURL otherwise by name
-	if o.id != "" && o.fs.opt.DownloadURL == "" {
+	// Download by id if set otherwise by name
+	if o.id != "" {
 		opts.Path += "/b2api/v1/b2_download_file_by_id?fileId=" + urlEncode(o.id)
 	} else {
 		bucket, bucketPath := o.split()

backend/box/box.go

@@ -87,23 +87,26 @@ func init() {
 		Config: func(name string, m configmap.Mapper) {
 			jsonFile, ok := m.Get("box_config_file")
 			boxSubType, boxSubTypeOk := m.Get("box_sub_type")
-			boxAccessToken, boxAccessTokenOk := m.Get("access_token")
 			var err error
-			// If using box config.json, use JWT auth
 			if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
 				err = refreshJWTToken(jsonFile, boxSubType, name, m)
 				if err != nil {
 					log.Fatalf("Failed to configure token with jwt authentication: %v", err)
 				}
-				// Else, if not using an access token, use oauth2
-			} else if boxAccessToken == "" || !boxAccessTokenOk {
+			} else {
 				err = oauthutil.Config("box", name, m, oauthConfig, nil)
 				if err != nil {
 					log.Fatalf("Failed to configure token with oauth authentication: %v", err)
 				}
 			}
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Box App Client Id.\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Box App Client Secret\nLeave blank normally.",
+		}, {
 			Name:     "root_folder_id",
 			Help:     "Fill in for rclone to use a non root folder as its starting point.",
 			Default:  "0",
@@ -111,9 +114,6 @@ func init() {
 		}, {
 			Name: "box_config_file",
 			Help: "Box App config.json location\nLeave blank normally." + env.ShellExpandHelp,
-		}, {
-			Name: "access_token",
-			Help: "Box App Primary Access Token\nLeave blank normally.",
 		}, {
 			Name:    "box_sub_type",
 			Default: "user",
@@ -149,7 +149,7 @@ func init() {
 				encoder.EncodeBackSlash |
 				encoder.EncodeRightSpace |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
@@ -247,7 +247,6 @@ type Options struct {
 	CommitRetries int                  `config:"commit_retries"`
 	Enc           encoder.MultiEncoder `config:"encoding"`
 	RootFolderID  string               `config:"root_folder_id"`
-	AccessToken   string               `config:"access_token"`
 }
 
 // Fs represents a remote box
@@ -386,22 +385,16 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	root = parsePath(root)
-
-	client := fshttp.NewClient(fs.Config)
-	var ts *oauthutil.TokenSource
-	// If not using an accessToken, create an oauth client and tokensource
-	if opt.AccessToken == "" {
-		client, ts, err = oauthutil.NewClient(name, m, oauthConfig)
-		if err != nil {
-			return nil, errors.Wrap(err, "failed to configure Box")
-		}
+	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to configure Box")
 	}
 
 	f := &Fs{
 		name:        name,
 		root:        root,
 		opt:         *opt,
-		srv:         rest.NewClient(client).SetRoot(rootURL),
+		srv:         rest.NewClient(oAuthClient).SetRoot(rootURL),
 		pacer:       fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
@@ -411,30 +404,23 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	f.srv.SetErrorHandler(errorHandler)
 
-	// If using an accessToken, set the Authorization header
-	if f.opt.AccessToken != "" {
-		f.srv.SetHeader("Authorization", "Bearer "+f.opt.AccessToken)
-	}
-
 	jsonFile, ok := m.Get("box_config_file")
 	boxSubType, boxSubTypeOk := m.Get("box_sub_type")
 
-	if ts != nil {
-		// If using box config.json and JWT, renewing should just refresh the token and
-		// should do so whether there are uploads pending or not.
-		if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
-			f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-				err := refreshJWTToken(jsonFile, boxSubType, name, m)
-				return err
-			})
-			f.tokenRenewer.Start()
-		} else {
-			// Renew the token in the background
-			f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
-				_, err := f.readMetaDataForPath(ctx, "")
-				return err
-			})
-		}
+	// If using box config.json and JWT, renewing should just refresh the token and
+	// should do so whether there are uploads pending or not.
+	if ok && boxSubTypeOk && jsonFile != "" && boxSubType != "" {
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+			err := refreshJWTToken(jsonFile, boxSubType, name, m)
+			return err
+		})
+		f.tokenRenewer.Start()
+	} else {
+		// Renew the token in the background
+		f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+			_, err := f.readMetaDataForPath(ctx, "")
+			return err
+		})
 	}
 
 	// Get rootFolderID
@@ -856,8 +842,8 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // move a file or folder
@@ -1013,7 +999,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	return info.SharedLink.URL, err
 }
 
-// deletePermanently permanently deletes a trashed file
+// deletePermanently permenently deletes a trashed file
 func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
 	opts := rest.Opts{
 		Method:     "DELETE",
@@ -1272,10 +1258,8 @@ func (o *Object) upload(ctx context.Context, in io.Reader, leaf, directoryID str
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
-	if o.fs.tokenRenewer != nil {
-		o.fs.tokenRenewer.Start()
-		defer o.fs.tokenRenewer.Stop()
-	}
+	o.fs.tokenRenewer.Start()
+	defer o.fs.tokenRenewer.Stop()
 
 	size := src.Size()
 	modTime := src.ModTime(ctx)

backend/box/upload.go

@@ -1,4 +1,4 @@
-// multipart upload for box
+// multpart upload for box
 
 package box
 

backend/cache/cache.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 
@@ -361,10 +361,15 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.Wrapf(err, "failed to clean root path %q", rootPath)
 	}
 
-	remotePath := fspath.JoinRootPath(opt.Remote, rootPath)
-	wrappedFs, wrapErr := cache.Get(remotePath)
+	wInfo, wName, wPath, wConfig, err := fs.ConfigFs(opt.Remote)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", opt.Remote)
+	}
+
+	remotePath := fspath.JoinRootPath(wPath, rootPath)
+	wrappedFs, wrapErr := wInfo.NewFs(wName, remotePath, wConfig)
 	if wrapErr != nil && wrapErr != fs.ErrorIsFile {
-		return nil, errors.Wrapf(wrapErr, "failed to make remote %q to wrap", remotePath)
+		return nil, errors.Wrapf(wrapErr, "failed to make remote %s:%s to wrap", wName, remotePath)
 	}
 	var fsErr error
 	fs.Debugf(name, "wrapped %v:%v at root %v", wrappedFs.Name(), wrappedFs.Root(), rpath)
@@ -385,7 +390,6 @@ func NewFs(name, rootPath string, m configmap.Mapper) (fs.Fs, error) {
 		cleanupChan:      make(chan bool, 1),
 		notifiedRemotes:  make(map[string]bool),
 	}
-	cache.PinUntilFinalized(f.Fs, f)
 	f.rateLimiter = rate.NewLimiter(rate.Limit(float64(opt.Rps)), opt.TotalWorkers)
 
 	f.plexConnector = &plexConnector{}
@@ -1698,20 +1702,17 @@ func (f *Fs) Hashes() hash.Set {
 	return f.Fs.Hashes()
 }
 
-// Purge all files in the directory
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	if dir == "" {
-		// FIXME this isn't quite right as it should purge the dir prefix
-		fs.Infof(f, "purging cache")
-		f.cache.Purge()
-	}
+// Purge all files in the root and the root directory
+func (f *Fs) Purge(ctx context.Context) error {
+	fs.Infof(f, "purging cache")
+	f.cache.Purge()
 
 	do := f.Fs.Features().Purge
 	if do == nil {
-		return fs.ErrorCantPurge
+		return nil
 	}
 
-	err := do(ctx, dir)
+	err := do(ctx)
 	if err != nil {
 		return err
 	}

backend/cache/cache_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 // +build !race
 
 package cache_test
@@ -16,6 +16,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"runtime/debug"
 	"strings"
 	"testing"
@@ -30,10 +31,13 @@ import (
 	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/object"
+	"github.com/rclone/rclone/fs/rc"
 	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/testy"
 	"github.com/rclone/rclone/lib/random"
+	"github.com/rclone/rclone/vfs"
 	"github.com/rclone/rclone/vfs/vfsflags"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
@@ -49,7 +53,9 @@ const (
 
 var (
 	remoteName                  string
+	mountDir                    string
 	uploadDir                   string
+	useMount                    bool
 	runInstance                 *run
 	errNotSupported             = errors.New("not supported")
 	decryptedToEncryptedRemotes = map[string]string{
@@ -85,7 +91,9 @@ var (
 
 func init() {
 	goflag.StringVar(&remoteName, "remote-internal", "TestInternalCache", "Remote to test with, defaults to local filesystem")
+	goflag.StringVar(&mountDir, "mount-dir-internal", "", "")
 	goflag.StringVar(&uploadDir, "upload-dir-internal", "", "")
+	goflag.BoolVar(&useMount, "cache-use-mount", false, "Test only with mount")
 }
 
 // TestMain drives the tests
@@ -93,7 +101,7 @@ func TestMain(m *testing.M) {
 	goflag.Parse()
 	var rc int
 
-	log.Printf("Running with the following params: \n remote: %v", remoteName)
+	log.Printf("Running with the following params: \n remote: %v, \n mount: %v", remoteName, useMount)
 	runInstance = newRun()
 	rc = m.Run()
 	os.Exit(rc)
@@ -266,6 +274,31 @@ func TestInternalObjNotFound(t *testing.T) {
 	require.Nil(t, obj)
 }
 
+func TestInternalRemoteWrittenFileFoundInMount(t *testing.T) {
+	if !runInstance.useMount {
+		t.Skip("test needs mount mode")
+	}
+	id := fmt.Sprintf("tirwffim%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+
+	var testData []byte
+	if runInstance.rootIsCrypt {
+		testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64)
+		require.NoError(t, err)
+	} else {
+		testData = []byte("test content")
+	}
+
+	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test"), testData)
+	data, err := runInstance.readDataFromRemote(t, rootFs, "test", 0, int64(len([]byte("test content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, "test content", string(data))
+}
+
 func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	testy.SkipUnreliable(t)
 	id := fmt.Sprintf("ticwcm%v", time.Now().Unix())
@@ -661,6 +694,79 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
 }
 
+func TestInternalChangeSeenAfterRc(t *testing.T) {
+	cacheExpire := rc.Calls.Get("cache/expire")
+	assert.NotNil(t, cacheExpire)
+
+	id := fmt.Sprintf("ticsarc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	if !runInstance.useMount {
+		t.Skipf("needs mount")
+	}
+	if !runInstance.wrappedIsExternal {
+		t.Skipf("needs drive")
+	}
+
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+	chunkSize := cfs.ChunkSize()
+
+	// create some rand test data
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
+	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
+
+	// update in the wrapped fs
+	o, err := cfs.UnWrap().NewObject(context.Background(), runInstance.encryptRemoteIfNeeded(t, "data.bin"))
+	require.NoError(t, err)
+	wrappedTime := time.Now().Add(-1 * time.Hour)
+	err = o.SetModTime(context.Background(), wrappedTime)
+	require.NoError(t, err)
+
+	// get a new instance from the cache
+	co, err := rootFs.NewObject(context.Background(), "data.bin")
+	require.NoError(t, err)
+	require.NotEqual(t, o.ModTime(context.Background()).String(), co.ModTime(context.Background()).String())
+
+	// Call the rc function
+	m, err := cacheExpire.Fn(context.Background(), rc.Params{"remote": "data.bin"})
+	require.NoError(t, err)
+	require.Contains(t, m, "status")
+	require.Contains(t, m, "message")
+	require.Equal(t, "ok", m["status"])
+	require.Contains(t, m["message"], "cached file cleared")
+
+	// get a new instance from the cache
+	co, err = rootFs.NewObject(context.Background(), "data.bin")
+	require.NoError(t, err)
+	require.Equal(t, wrappedTime.Unix(), co.ModTime(context.Background()).Unix())
+	_, err = runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testData2 := randStringBytes(int(chunkSize))
+	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
+
+	// list should have 1 item only
+	li1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+	require.Len(t, li1, 1)
+
+	// Call the rc function
+	m, err = cacheExpire.Fn(context.Background(), rc.Params{"remote": "/"})
+	require.NoError(t, err)
+	require.Contains(t, m, "status")
+	require.Contains(t, m, "message")
+	require.Equal(t, "ok", m["status"])
+	require.Contains(t, m["message"], "cached directory cleared")
+
+	// list should have 2 items now
+	li2, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+	require.Len(t, li2, 2)
+}
+
 func TestInternalCacheWrites(t *testing.T) {
 	id := "ticw"
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, map[string]string{"writes": "true"})
@@ -808,9 +914,15 @@ func TestInternalBug2117(t *testing.T) {
 type run struct {
 	okDiff            time.Duration
 	runDefaultCfgMap  configmap.Simple
+	mntDir            string
 	tmpUploadDir      string
+	useMount          bool
+	isMounted         bool
 	rootIsCrypt       bool
 	wrappedIsExternal bool
+	unmountFn         func() error
+	unmountRes        chan error
+	vfs               *vfs.VFS
 	tempFiles         []*os.File
 	dbPath            string
 	chunkPath         string
@@ -820,7 +932,9 @@ type run struct {
 func newRun() *run {
 	var err error
 	r := &run{
-		okDiff: time.Second * 9, // really big diff here but the build machines seem to be slow. need a different way for this
+		okDiff:    time.Second * 9, // really big diff here but the build machines seem to be slow. need a different way for this
+		useMount:  useMount,
+		isMounted: false,
 	}
 
 	// Read in all the defaults for all the options
@@ -833,6 +947,32 @@ func newRun() *run {
 		r.runDefaultCfgMap.Set(option.Name, fmt.Sprint(option.Default))
 	}
 
+	if mountDir == "" {
+		if runtime.GOOS != "windows" {
+			r.mntDir, err = ioutil.TempDir("", "rclonecache-mount")
+			if err != nil {
+				log.Fatalf("Failed to create mount dir: %v", err)
+				return nil
+			}
+		} else {
+			// Find a free drive letter
+			drive := ""
+			for letter := 'E'; letter <= 'Z'; letter++ {
+				drive = string(letter) + ":"
+				_, err := os.Stat(drive + "\\")
+				if os.IsNotExist(err) {
+					goto found
+				}
+			}
+			log.Print("Couldn't find free drive letter for test")
+		found:
+			r.mntDir = drive
+		}
+	} else {
+		r.mntDir = mountDir
+	}
+	log.Printf("Mount Dir: %v", r.mntDir)
+
 	if uploadDir == "" {
 		r.tmpUploadDir, err = ioutil.TempDir("", "rclonecache-tmp")
 		if err != nil {
@@ -946,21 +1086,33 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 	}
 
 	if purge {
-		_ = f.Features().Purge(context.Background(), "")
+		_ = f.Features().Purge(context.Background())
 		require.NoError(t, err)
 	}
 	err = f.Mkdir(context.Background(), "")
 	require.NoError(t, err)
+	if r.useMount && !r.isMounted {
+		r.mountFs(t, f)
+	}
+
 	return f, boltDb
 }
 
 func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
-	err := f.Features().Purge(context.Background(), "")
+	if r.useMount && r.isMounted {
+		r.unmountFs(t, f)
+	}
+
+	err := f.Features().Purge(context.Background())
 	require.NoError(t, err)
 	cfs, err := r.getCacheFs(f)
 	require.NoError(t, err)
 	cfs.StopBackgroundRunners()
 
+	if r.useMount && runtime.GOOS != "windows" {
+		err = os.RemoveAll(r.mntDir)
+		require.NoError(t, err)
+	}
 	err = os.RemoveAll(r.tmpUploadDir)
 	require.NoError(t, err)
 
@@ -1000,11 +1152,37 @@ func (r *run) writeObjectString(t *testing.T, f fs.Fs, remote, content string) f
 }
 
 func (r *run) writeRemoteBytes(t *testing.T, f fs.Fs, remote string, data []byte) {
-	r.writeObjectBytes(t, f, remote, data)
+	var err error
+
+	if r.useMount {
+		err = r.retryBlock(func() error {
+			return ioutil.WriteFile(path.Join(r.mntDir, remote), data, 0600)
+		}, 3, time.Second*3)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else {
+		r.writeObjectBytes(t, f, remote, data)
+	}
 }
 
 func (r *run) writeRemoteReader(t *testing.T, f fs.Fs, remote string, in io.ReadCloser) {
-	r.writeObjectReader(t, f, remote, in)
+	defer func() {
+		_ = in.Close()
+	}()
+
+	if r.useMount {
+		out, err := os.Create(path.Join(r.mntDir, remote))
+		require.NoError(t, err)
+		defer func() {
+			_ = out.Close()
+		}()
+
+		_, err = io.Copy(out, in)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else {
+		r.writeObjectReader(t, f, remote, in)
+	}
 }
 
 func (r *run) writeObjectBytes(t *testing.T, f fs.Fs, remote string, data []byte) fs.Object {
@@ -1021,6 +1199,10 @@ func (r *run) writeObjectReader(t *testing.T, f fs.Fs, remote string, in io.Read
 	objInfo := object.NewStaticObjectInfo(remote, modTime, -1, true, nil, f)
 	obj, err := f.Put(context.Background(), in, objInfo)
 	require.NoError(t, err)
+	if r.useMount {
+		r.vfs.WaitForWriters(10 * time.Second)
+	}
+
 	return obj
 }
 
@@ -1028,16 +1210,26 @@ func (r *run) updateObjectRemote(t *testing.T, f fs.Fs, remote string, data1 []b
 	var err error
 	var obj fs.Object
 
-	in1 := bytes.NewReader(data1)
-	in2 := bytes.NewReader(data2)
-	objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
-	objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
+	if r.useMount {
+		err = ioutil.WriteFile(path.Join(r.mntDir, remote), data1, 0600)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+		err = ioutil.WriteFile(path.Join(r.mntDir, remote), data2, 0600)
+		require.NoError(t, err)
+		r.vfs.WaitForWriters(10 * time.Second)
+		obj, err = f.NewObject(context.Background(), remote)
+	} else {
+		in1 := bytes.NewReader(data1)
+		in2 := bytes.NewReader(data2)
+		objInfo1 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data1)), true, nil, f)
+		objInfo2 := object.NewStaticObjectInfo(remote, time.Now(), int64(len(data2)), true, nil, f)
 
-	obj, err = f.Put(context.Background(), in1, objInfo1)
-	require.NoError(t, err)
-	obj, err = f.NewObject(context.Background(), remote)
-	require.NoError(t, err)
-	err = obj.Update(context.Background(), in2, objInfo2)
+		obj, err = f.Put(context.Background(), in1, objInfo1)
+		require.NoError(t, err)
+		obj, err = f.NewObject(context.Background(), remote)
+		require.NoError(t, err)
+		err = obj.Update(context.Background(), in2, objInfo2)
+	}
 	require.NoError(t, err)
 
 	return obj
@@ -1047,12 +1239,30 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 	size := end - offset
 	checkSample := make([]byte, size)
 
-	co, err := f.NewObject(context.Background(), remote)
-	if err != nil {
-		return checkSample, err
+	if r.useMount {
+		f, err := os.Open(path.Join(r.mntDir, remote))
+		defer func() {
+			_ = f.Close()
+		}()
+		if err != nil {
+			return checkSample, err
+		}
+		_, _ = f.Seek(offset, io.SeekStart)
+		totalRead, err := io.ReadFull(f, checkSample)
+		checkSample = checkSample[:totalRead]
+		if err == io.EOF || err == io.ErrUnexpectedEOF {
+			err = nil
+		}
+		if err != nil {
+			return checkSample, err
+		}
+	} else {
+		co, err := f.NewObject(context.Background(), remote)
+		if err != nil {
+			return checkSample, err
+		}
+		checkSample = r.readDataFromObj(t, co, offset, end, noLengthCheck)
 	}
-	checkSample = r.readDataFromObj(t, co, offset, end, noLengthCheck)
-
 	if !noLengthCheck && size != int64(len(checkSample)) {
 		return checkSample, errors.Errorf("read size doesn't match expected: %v <> %v", len(checkSample), size)
 	}
@@ -1075,19 +1285,28 @@ func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLe
 }
 
 func (r *run) mkdir(t *testing.T, f fs.Fs, remote string) {
-	err := f.Mkdir(context.Background(), remote)
+	var err error
+	if r.useMount {
+		err = os.Mkdir(path.Join(r.mntDir, remote), 0700)
+	} else {
+		err = f.Mkdir(context.Background(), remote)
+	}
 	require.NoError(t, err)
 }
 
 func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 	var err error
 
-	var obj fs.Object
-	obj, err = f.NewObject(context.Background(), remote)
-	if err != nil {
-		err = f.Rmdir(context.Background(), remote)
+	if r.useMount {
+		err = os.Remove(path.Join(r.mntDir, remote))
 	} else {
-		err = obj.Remove(context.Background())
+		var obj fs.Object
+		obj, err = f.NewObject(context.Background(), remote)
+		if err != nil {
+			err = f.Rmdir(context.Background(), remote)
+		} else {
+			err = obj.Remove(context.Background())
+		}
 	}
 
 	return err
@@ -1096,10 +1315,18 @@ func (r *run) rm(t *testing.T, f fs.Fs, remote string) error {
 func (r *run) list(t *testing.T, f fs.Fs, remote string) ([]interface{}, error) {
 	var err error
 	var l []interface{}
-	var list fs.DirEntries
-	list, err = f.List(context.Background(), remote)
-	for _, ll := range list {
-		l = append(l, ll)
+	if r.useMount {
+		var list []os.FileInfo
+		list, err = ioutil.ReadDir(path.Join(r.mntDir, remote))
+		for _, ll := range list {
+			l = append(l, ll)
+		}
+	} else {
+		var list fs.DirEntries
+		list, err = f.List(context.Background(), remote)
+		for _, ll := range list {
+			l = append(l, ll)
+		}
 	}
 	return l, err
 }
@@ -1128,7 +1355,13 @@ func (r *run) copyFile(t *testing.T, f fs.Fs, src, dst string) error {
 func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 
-	if rootFs.Features().DirMove != nil {
+	if runInstance.useMount {
+		err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst))
+		if err != nil {
+			return err
+		}
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else if rootFs.Features().DirMove != nil {
 		err = rootFs.Features().DirMove(context.Background(), rootFs, src, dst)
 		if err != nil {
 			return err
@@ -1144,7 +1377,13 @@ func (r *run) dirMove(t *testing.T, rootFs fs.Fs, src, dst string) error {
 func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 
-	if rootFs.Features().Move != nil {
+	if runInstance.useMount {
+		err = os.Rename(path.Join(runInstance.mntDir, src), path.Join(runInstance.mntDir, dst))
+		if err != nil {
+			return err
+		}
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else if rootFs.Features().Move != nil {
 		obj1, err := rootFs.NewObject(context.Background(), src)
 		if err != nil {
 			return err
@@ -1164,7 +1403,13 @@ func (r *run) move(t *testing.T, rootFs fs.Fs, src, dst string) error {
 func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
 	var err error
 
-	if rootFs.Features().Copy != nil {
+	if r.useMount {
+		err = r.copyFile(t, rootFs, path.Join(r.mntDir, src), path.Join(r.mntDir, dst))
+		if err != nil {
+			return err
+		}
+		r.vfs.WaitForWriters(10 * time.Second)
+	} else if rootFs.Features().Copy != nil {
 		obj, err := rootFs.NewObject(context.Background(), src)
 		if err != nil {
 			return err
@@ -1184,6 +1429,13 @@ func (r *run) copy(t *testing.T, rootFs fs.Fs, src, dst string) error {
 func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error) {
 	var err error
 
+	if r.useMount {
+		fi, err := os.Stat(path.Join(runInstance.mntDir, src))
+		if err != nil {
+			return time.Time{}, err
+		}
+		return fi.ModTime(), nil
+	}
 	obj1, err := rootFs.NewObject(context.Background(), src)
 	if err != nil {
 		return time.Time{}, err
@@ -1194,6 +1446,13 @@ func (r *run) modTime(t *testing.T, rootFs fs.Fs, src string) (time.Time, error)
 func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
 	var err error
 
+	if r.useMount {
+		fi, err := os.Stat(path.Join(runInstance.mntDir, src))
+		if err != nil {
+			return int64(0), err
+		}
+		return fi.Size(), nil
+	}
 	obj1, err := rootFs.NewObject(context.Background(), src)
 	if err != nil {
 		return int64(0), err
@@ -1204,15 +1463,28 @@ func (r *run) size(t *testing.T, rootFs fs.Fs, src string) (int64, error) {
 func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) error {
 	var err error
 
-	var obj1 fs.Object
-	obj1, err = rootFs.NewObject(context.Background(), src)
-	if err != nil {
-		return err
+	if r.useMount {
+		var f *os.File
+		f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			return err
+		}
+		defer func() {
+			_ = f.Close()
+			r.vfs.WaitForWriters(10 * time.Second)
+		}()
+		_, err = f.WriteString(data + append)
+	} else {
+		var obj1 fs.Object
+		obj1, err = rootFs.NewObject(context.Background(), src)
+		if err != nil {
+			return err
+		}
+		data1 := []byte(data + append)
+		r := bytes.NewReader(data1)
+		objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
+		err = obj1.Update(context.Background(), r, objInfo1)
 	}
-	data1 := []byte(data + append)
-	reader := bytes.NewReader(data1)
-	objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
-	err = obj1.Update(context.Background(), reader, objInfo1)
 
 	return err
 }

backend/cache/cache_mount_other_test.go

@@ -0,0 +1,21 @@
+// +build !linux !go1.13
+// +build !darwin !go1.13
+// +build !freebsd !go1.13
+// +build !windows
+// +build !race
+
+package cache_test
+
+import (
+	"testing"
+
+	"github.com/rclone/rclone/fs"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	panic("mountFs not defined for this platform")
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	panic("unmountFs not defined for this platform")
+}

backend/cache/cache_mount_unix_test.go

@@ -0,0 +1,79 @@
+// +build linux,go1.13 darwin,go1.13 freebsd,go1.13
+// +build !race
+
+package cache_test
+
+import (
+	"os"
+	"testing"
+	"time"
+
+	"bazil.org/fuse"
+	fusefs "bazil.org/fuse/fs"
+	"github.com/rclone/rclone/cmd/mount"
+	"github.com/rclone/rclone/cmd/mountlib"
+	"github.com/rclone/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	device := f.Name() + ":" + f.Root()
+	var options = []fuse.MountOption{
+		fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)),
+		fuse.Subtype("rclone"),
+		fuse.FSName(device), fuse.VolumeName(device),
+		fuse.NoAppleDouble(),
+		fuse.NoAppleXattr(),
+		//fuse.AllowOther(),
+	}
+	err := os.MkdirAll(r.mntDir, os.ModePerm)
+	require.NoError(t, err)
+	c, err := fuse.Mount(r.mntDir, options...)
+	require.NoError(t, err)
+	filesys := mount.NewFS(f)
+	server := fusefs.New(c, nil)
+
+	// Serve the mount point in the background returning error to errChan
+	r.unmountRes = make(chan error, 1)
+	go func() {
+		err := server.Serve(filesys)
+		closeErr := c.Close()
+		if err == nil {
+			err = closeErr
+		}
+		r.unmountRes <- err
+	}()
+
+	// check if the mount process has an error to report
+	<-c.Ready
+	require.NoError(t, c.MountError)
+
+	r.unmountFn = func() error {
+		// Shutdown the VFS
+		filesys.VFS.Shutdown()
+		return fuse.Unmount(r.mntDir)
+	}
+
+	r.vfs = filesys.VFS
+	r.isMounted = true
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	var err error
+
+	for i := 0; i < 4; i++ {
+		err = r.unmountFn()
+		if err != nil {
+			//log.Printf("signal to umount failed - retrying: %v", err)
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(t, err)
+	err = <-r.unmountRes
+	require.NoError(t, err)
+	err = r.vfs.CleanUp()
+	require.NoError(t, err)
+	r.isMounted = false
+}

backend/cache/cache_mount_windows_test.go

@@ -0,0 +1,125 @@
+// +build windows
+// +build !race
+
+package cache_test
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/billziss-gh/cgofuse/fuse"
+	"github.com/pkg/errors"
+	"github.com/rclone/rclone/cmd/cmount"
+	"github.com/rclone/rclone/cmd/mountlib"
+	"github.com/rclone/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+// waitFor runs fn() until it returns true or the timeout expires
+func waitFor(fn func() bool) (ok bool) {
+	const totalWait = 10 * time.Second
+	const individualWait = 10 * time.Millisecond
+	for i := 0; i < int(totalWait/individualWait); i++ {
+		ok = fn()
+		if ok {
+			return ok
+		}
+		time.Sleep(individualWait)
+	}
+	return false
+}
+
+func (r *run) mountFs(t *testing.T, f fs.Fs) {
+	// FIXME implement cmount
+	t.Skip("windows not supported yet")
+
+	device := f.Name() + ":" + f.Root()
+	options := []string{
+		"-o", "fsname=" + device,
+		"-o", "subtype=rclone",
+		"-o", fmt.Sprintf("max_readahead=%d", mountlib.MaxReadAhead),
+		"-o", "uid=-1",
+		"-o", "gid=-1",
+		"-o", "allow_other",
+		// This causes FUSE to supply O_TRUNC with the Open
+		// call which is more efficient for cmount.  However
+		// it does not work with cgofuse on Windows with
+		// WinFSP so cmount must work with or without it.
+		"-o", "atomic_o_trunc",
+		"--FileSystemName=rclone",
+	}
+
+	fsys := cmount.NewFS(f)
+	host := fuse.NewFileSystemHost(fsys)
+
+	// Serve the mount point in the background returning error to errChan
+	r.unmountRes = make(chan error, 1)
+	go func() {
+		var err error
+		ok := host.Mount(r.mntDir, options)
+		if !ok {
+			err = errors.New("mount failed")
+		}
+		r.unmountRes <- err
+	}()
+
+	// unmount
+	r.unmountFn = func() error {
+		// Shutdown the VFS
+		fsys.VFS.Shutdown()
+		if host.Unmount() {
+			if !waitFor(func() bool {
+				_, err := os.Stat(r.mntDir)
+				return err != nil
+			}) {
+				t.Fatalf("mountpoint %q didn't disappear after unmount - continuing anyway", r.mntDir)
+			}
+			return nil
+		}
+		return errors.New("host unmount failed")
+	}
+
+	// Wait for the filesystem to become ready, checking the file
+	// system didn't blow up before starting
+	select {
+	case err := <-r.unmountRes:
+		require.NoError(t, err)
+	case <-time.After(time.Second * 3):
+	}
+
+	// Wait for the mount point to be available on Windows
+	// On Windows the Init signal comes slightly before the mount is ready
+	if !waitFor(func() bool {
+		_, err := os.Stat(r.mntDir)
+		return err == nil
+	}) {
+		t.Errorf("mountpoint %q didn't became available on mount", r.mntDir)
+	}
+
+	r.vfs = fsys.VFS
+	r.isMounted = true
+}
+
+func (r *run) unmountFs(t *testing.T, f fs.Fs) {
+	// FIXME implement cmount
+	t.Skip("windows not supported yet")
+	var err error
+
+	for i := 0; i < 4; i++ {
+		err = r.unmountFn()
+		if err != nil {
+			//log.Printf("signal to umount failed - retrying: %v", err)
+			time.Sleep(3 * time.Second)
+			continue
+		}
+		break
+	}
+	require.NoError(t, err)
+	err = <-r.unmountRes
+	require.NoError(t, err)
+	err = r.vfs.CleanUp()
+	require.NoError(t, err)
+	r.isMounted = false
+}

backend/cache/cache_test.go

@@ -1,6 +1,6 @@
 // Test Cache filesystem interface
 
-// +build !plan9,!js
+// +build !plan9
 // +build !race
 
 package cache_test

backend/cache/cache_unsupported.go

@@ -1,6 +1,6 @@
 // Build for cache for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 js
+// +build plan9
 
 package cache

backend/cache/cache_upload_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 // +build !race
 
 package cache_test

backend/cache/directory.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 

backend/cache/handle.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 

backend/cache/object.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 

backend/cache/plex.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 

backend/cache/storage_memory.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 

backend/cache/storage_persistent.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cache
 

backend/chunker/chunker.go

@@ -24,7 +24,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/fspath"
@@ -239,18 +238,15 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		return nil, errors.New("can't point remote at itself - check the value of the remote setting")
 	}
 
-	baseName, basePath, err := fspath.Parse(remote)
+	baseInfo, baseName, basePath, baseConfig, err := fs.ConfigFs(remote)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
 	}
-	if baseName != "" {
-		baseName += ":"
-	}
 	// Look for a file first
 	remotePath := fspath.JoinRootPath(basePath, rpath)
-	baseFs, err := cache.Get(baseName + remotePath)
+	baseFs, err := baseInfo.NewFs(baseName, remotePath, baseConfig)
 	if err != fs.ErrorIsFile && err != nil {
-		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", baseName+remotePath)
+		return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", baseName, remotePath)
 	}
 	if !operations.CanServerSideMove(baseFs) {
 		return nil, errors.New("can't use chunker on a backend which doesn't support server side move or copy")
@@ -262,7 +258,6 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		root: rpath,
 		opt:  *opt,
 	}
-	cache.PinUntilFinalized(f.base, f)
 	f.dirSort = true // processEntries requires that meta Objects prerun data chunks atm.
 
 	if err := f.configure(opt.NameFormat, opt.MetaFormat, opt.HashType); err != nil {
@@ -276,7 +271,7 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// (yet can't satisfy fstest.CheckListing, will ignore)
 	if err == nil && !f.useMeta && strings.Contains(rpath, "/") {
 		firstChunkPath := f.makeChunkName(remotePath, 0, "", "")
-		_, testErr := cache.Get(baseName + firstChunkPath)
+		_, testErr := baseInfo.NewFs(baseName, firstChunkPath, baseConfig)
 		if testErr == fs.ErrorIsFile {
 			err = testErr
 		}
@@ -296,8 +291,6 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		ServerSideAcrossConfigs: true,
 	}).Fill(f).Mask(baseFs).WrapsFs(f, baseFs)
 
-	f.features.Disable("ListR") // Recursive listing may cause chunker skip files
-
 	return f, err
 }
 
@@ -960,8 +953,6 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, remote st
 		}
 		info := f.wrapInfo(src, chunkRemote, size)
 
-		// Refill chunkLimit and let basePut repeatedly call chunkingReader.Read()
-		c.chunkLimit = c.chunkSize
 		// TODO: handle range/limit options
 		chunk, errChunk := basePut(ctx, wrapIn, info, options...)
 		if errChunk != nil {
@@ -1170,14 +1161,10 @@ func (c *chunkingReader) updateHashes() {
 func (c *chunkingReader) Read(buf []byte) (bytesRead int, err error) {
 	if c.chunkLimit <= 0 {
 		// Chunk complete - switch to next one.
-		// Note #1:
 		// We might not get here because some remotes (eg. box multi-uploader)
 		// read the specified size exactly and skip the concluding EOF Read.
 		// Then a check in the put loop will kick in.
-		// Note #2:
-		// The crypt backend after receiving EOF here will call Read again
-		// and we must insist on returning EOF, so we postpone refilling
-		// chunkLimit to the main loop.
+		c.chunkLimit = c.chunkSize
 		return 0, io.EOF
 	}
 	if int64(len(buf)) > c.chunkLimit {
@@ -1346,7 +1333,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return f.base.Rmdir(ctx, dir)
 }
 
-// Purge all files in the directory
+// Purge all files in the root and the root directory
 //
 // Implement this if you have a way of deleting all the files
 // quicker than just running Remove() on the result of List()
@@ -1357,12 +1344,12 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 // As a result it removes not only composite chunker files with their
 // active chunks but also all hidden temporary chunks in the directory.
 //
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
 	do := f.base.Features().Purge
 	if do == nil {
 		return fs.ErrorCantPurge
 	}
-	return do(ctx, dir)
+	return do(ctx)
 }
 
 // Remove an object (chunks and metadata, if any)

backend/crypt/cipher.go

@@ -147,7 +147,7 @@ func newCipher(mode NameEncryptionMode, password, salt string, dirNameEncrypt bo
 // If salt is "" we use a fixed salt just to make attackers lives
 // slighty harder than using no salt.
 //
-// Note that empty password makes all 0x00 keys which is used in the
+// Note that empty passsword makes all 0x00 keys which is used in the
 // tests.
 func (c *Cipher) Key(password, salt string) (err error) {
 	const keySize = len(c.dataKey) + len(c.nameKey) + len(c.nameTweak)
@@ -633,8 +633,11 @@ func (fh *encrypter) Read(p []byte) (n int, err error) {
 		}
 		// possibly err != nil here, but we will process the
 		// data and the next call to ReadFull will return 0, err
+		// Write nonce to start of block
+		copy(fh.buf, fh.nonce[:])
 		// Encrypt the block using the nonce
-		secretbox.Seal(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+		block := fh.buf
+		secretbox.Seal(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 		fh.bufIndex = 0
 		fh.bufSize = blockHeaderSize + n
 		fh.nonce.increment()
@@ -779,7 +782,8 @@ func (fh *decrypter) fillBuffer() (err error) {
 		return ErrorEncryptedFileBadHeader
 	}
 	// Decrypt the block using the nonce
-	_, ok := secretbox.Open(fh.buf[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
+	block := fh.buf
+	_, ok := secretbox.Open(block[:0], readBuf[:n], fh.nonce.pointer(), &fh.c.dataKey)
 	if !ok {
 		if err != nil {
 			return err // return pending error as it is likely more accurate

backend/crypt/crypt.go

@@ -12,7 +12,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/accounting"
-	"github.com/rclone/rclone/fs/cache"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -159,25 +158,24 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	if strings.HasPrefix(remote, name+":") {
 		return nil, errors.New("can't point crypt remote at itself - check the value of the remote setting")
 	}
-	// Make sure to remove trailing . referring to the current dir
+	wInfo, wName, wPath, wConfig, err := fs.ConfigFs(remote)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to parse remote %q to wrap", remote)
+	}
+	// Make sure to remove trailing . reffering to the current dir
 	if path.Base(rpath) == "." {
 		rpath = strings.TrimSuffix(rpath, ".")
 	}
 	// Look for a file first
-	var wrappedFs fs.Fs
-	if rpath == "" {
-		wrappedFs, err = cache.Get(remote)
-	} else {
-		remotePath := fspath.JoinRootPath(remote, cipher.EncryptFileName(rpath))
-		wrappedFs, err = cache.Get(remotePath)
-		// if that didn't produce a file, look for a directory
-		if err != fs.ErrorIsFile {
-			remotePath = fspath.JoinRootPath(remote, cipher.EncryptDirName(rpath))
-			wrappedFs, err = cache.Get(remotePath)
-		}
+	remotePath := fspath.JoinRootPath(wPath, cipher.EncryptFileName(rpath))
+	wrappedFs, err := wInfo.NewFs(wName, remotePath, wConfig)
+	// if that didn't produce a file, look for a directory
+	if err != fs.ErrorIsFile {
+		remotePath = fspath.JoinRootPath(wPath, cipher.EncryptDirName(rpath))
+		wrappedFs, err = wInfo.NewFs(wName, remotePath, wConfig)
 	}
 	if err != fs.ErrorIsFile && err != nil {
-		return nil, errors.Wrapf(err, "failed to make remote %q to wrap", remote)
+		return nil, errors.Wrapf(err, "failed to make remote %s:%q to wrap", wName, remotePath)
 	}
 	f := &Fs{
 		Fs:     wrappedFs,
@@ -186,7 +184,6 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 		opt:    *opt,
 		cipher: cipher,
 	}
-	cache.PinUntilFinalized(f.Fs, f)
 	// the features here are ones we could support, and they are
 	// ANDed with the ones from wrappedFs
 	f.features = (&fs.Features{
@@ -430,18 +427,18 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return f.Fs.Rmdir(ctx, f.cipher.EncryptDirName(dir))
 }
 
-// Purge all files in the directory specified
+// Purge all files in the root and the root directory
 //
 // Implement this if you have a way of deleting all the files
 // quicker than just running Remove() on the result of List()
 //
 // Return an error if it doesn't exist
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
 	do := f.Fs.Features().Purge
 	if do == nil {
 		return fs.ErrorCantPurge
 	}
-	return do(ctx, f.cipher.EncryptDirName(dir))
+	return do(ctx)
 }
 
 // Copy src to this remote using server side copy operations.

backend/crypt/crypt_internal_test.go

@@ -87,7 +87,7 @@ func testObjectInfo(t *testing.T, f *Fs, wrap bool) {
 	}
 
 	// wrap the object in a crypt for upload using the nonce we
-	// saved from the encrypter
+	// saved from the encryptor
 	src := f.newObjectInfo(oi, nonce)
 
 	// Test ObjectInfo methods

backend/drive/drive.go

@@ -17,6 +17,7 @@ import (
 	"log"
 	"mime"
 	"net/http"
+	"net/url"
 	"path"
 	"sort"
 	"strconv"
@@ -35,9 +36,7 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/fshttp"
-	"github.com/rclone/rclone/fs/fspath"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
 	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
@@ -58,7 +57,6 @@ const (
 	rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg"
 	driveFolderType             = "application/vnd.google-apps.folder"
 	shortcutMimeType            = "application/vnd.google-apps.shortcut"
-	shortcutMimeTypeDangling    = "application/vnd.google-apps.shortcut.dangling" // synthetic mime type for internal use
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
 	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
@@ -70,7 +68,7 @@ const (
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	minChunkSize     = 256 * fs.KibiByte
 	defaultChunkSize = 8 * fs.MebiByte
-	partialFields    = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
+	partialFields    = "id,name,size,md5Checksum,trashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails"
 	listRGrouping    = 50   // number of IDs to search at once when using ListR
 	listRInputBuffer = 1000 // size of input buffer when using ListR
 )
@@ -158,17 +156,6 @@ func driveScopesContainsAppFolder(scopes []string) bool {
 	return false
 }
 
-func driveOAuthOptions() []fs.Option {
-	opts := []fs.Option{}
-	for _, opt := range oauthutil.SharedOptions {
-		if opt.Name == config.ConfigClientID {
-			opt.Help = "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance."
-		}
-		opts = append(opts, opt)
-	}
-	return opts
-}
-
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -204,7 +191,13 @@ func init() {
 				log.Fatalf("Failed to configure team drive: %v", err)
 			}
 		},
-		Options: append(driveOAuthOptions(), []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Google Application Client Id\nSetting your own is recommended.\nSee https://rclone.org/drive/#making-your-own-client-id for how to create your own.\nIf you leave this blank, it will use an internal key which is low performance.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Google Application Client Secret\nSetting your own is recommended.",
+		}, {
 			Name: "scope",
 			Help: "Scope that rclone should use when requesting access from drive.",
 			Examples: []fs.OptionExample{{
@@ -230,6 +223,9 @@ Leave blank normally.
 
 Fill in to access "Computers" folders (see docs), or for rclone to use
 a non root folder as its starting point.
+
+Note that if this is blank, the first time rclone runs it will fill it
+in with the ID of the root folder.
 `,
 		}, {
 			Name: "service_account_file",
@@ -292,11 +288,6 @@ commands (copy, sync, etc), and with all other commands too.`,
 			Default:  false,
 			Help:     "Only show files that are in the trash.\nThis will show trashed files in their original directory structure.",
 			Advanced: true,
-		}, {
-			Name:     "starred_only",
-			Default:  false,
-			Help:     "Only show files that are starred.",
-			Advanced: true,
 		}, {
 			Name:     "formats",
 			Default:  "",
@@ -358,15 +349,27 @@ date is used.`,
 			Help:     "Size of listing chunk 100-1000. 0 to disable.",
 			Advanced: true,
 		}, {
-			Name:     "impersonate",
-			Default:  "",
-			Help:     `Impersonate this user when using a service account.`,
+			Name:    "impersonate",
+			Default: "",
+			Help: `Impersonate this user when using a service account.
+
+Note that if this is used then "root_folder_id" will be ignored.
+`,
 			Advanced: true,
 		}, {
 			Name:    "alternate_export",
 			Default: false,
-			Help:    "Deprecated: no longer needed",
-			Hide:    fs.OptionHideBoth,
+			Help: `Use alternate export URLs for google documents export.,
+
+If this option is set this instructs rclone to use an alternate set of
+export URLs for drive documents.  Users have reported that the
+official export URLs can't export large documents, whereas these
+unofficial ones can.
+
+See rclone issue [#2243](https://github.com/rclone/rclone/issues/2243) for background,
+[this google drive issue](https://issuetracker.google.com/issues/36761333) and
+[this helpful post](https://www.labnol.org/internet/direct-links-for-google-drive/28356/).`,
+			Advanced: true,
 		}, {
 			Name:     "upload_cutoff",
 			Default:  defaultChunkSize,
@@ -471,21 +474,6 @@ Note that this detection is relying on error message strings which
 Google don't document so it may break in the future.
 
 See: https://github.com/rclone/rclone/issues/3857
-`,
-			Advanced: true,
-		}, {
-			Name:    "stop_on_download_limit",
-			Default: false,
-			Help: `Make download limit errors be fatal
-
-At the time of writing it is only possible to download 10TB of data from
-Google Drive a day (this is an undocumented limit). When this limit is
-reached Google Drive produces a slightly different error message. When
-this flag is set it causes these errors to be fatal.  These will stop
-the in-progress sync.
-
-Note that this detection is relying on error message strings which
-Google don't document so it may break in the future.
 `,
 			Advanced: true,
 		}, {
@@ -505,7 +493,7 @@ If this flag is set then rclone will ignore shortcut files completely.
 			// Encode invalid UTF-8 bytes as json doesn't handle them properly.
 			// Don't encode / as it's a valid name character in drive.
 			Default: encoder.EncodeInvalidUtf8,
-		}}...),
+		}},
 	})
 
 	// register duplicate MIME types first
@@ -535,7 +523,6 @@ type Options struct {
 	SkipChecksumGphotos       bool                 `config:"skip_checksum_gphotos"`
 	SharedWithMe              bool                 `config:"shared_with_me"`
 	TrashedOnly               bool                 `config:"trashed_only"`
-	StarredOnly               bool                 `config:"starred_only"`
 	Extensions                string               `config:"formats"`
 	ExportExtensions          string               `config:"export_formats"`
 	ImportExtensions          string               `config:"import_formats"`
@@ -544,6 +531,7 @@ type Options struct {
 	UseSharedDate             bool                 `config:"use_shared_date"`
 	ListChunk                 int64                `config:"list_chunk"`
 	Impersonate               string               `config:"impersonate"`
+	AlternateExport           bool                 `config:"alternate_export"`
 	UploadCutoff              fs.SizeSuffix        `config:"upload_cutoff"`
 	ChunkSize                 fs.SizeSuffix        `config:"chunk_size"`
 	AcknowledgeAbuse          bool                 `config:"acknowledge_abuse"`
@@ -555,7 +543,6 @@ type Options struct {
 	ServerSideAcrossConfigs   bool                 `config:"server_side_across_configs"`
 	DisableHTTP2              bool                 `config:"disable_http2"`
 	StopOnUploadLimit         bool                 `config:"stop_on_upload_limit"`
-	StopOnDownloadLimit       bool                 `config:"stop_on_download_limit"`
 	SkipShortcuts             bool                 `config:"skip_shortcuts"`
 	Enc                       encoder.MultiEncoder `config:"encoding"`
 }
@@ -655,9 +642,6 @@ func (f *Fs) shouldRetry(err error) (bool, error) {
 					return false, fserrors.FatalError(err)
 				}
 				return true, err
-			} else if f.opt.StopOnDownloadLimit && reason == "downloadQuotaExceeded" {
-				fs.Errorf(f, "Received download limit error: %v", err)
-				return false, fserrors.FatalError(err)
 			} else if f.opt.StopOnUploadLimit && reason == "teamDriveFileLimitExceeded" {
 				fs.Errorf(f, "Received team drive file limit error: %v", err)
 				return false, fserrors.FatalError(err)
@@ -722,7 +706,6 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		}
 		query = append(query, q)
 	}
-
 	// Search with sharedWithMe will always return things listed in "Shared With Me" (without any parents)
 	// We must not filter with parent when we try list "ROOT" with drive-shared-with-me
 	// If we need to list file inside those shared folders, we must search it without sharedWithMe
@@ -734,16 +717,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
 		if parentsQuery.Len() > 1 {
 			_, _ = parentsQuery.WriteString(" or ")
 		}
-		if (f.opt.SharedWithMe || f.opt.StarredOnly) && dirID == f.rootFolderID {
-			if f.opt.SharedWithMe {
-				_, _ = parentsQuery.WriteString("sharedWithMe=true")
-			}
-			if f.opt.StarredOnly {
-				if f.opt.SharedWithMe {
-					_, _ = parentsQuery.WriteString(" and ")
-				}
-				_, _ = parentsQuery.WriteString("starred=true")
-			}
+		if f.opt.SharedWithMe && dirID == f.rootFolderID {
+			_, _ = parentsQuery.WriteString("sharedWithMe=true")
 		} else {
 			_, _ = fmt.Fprintf(parentsQuery, "'%s' in parents", dirID)
 		}
@@ -953,32 +928,55 @@ func configTeamDrive(ctx context.Context, opt *Options, m configmap.Mapper, name
 	if !config.Confirm(false) {
 		return nil
 	}
-	f, err := newFs(name, "", m)
+	client, err := createOAuthClient(opt, name, m)
 	if err != nil {
-		return errors.Wrap(err, "failed to make Fs to list teamdrives")
+		return errors.Wrap(err, "config team drive failed to create oauth client")
+	}
+	svc, err := drive.New(client)
+	if err != nil {
+		return errors.Wrap(err, "config team drive failed to make drive client")
 	}
 	fmt.Printf("Fetching team drive list...\n")
-	teamDrives, err := f.listTeamDrives(ctx)
-	if err != nil {
-		return err
-	}
-	if len(teamDrives) == 0 {
-		fmt.Printf("No team drives found in your account")
-		return nil
-	}
 	var driveIDs, driveNames []string
-	for _, teamDrive := range teamDrives {
-		driveIDs = append(driveIDs, teamDrive.Id)
-		driveNames = append(driveNames, teamDrive.Name)
+	listTeamDrives := svc.Teamdrives.List().PageSize(100)
+	listFailed := false
+	var defaultFs Fs // default Fs with default Options
+	for {
+		var teamDrives *drive.TeamDriveList
+		err = newPacer(opt).Call(func() (bool, error) {
+			teamDrives, err = listTeamDrives.Context(ctx).Do()
+			return defaultFs.shouldRetry(err)
+		})
+		if err != nil {
+			fmt.Printf("Listing team drives failed: %v\n", err)
+			listFailed = true
+			break
+		}
+		for _, drive := range teamDrives.TeamDrives {
+			driveIDs = append(driveIDs, drive.Id)
+			driveNames = append(driveNames, drive.Name)
+		}
+		if teamDrives.NextPageToken == "" {
+			break
+		}
+		listTeamDrives.PageToken(teamDrives.NextPageToken)
+	}
+	var driveID string
+	if !listFailed && len(driveIDs) == 0 {
+		fmt.Printf("No team drives found in your account")
+	} else {
+		driveID = config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true)
 	}
-	driveID := config.Choose("Enter a Team Drive ID", driveIDs, driveNames, true)
 	m.Set("team_drive", driveID)
-	m.Set("root_folder_id", "")
 	opt.TeamDriveID = driveID
-	opt.RootFolderID = ""
 	return nil
 }
 
+// newPacer makes a pacer configured for drive
+func newPacer(opt *Options) *fs.Pacer {
+	return fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst)))
+}
+
 // getClient makes an http client according to the options
 func getClient(opt *Options) *http.Client {
 	t := fshttp.NewTransportCustom(fs.Config, func(t *http.Transport) {
@@ -1061,11 +1059,9 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// newFs partially constructs Fs from the path
-//
-// It constructs a valid Fs but doesn't attempt to figure out whether
-// it is a file or a directory.
-func newFs(name, path string, m configmap.Mapper) (*Fs, error) {
+// NewFs constructs an Fs from the path, container:path
+func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
+	ctx := context.Background()
 	// Parse config into Options struct
 	opt := new(Options)
 	err := configstruct.Set(m, opt)
@@ -1095,7 +1091,7 @@ func newFs(name, path string, m configmap.Mapper) (*Fs, error) {
 		name:         name,
 		root:         root,
 		opt:          *opt,
-		pacer:        fs.NewPacer(pacer.NewGoogleDrive(pacer.MinSleep(opt.PacerMinSleep), pacer.Burst(opt.PacerBurst))),
+		pacer:        newPacer(opt),
 		m:            m,
 		grouping:     listRGrouping,
 		listRmu:      new(sync.Mutex),
@@ -1125,26 +1121,25 @@ func newFs(name, path string, m configmap.Mapper) (*Fs, error) {
 		}
 	}
 
-	return f, nil
-}
-
-// NewFs constructs an Fs from the path, container:path
-func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
-	ctx := context.Background()
-	f, err := newFs(name, path, m)
-	if err != nil {
-		return nil, err
+	// If impersonating warn about root_folder_id if set and unset it
+	//
+	// This is because rclone v1.51 and v1.52 cached root_folder_id when
+	// using impersonate which they shouldn't have done. It is possible
+	// someone is using impersonate and root_folder_id in which case this
+	// breaks their workflow. There isn't an easy way around that.
+	if opt.RootFolderID != "" && opt.RootFolderID != "appDataFolder" && opt.Impersonate != "" {
+		fs.Logf(f, "Ignoring cached root_folder_id when using --drive-impersonate")
+		opt.RootFolderID = ""
 	}
 
-	// Set the root folder ID
-	if f.opt.RootFolderID != "" {
-		// use root_folder ID if set
-		f.rootFolderID = f.opt.RootFolderID
+	// set root folder for a team drive or query the user root folder
+	if opt.RootFolderID != "" {
+		// override root folder if set or cached in the config and not impersonating
+		f.rootFolderID = opt.RootFolderID
 	} else if f.isTeamDrive {
-		// otherwise use team_drive if set
 		f.rootFolderID = f.opt.TeamDriveID
 	} else {
-		// otherwise look up the actual root ID
+		// Look up the root ID and cache it in the config
 		rootID, err := f.getRootID()
 		if err != nil {
 			if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
@@ -1156,24 +1151,27 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 			}
 		}
 		f.rootFolderID = rootID
-		fs.Debugf(f, "root_folder_id = %q - save this in the config to speed up startup", rootID)
+		// Don't cache the root folder ID if impersonating
+		if opt.Impersonate == "" {
+			m.Set("root_folder_id", rootID)
+		}
 	}
 
-	f.dirCache = dircache.New(f.root, f.rootFolderID, f)
+	f.dirCache = dircache.New(root, f.rootFolderID, f)
 
 	// Parse extensions
-	if f.opt.Extensions != "" {
-		if f.opt.ExportExtensions != defaultExportExtensions {
+	if opt.Extensions != "" {
+		if opt.ExportExtensions != defaultExportExtensions {
 			return nil, errors.New("only one of 'formats' and 'export_formats' can be specified")
 		}
-		f.opt.Extensions, f.opt.ExportExtensions = "", f.opt.Extensions
+		opt.Extensions, opt.ExportExtensions = "", opt.Extensions
 	}
-	f.exportExtensions, _, err = parseExtensions(f.opt.ExportExtensions, defaultExportExtensions)
+	f.exportExtensions, _, err = parseExtensions(opt.ExportExtensions, defaultExportExtensions)
 	if err != nil {
 		return nil, err
 	}
 
-	_, f.importMimeTypes, err = parseExtensions(f.opt.ImportExtensions)
+	_, f.importMimeTypes, err = parseExtensions(opt.ImportExtensions)
 	if err != nil {
 		return nil, err
 	}
@@ -1182,7 +1180,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	err = f.dirCache.FindRoot(ctx, false)
 	if err != nil {
 		// Assume it is a file
-		newRoot, remote := dircache.SplitPath(f.root)
+		newRoot, remote := dircache.SplitPath(root)
 		tempF := *f
 		tempF.dirCache = dircache.New(newRoot, f.rootFolderID, &tempF)
 		tempF.root = newRoot
@@ -1273,7 +1271,20 @@ func (f *Fs) newDocumentObject(remote string, info *drive.File, extension, expor
 	if err != nil {
 		return nil, err
 	}
-	url := info.ExportLinks[mediaType]
+	id := actualID(info.Id)
+	url := fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, id, url.QueryEscape(mediaType))
+	if f.opt.AlternateExport {
+		switch info.MimeType {
+		case "application/vnd.google-apps.drawing":
+			url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", id, extension[1:])
+		case "application/vnd.google-apps.document":
+			url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", id, extension[1:])
+		case "application/vnd.google-apps.spreadsheet":
+			url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", id, extension[1:])
+		case "application/vnd.google-apps.presentation":
+			url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", id, extension[1:])
+		}
+	}
 	baseObject := f.newBaseObject(remote+extension, info)
 	baseObject.bytes = -1
 	baseObject.mimeType = exportMimeType
@@ -1345,10 +1356,6 @@ func (f *Fs) newObjectWithExportInfo(
 		// and not from a listing. This is unlikely.
 		fs.Debugf(remote, "Ignoring shortcut as skip shortcuts is set")
 		return nil, fs.ErrorObjectNotFound
-	case info.MimeType == shortcutMimeTypeDangling:
-		// Pretend a dangling shortcut is a regular object
-		// It will error if used, but appear in listings so it can be deleted
-		return f.newRegularObject(remote, info), nil
 	case info.Md5Checksum != "" || info.Size > 0:
 		// If item has MD5 sum or a length it is a file stored on drive
 		return f.newRegularObject(remote, info), nil
@@ -1650,7 +1657,7 @@ func (s listRSlices) Less(i, j int) bool {
 // In each cycle it will read up to grouping entries from the in channel without blocking.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error, sendJob func(listREntry)) {
+func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listREntry, out chan<- error, cb func(fs.DirEntry) error) {
 	var dirs []string
 	var paths []string
 	var grouping int32
@@ -1729,19 +1736,26 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 		// https://issuetracker.google.com/issues/149522397
 		if len(dirs) > 1 && !foundItems {
 			if atomic.SwapInt32(&f.grouping, 1) != 1 {
-				fs.Debugf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
+				fs.Logf(f, "Disabling ListR to work around bug in drive as multi listing (%d) returned no entries", len(dirs))
 			}
+			var recycled = make([]listREntry, len(dirs))
 			f.listRmu.Lock()
 			for i := range dirs {
-				// Requeue the jobs
-				job := listREntry{id: dirs[i], path: paths[i]}
-				sendJob(job)
+				recycled[i] = listREntry{id: dirs[i], path: paths[i]}
 				// Make a note of these dirs - if they all turn
 				// out to be empty then we can re-enable grouping
 				f.listRempties[dirs[i]] = struct{}{}
 			}
 			f.listRmu.Unlock()
-			fs.Debugf(f, "Recycled %d entries", len(dirs))
+			// recycle these in the background so we don't deadlock
+			// the listR runners if they all get here
+			wg.Add(len(recycled))
+			go func() {
+				for _, entry := range recycled {
+					in <- entry
+				}
+				fs.Debugf(f, "Recycled %d entries", len(recycled))
+			}()
 		}
 		// If using a grouping of 1 and dir was empty then check to see if it
 		// is part of the group that caused grouping to be disabled.
@@ -1755,7 +1769,7 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
 				// empty so must have made a mistake
 				if len(f.listRempties) == 0 {
 					if atomic.SwapInt32(&f.grouping, listRGrouping) != listRGrouping {
-						fs.Debugf(f, "Re-enabling ListR as previous detection was in error")
+						fs.Logf(f, "Re-enabling ListR as previous detection was in error")
 					}
 				}
 			}
@@ -1810,33 +1824,21 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	overflow := []listREntry{}
 	listed := 0
 
-	// Send a job to the input channel if not closed. If the job
-	// won't fit then queue it in the overflow slice.
-	//
-	// This will not block if the channel is full.
-	sendJob := func(job listREntry) {
-		mu.Lock()
-		defer mu.Unlock()
-		if in == nil {
-			return
-		}
-		wg.Add(1)
-		select {
-		case in <- job:
-		default:
-			overflow = append(overflow, job)
-			wg.Add(-1)
-		}
-	}
-
-	// Send the entry to the caller, queueing any directories as new jobs
 	cb := func(entry fs.DirEntry) error {
-		if d, isDir := entry.(*fs.Dir); isDir {
-			job := listREntry{actualID(d.ID()), d.Remote()}
-			sendJob(job)
-		}
 		mu.Lock()
 		defer mu.Unlock()
+		if d, isDir := entry.(*fs.Dir); isDir && in != nil {
+			job := listREntry{actualID(d.ID()), d.Remote()}
+			select {
+			case in <- job:
+				// Adding the wg after we've entered the item is
+				// safe here because we know when the callback
+				// is called we are holding a waitgroup.
+				wg.Add(1)
+			default:
+				overflow = append(overflow, job)
+			}
+		}
 		listed++
 		return list.Add(entry)
 	}
@@ -1845,7 +1847,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	in <- listREntry{directoryID, dir}
 
 	for i := 0; i < fs.Config.Checkers; i++ {
-		go f.listRRunner(ctx, &wg, in, out, cb, sendJob)
+		go f.listRRunner(ctx, &wg, in, out, cb)
 	}
 	go func() {
 		// wait until the all directories are processed
@@ -1978,12 +1980,6 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
 	}
 	newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
 	if err != nil {
-		if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
-			// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
-			fs.Logf(nil, "Dangling shortcut %q detected", item.Name)
-			item.MimeType = shortcutMimeTypeDangling
-			return item, nil
-		}
 		return nil, errors.Wrap(err, "failed to resolve shortcut")
 	}
 	// make sure we use the Name, Parents and Trashed from the original item
@@ -2045,10 +2041,10 @@ func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Tim
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.NewObject(ctx, src.Remote())
+	exisitingObj, err := f.NewObject(ctx, src.Remote())
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(ctx, in, src, options...)
+		return exisitingObj, exisitingObj.Update(ctx, in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(ctx, in, src, options...)
@@ -2211,9 +2207,10 @@ func (f *Fs) delete(ctx context.Context, id string, useTrash bool) error {
 	})
 }
 
-// purgeCheck removes the dir directory, if check is set then it
-// refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
+// Rmdir deletes a directory
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	root := path.Join(f.root, dir)
 	dc := f.dirCache
 	directoryID, err := dc.FindDir(ctx, dir, false)
@@ -2226,22 +2223,20 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		return f.delete(ctx, shortcutID, f.opt.UseTrash)
 	}
 	var trashedFiles = false
-	if check {
-		found, err := f.list(ctx, []string{directoryID}, "", false, false, true, func(item *drive.File) bool {
-			if !item.Trashed {
-				fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
-				return true
-			}
-			fs.Debugf(dir, "Rmdir: contains trashed file: %q", item.Name)
-			trashedFiles = true
-			return false
-		})
-		if err != nil {
-			return err
-		}
-		if found {
-			return errors.Errorf("directory not empty")
+	found, err := f.list(ctx, []string{directoryID}, "", false, false, true, func(item *drive.File) bool {
+		if !item.Trashed {
+			fs.Debugf(dir, "Rmdir: contains file: %q", item.Name)
+			return true
 		}
+		fs.Debugf(dir, "Rmdir: contains trashed file: %q", item.Name)
+		trashedFiles = true
+		return false
+	})
+	if err != nil {
+		return err
+	}
+	if found {
+		return errors.Errorf("directory not empty")
 	}
 	if root != "" {
 		// trash the directory if it had trashed files
@@ -2251,8 +2246,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 		if err != nil {
 			return err
 		}
-	} else if check {
-		return errors.New("can't purge root directory")
 	}
 	f.dirCache.FlushDir(dir)
 	if err != nil {
@@ -2261,13 +2254,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
 	return nil
 }
 
-// Rmdir deletes a directory
-//
-// Returns an error if it isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, true)
-}
-
 // Precision of the object storage system
 func (f *Fs) Precision() time.Duration {
 	return time.Millisecond
@@ -2285,13 +2271,13 @@ func (f *Fs) Precision() time.Duration {
 func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
 	var srcObj *baseObject
 	ext := ""
-	isDoc := false
+	readDescription := false
 	switch src := src.(type) {
 	case *Object:
 		srcObj = &src.baseObject
 	case *documentObject:
 		srcObj, ext = &src.baseObject, src.ext()
-		isDoc = true
+		readDescription = true
 	case *linkObject:
 		srcObj, ext = &src.baseObject, src.ext()
 	default:
@@ -2299,12 +2285,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		return nil, fs.ErrorCantCopy
 	}
 
-	// Look to see if there is an existing object before we remove
-	// the extension from the remote
-	existingObject, _ := f.NewObject(ctx, remote)
-
-	// Adjust the remote name to be without the extension if we
-	// are about to create a doc.
 	if ext != "" {
 		if !strings.HasSuffix(remote, ext) {
 			fs.Debugf(src, "Can't copy - not same document type")
@@ -2313,12 +2293,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		remote = remote[:len(remote)-len(ext)]
 	}
 
+	// Look to see if there is an existing object
+	existingObject, _ := f.NewObject(ctx, remote)
+
 	createInfo, err := f.createFileInfo(ctx, remote, src.ModTime(ctx))
 	if err != nil {
 		return nil, err
 	}
 
-	if isDoc {
+	if readDescription {
 		// preserve the description on copy for docs
 		info, err := f.getFile(actualID(srcObj.id), "description")
 		if err != nil {
@@ -2350,22 +2333,6 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	if err != nil {
 		return nil, err
 	}
-	// Google docs aren't preserving their mod time after copy, so set them explicitly
-	// See: https://github.com/rclone/rclone/issues/4517
-	//
-	// FIXME remove this when google fixes the problem!
-	if isDoc {
-		// A short sleep is needed here in order to make the
-		// change effective, without it is is ignored. This is
-		// probably some eventual consistency nastiness.
-		sleepTime := 2 * time.Second
-		fs.Debugf(f, "Sleeping for %v before setting the modtime to work around drive bug - see #4517", sleepTime)
-		time.Sleep(sleepTime)
-		err = newObject.SetModTime(ctx, src.ModTime(ctx))
-		if err != nil {
-			return nil, err
-		}
-	}
 	if existingObject != nil {
 		err = existingObject.Remove(ctx)
 		if err != nil {
@@ -2380,11 +2347,23 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
+	if f.root == "" {
+		return errors.New("can't purge root directory")
+	}
 	if f.opt.TrashedOnly {
 		return errors.New("Can't purge with --drive-trashed-only. Use delete if you want to selectively delete files")
 	}
-	return f.purgeCheck(ctx, dir, false)
+	rootID, err := f.dirCache.RootID(ctx, false)
+	if err != nil {
+		return err
+	}
+	err = f.delete(ctx, shortcutID(rootID), f.opt.UseTrash)
+	f.dirCache.ResetRoot()
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // CleanUp empties the trash
@@ -2887,130 +2866,6 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
 	return dstFs.newObjectWithInfo(dstPath, info)
 }
 
-// List all team drives
-func (f *Fs) listTeamDrives(ctx context.Context) (drives []*drive.TeamDrive, err error) {
-	drives = []*drive.TeamDrive{}
-	listTeamDrives := f.svc.Teamdrives.List().PageSize(100)
-	var defaultFs Fs // default Fs with default Options
-	for {
-		var teamDrives *drive.TeamDriveList
-		err = f.pacer.Call(func() (bool, error) {
-			teamDrives, err = listTeamDrives.Context(ctx).Do()
-			return defaultFs.shouldRetry(err)
-		})
-		if err != nil {
-			return drives, errors.Wrap(err, "listing team drives failed")
-		}
-		drives = append(drives, teamDrives.TeamDrives...)
-		if teamDrives.NextPageToken == "" {
-			break
-		}
-		listTeamDrives.PageToken(teamDrives.NextPageToken)
-	}
-	return drives, nil
-}
-
-type unTrashResult struct {
-	Untrashed int
-	Errors    int
-}
-
-func (r unTrashResult) Error() string {
-	return fmt.Sprintf("%d errors while untrashing - see log", r.Errors)
-}
-
-// Restore the trashed files from dir, directoryID recursing if needed
-func (f *Fs) unTrash(ctx context.Context, dir string, directoryID string, recurse bool) (r unTrashResult, err error) {
-	directoryID = actualID(directoryID)
-	fs.Debugf(dir, "finding trash to restore in directory %q", directoryID)
-	_, err = f.list(ctx, []string{directoryID}, "", false, false, true, func(item *drive.File) bool {
-		remote := path.Join(dir, item.Name)
-		if item.ExplicitlyTrashed {
-			fs.Infof(remote, "restoring %q", item.Id)
-			if operations.SkipDestructive(ctx, remote, "restore") {
-				return false
-			}
-			update := drive.File{
-				ForceSendFields: []string{"Trashed"}, // necessary to set false value
-				Trashed:         false,
-			}
-			err := f.pacer.Call(func() (bool, error) {
-				_, err := f.svc.Files.Update(item.Id, &update).
-					SupportsAllDrives(true).
-					Fields("trashed").
-					Do()
-				return f.shouldRetry(err)
-			})
-			if err != nil {
-				err = errors.Wrap(err, "failed to restore")
-				r.Errors++
-				fs.Errorf(remote, "%v", err)
-			} else {
-				r.Untrashed++
-			}
-		}
-		if recurse && item.MimeType == "application/vnd.google-apps.folder" {
-			if !isShortcutID(item.Id) {
-				rNew, _ := f.unTrash(ctx, remote, item.Id, recurse)
-				r.Untrashed += rNew.Untrashed
-				r.Errors += rNew.Errors
-			}
-		}
-		return false
-	})
-	if err != nil {
-		err = errors.Wrap(err, "failed to list directory")
-		r.Errors++
-		fs.Errorf(dir, "%v", err)
-	}
-	if r.Errors != 0 {
-		return r, r
-	}
-	return r, nil
-}
-
-// Untrash dir
-func (f *Fs) unTrashDir(ctx context.Context, dir string, recurse bool) (r unTrashResult, err error) {
-	directoryID, err := f.dirCache.FindDir(ctx, dir, false)
-	if err != nil {
-		r.Errors++
-		return r, err
-	}
-	return f.unTrash(ctx, dir, directoryID, true)
-}
-
-// copy file with id to dest
-func (f *Fs) copyID(ctx context.Context, id, dest string) (err error) {
-	info, err := f.getFile(id, f.fileFields)
-	if err != nil {
-		return errors.Wrap(err, "couldn't find id")
-	}
-	if info.MimeType == driveFolderType {
-		return errors.Errorf("can't copy directory use: rclone copy --drive-root-folder-id %s %s %s", id, fs.ConfigString(f), dest)
-	}
-	info.Name = f.opt.Enc.ToStandardName(info.Name)
-	o, err := f.newObjectWithInfo(info.Name, info)
-	if err != nil {
-		return err
-	}
-	destDir, destLeaf, err := fspath.Split(dest)
-	if err != nil {
-		return err
-	}
-	if destLeaf == "" {
-		destLeaf = info.Name
-	}
-	dstFs, err := cache.Get(destDir)
-	if err != nil {
-		return err
-	}
-	_, err = operations.Copy(ctx, dstFs, nil, destLeaf, o)
-	if err != nil {
-		return errors.Wrap(err, "copy failed")
-	}
-	return nil
-}
-
 var commandHelp = []fs.CommandHelp{{
 	Name:  "get",
 	Short: "Get command for fetching the drive config parameters",
@@ -3062,78 +2917,6 @@ authenticated with "drive2:" can't read files from "drive:".
 	Opts: map[string]string{
 		"target": "optional target remote for the shortcut destination",
 	},
-}, {
-	Name:  "drives",
-	Short: "List the shared drives available to this account",
-	Long: `This command lists the shared drives (teamdrives) available to this
-account.
-
-Usage:
-
-    rclone backend drives drive:
-
-This will return a JSON list of objects like this
-
-    [
-        {
-            "id": "0ABCDEF-01234567890",
-            "kind": "drive#teamDrive",
-            "name": "My Drive"
-        },
-        {
-            "id": "0ABCDEFabcdefghijkl",
-            "kind": "drive#teamDrive",
-            "name": "Test Drive"
-        }
-    ]
-
-`,
-}, {
-	Name:  "untrash",
-	Short: "Untrash files and directories",
-	Long: `This command untrashes all the files and directories in the directory
-passed in recursively.
-
-Usage:
-
-This takes an optional directory to trash which make this easier to
-use via the API.
-
-    rclone backend untrash drive:directory
-    rclone backend -i untrash drive:directory subdir
-
-Use the -i flag to see what would be restored before restoring it.
-
-Result:
-
-    {
-        "Untrashed": 17,
-        "Errors": 0
-    }
-`,
-}, {
-	Name:  "copyid",
-	Short: "Copy files by ID",
-	Long: `This command copies files by ID
-
-Usage:
-
-    rclone backend copyid drive: ID path
-    rclone backend copyid drive: ID1 path1 ID2 path2
-
-It copies the drive file with ID given to the path (an rclone path which
-will be passed internally to rclone copyto). The ID and path pairs can be
-repeated.
-
-The path should end with a / to indicate copy the file as named to
-this directory. If it doesn't end with a / then the last path
-component will be used as the file name.
-
-If the destination is a drive backend then server side copying will be
-attempted if possible.
-
-Use the -i flag to see what would be copied before copying.
-`,
 }}
 
 // Command the backend to run a named command
@@ -3197,27 +2980,6 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
 			}
 		}
 		return f.makeShortcut(ctx, arg[0], dstFs, arg[1])
-	case "drives":
-		return f.listTeamDrives(ctx)
-	case "untrash":
-		dir := ""
-		if len(arg) > 0 {
-			dir = arg[0]
-		}
-		return f.unTrashDir(ctx, dir, true)
-	case "copyid":
-		if len(arg)%2 != 0 {
-			return nil, errors.New("need an even number of arguments")
-		}
-		for len(arg) > 0 {
-			id, dest := arg[0], arg[1]
-			arg = arg[2:]
-			err = f.copyID(ctx, id, dest)
-			if err != nil {
-				return nil, errors.Wrapf(err, "failed copying %q to %q", id, dest)
-			}
-		}
-		return nil, nil
 	default:
 		return nil, fs.ErrorCommandNotFound
 	}
@@ -3461,9 +3223,6 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	if o.mimeType == shortcutMimeTypeDangling {
-		return nil, errors.New("can't read dangling shortcut")
-	}
 	if o.v2Download {
 		var v2File *drive_v2.File
 		err = o.fs.pacer.Call(func() (bool, error) {

backend/drive/drive_internal_test.go

@@ -7,21 +7,16 @@ import (
 	"io"
 	"io/ioutil"
 	"mime"
-	"os"
-	"path"
 	"path/filepath"
 	"strings"
 	"testing"
-	"time"
 
 	"github.com/pkg/errors"
 	_ "github.com/rclone/rclone/backend/local"
 	"github.com/rclone/rclone/fs"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fstest"
 	"github.com/rclone/rclone/fstest/fstests"
-	"github.com/rclone/rclone/lib/random"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"google.golang.org/api/drive/v3"
@@ -274,15 +269,14 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 	}
 }
 
-const (
-	// from fstest/fstests/fstests.go
-	existingDir    = "hello? sausage"
-	existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
-	existingSubDir = "êé"
-)
-
 // TestIntegration/FsMkdir/FsPutFiles/Internal/Shortcuts
 func (f *Fs) InternalTestShortcuts(t *testing.T) {
+	const (
+		// from fstest/fstests/fstests.go
+		existingDir    = "hello? sausage"
+		existingFile   = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
+		existingSubDir = "êé"
+	)
 	ctx := context.Background()
 	srcObj, err := f.NewObject(ctx, existingFile)
 	require.NoError(t, err)
@@ -367,99 +361,6 @@ func (f *Fs) InternalTestShortcuts(t *testing.T) {
 	})
 }
 
-// TestIntegration/FsMkdir/FsPutFiles/Internal/UnTrash
-func (f *Fs) InternalTestUnTrash(t *testing.T) {
-	ctx := context.Background()
-
-	// Make some objects, one in a subdir
-	contents := random.String(100)
-	file1 := fstest.NewItem("trashDir/toBeTrashed", contents, time.Now())
-	_, obj1 := fstests.PutTestContents(ctx, t, f, &file1, contents, false)
-	file2 := fstest.NewItem("trashDir/subdir/toBeTrashed", contents, time.Now())
-	_, _ = fstests.PutTestContents(ctx, t, f, &file2, contents, false)
-
-	// Check objects
-	checkObjects := func() {
-		fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{
-			file1,
-			file2,
-		}, []string{
-			"trashDir/subdir",
-		}, f.Precision())
-	}
-	checkObjects()
-
-	// Make sure we are using the trash
-	require.Equal(t, true, f.opt.UseTrash)
-
-	// Remove the object and the dir
-	require.NoError(t, obj1.Remove(ctx))
-	require.NoError(t, f.Purge(ctx, "trashDir/subdir"))
-
-	// Check objects gone
-	fstest.CheckListingWithRoot(t, f, "trashDir", []fstest.Item{}, []string{}, f.Precision())
-
-	// Restore the object and directory
-	r, err := f.unTrashDir(ctx, "trashDir", true)
-	require.NoError(t, err)
-	assert.Equal(t, unTrashResult{Errors: 0, Untrashed: 2}, r)
-
-	// Check objects restored
-	checkObjects()
-
-	// Remove the test dir
-	require.NoError(t, f.Purge(ctx, "trashDir"))
-}
-
-// TestIntegration/FsMkdir/FsPutFiles/Internal/CopyID
-func (f *Fs) InternalTestCopyID(t *testing.T) {
-	ctx := context.Background()
-	obj, err := f.NewObject(ctx, existingFile)
-	require.NoError(t, err)
-	o := obj.(*Object)
-
-	dir, err := ioutil.TempDir("", "rclone-drive-copyid-test")
-	require.NoError(t, err)
-	defer func() {
-		_ = os.RemoveAll(dir)
-	}()
-
-	checkFile := func(name string) {
-		filePath := filepath.Join(dir, name)
-		fi, err := os.Stat(filePath)
-		require.NoError(t, err)
-		assert.Equal(t, int64(100), fi.Size())
-		err = os.Remove(filePath)
-		require.NoError(t, err)
-	}
-
-	t.Run("BadID", func(t *testing.T) {
-		err = f.copyID(ctx, "ID-NOT-FOUND", dir+"/")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "couldn't find id")
-	})
-
-	t.Run("Directory", func(t *testing.T) {
-		rootID, err := f.dirCache.RootID(ctx, false)
-		require.NoError(t, err)
-		err = f.copyID(ctx, rootID, dir+"/")
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), "can't copy directory")
-	})
-
-	t.Run("WithoutDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/")
-		require.NoError(t, err)
-		checkFile(path.Base(existingFile))
-	})
-
-	t.Run("WithDestName", func(t *testing.T) {
-		err = f.copyID(ctx, o.id, dir+"/potato.txt")
-		require.NoError(t, err)
-		checkFile("potato.txt")
-	})
-}
-
 func (f *Fs) InternalTest(t *testing.T) {
 	// These tests all depend on each other so run them as nested tests
 	t.Run("DocumentImport", func(t *testing.T) {
@@ -475,8 +376,6 @@ func (f *Fs) InternalTest(t *testing.T) {
 		})
 	})
 	t.Run("Shortcuts", f.InternalTestShortcuts)
-	t.Run("UnTrash", f.InternalTestUnTrash)
-	t.Run("CopyID", f.InternalTestCopyID)
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/dropbox/dropbox.go

@@ -125,7 +125,13 @@ func init() {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Dropbox App Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Dropbox App Client Secret\nLeave blank normally.",
+		}, {
 			Name: "chunk_size",
 			Help: fmt.Sprintf(`Upload chunk size. (< %v).
 
@@ -142,31 +148,6 @@ memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
 			Help:     "Impersonate this user when using a business account.",
 			Default:  "",
 			Advanced: true,
-		}, {
-			Name: "shared_files",
-			Help: `Instructs rclone to work on individual shared files.
-
-In this mode rclone's features are extremely limited - only list (ls, lsl, etc.) 
-operations and read operations (e.g. downloading) are supported in this mode.
-All other operations will be disabled.`,
-			Default:  false,
-			Advanced: true,
-		}, {
-			Name: "shared_folders",
-			Help: `Instructs rclone to work on shared folders.
-			
-When this flag is used with no path only the List operation is supported and 
-all available shared folders will be listed. If you specify a path the first part 
-will be interpreted as the name of shared folder. Rclone will then try to mount this 
-shared to the root namespace. On success shared folder rclone proceeds normally. 
-The shared folder is now pretty much a normal folder and all normal operations 
-are supported. 
-
-Note that we don't unmount the shared folder afterwards so the 
---dropbox-shared-folders can be omitted after the first use of a particular 
-shared folder.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -180,17 +161,15 @@ shared folder.`,
 				encoder.EncodeDel |
 				encoder.EncodeRightSpace |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize     fs.SizeSuffix        `config:"chunk_size"`
-	Impersonate   string               `config:"impersonate"`
-	SharedFiles   bool                 `config:"shared_files"`
-	SharedFolders bool                 `config:"shared_folders"`
-	Enc           encoder.MultiEncoder `config:"encoding"`
+	ChunkSize   fs.SizeSuffix        `config:"chunk_size"`
+	Impersonate string               `config:"impersonate"`
+	Enc         encoder.MultiEncoder `config:"encoding"`
 }
 
 // Fs represents a remote dropbox server
@@ -213,9 +192,7 @@ type Fs struct {
 //
 // Dropbox Objects always have full metadata
 type Object struct {
-	fs      *Fs // what this object is part of
-	id      string
-	url     string
+	fs      *Fs       // what this object is part of
 	remote  string    // The remote path
 	bytes   int64     // size of the object
 	modTime time.Time // time it was last modified
@@ -361,60 +338,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		CaseInsensitive:         true,
 		ReadMimeType:            true,
 		CanHaveEmptyDirectories: true,
-	})
-
-	// do not fill features yet
-	if f.opt.SharedFiles {
-		f.setRoot(root)
-		if f.root == "" {
-			return f, nil
-		}
-		_, err := f.findSharedFile(f.root)
-		f.root = ""
-		if err == nil {
-			return f, fs.ErrorIsFile
-		}
-		return f, nil
-	}
-
-	if f.opt.SharedFolders {
-		f.setRoot(root)
-		if f.root == "" {
-			return f, nil // our root it empty so we probably want to list shared folders
-		}
-
-		dir := path.Dir(f.root)
-		if dir == "." {
-			dir = f.root
-		}
-
-		// root is not empty so we have find the right shared folder if it exists
-		id, err := f.findSharedFolder(dir)
-		if err != nil {
-			// if we didn't find the specified shared folder we have to bail out here
-			return nil, err
-		}
-		// we found the specified shared folder so let's mount it
-		// this will add it to the users normal root namespace and allows us
-		// to actually perform operations on it using the normal api endpoints.
-		err = f.mountSharedFolder(id)
-		if err != nil {
-			switch e := err.(type) {
-			case sharing.MountFolderAPIError:
-				if e.EndpointError == nil || (e.EndpointError != nil && e.EndpointError.Tag != sharing.MountFolderErrorAlreadyMounted) {
-					return nil, err
-				}
-			default:
-				return nil, err
-			}
-			// if the moint failed we have to abort here
-		}
-		// if the mount succeeded it's now a normal folder in the users root namespace
-		// we disable shared folder mode and proceed normally
-		f.opt.SharedFolders = false
-	}
-
-	f.features.Fill(f)
+	}).Fill(f)
+	f.setRoot(root)
 
 	// If root starts with / then use the actual root
 	if strings.HasPrefix(root, "/") {
@@ -436,7 +361,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		fs.Debugf(f, "Using root namespace %q", f.ns)
 	}
-	f.setRoot(root)
 
 	// See if the root is actually an object
 	_, err = f.getFileMetadata(f.slashRoot)
@@ -547,150 +471,9 @@ func (f *Fs) newObjectWithInfo(remote string, info *files.FileMetadata) (fs.Obje
 // NewObject finds the Object at remote.  If it can't be found
 // it returns the error fs.ErrorObjectNotFound.
 func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
-	if f.opt.SharedFiles {
-		return f.findSharedFile(remote)
-	}
 	return f.newObjectWithInfo(remote, nil)
 }
 
-// listSharedFoldersApi lists all available shared folders mounted and not mounted
-// we'll need the id later so we have to return them in original format
-func (f *Fs) listSharedFolders() (entries fs.DirEntries, err error) {
-	started := false
-	var res *sharing.ListFoldersResult
-	for {
-		if !started {
-			arg := sharing.ListFoldersArgs{
-				Limit: 100,
-			}
-			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListFolders(&arg)
-				return shouldRetry(err)
-			})
-			if err != nil {
-				return nil, err
-			}
-			started = true
-		} else {
-			arg := sharing.ListFoldersContinueArg{
-				Cursor: res.Cursor,
-			}
-			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListFoldersContinue(&arg)
-				return shouldRetry(err)
-			})
-			if err != nil {
-				return nil, errors.Wrap(err, "list continue")
-			}
-		}
-		for _, entry := range res.Entries {
-			leaf := f.opt.Enc.ToStandardName(entry.Name)
-			d := fs.NewDir(leaf, time.Now()).SetID(entry.SharedFolderId)
-			entries = append(entries, d)
-			if err != nil {
-				return nil, err
-			}
-		}
-		if res.Cursor == "" {
-			break
-		}
-	}
-
-	return entries, nil
-}
-
-// findSharedFolder find the id for a given shared folder name
-// somewhat annoyingly there is no endpoint to query a shared folder by it's name
-// so our only option is to iterate over all shared folders
-func (f *Fs) findSharedFolder(name string) (id string, err error) {
-	entries, err := f.listSharedFolders()
-	if err != nil {
-		return "", err
-	}
-	for _, entry := range entries {
-		if entry.(*fs.Dir).Remote() == name {
-			return entry.(*fs.Dir).ID(), nil
-		}
-	}
-	return "", fs.ErrorDirNotFound
-}
-
-// mountSharedFolders mount a shared folder to the root namespace
-func (f *Fs) mountSharedFolder(id string) error {
-	arg := sharing.MountFolderArg{
-		SharedFolderId: id,
-	}
-	err := f.pacer.Call(func() (bool, error) {
-		_, err := f.sharing.MountFolder(&arg)
-		return shouldRetry(err)
-	})
-	return err
-}
-
-// listSharedFolders lists shared the user as access to (note this means individual
-// files not files contained in shared folders)
-func (f *Fs) listReceivedFiles() (entries fs.DirEntries, err error) {
-	started := false
-	var res *sharing.ListFilesResult
-	for {
-		if !started {
-			arg := sharing.ListFilesArg{
-				Limit: 100,
-			}
-			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListReceivedFiles(&arg)
-				return shouldRetry(err)
-			})
-			if err != nil {
-				return nil, err
-			}
-			started = true
-		} else {
-			arg := sharing.ListFilesContinueArg{
-				Cursor: res.Cursor,
-			}
-			err := f.pacer.Call(func() (bool, error) {
-				res, err = f.sharing.ListReceivedFilesContinue(&arg)
-				return shouldRetry(err)
-			})
-			if err != nil {
-				return nil, errors.Wrap(err, "list continue")
-			}
-		}
-		for _, entry := range res.Entries {
-			fmt.Printf("%+v\n", entry)
-			entryPath := entry.Name
-			o := &Object{
-				fs:      f,
-				url:     entry.PreviewUrl,
-				remote:  entryPath,
-				modTime: entry.TimeInvited,
-			}
-			if err != nil {
-				return nil, err
-			}
-			entries = append(entries, o)
-		}
-		if res.Cursor == "" {
-			break
-		}
-	}
-	return entries, nil
-}
-
-func (f *Fs) findSharedFile(name string) (o *Object, err error) {
-	files, err := f.listReceivedFiles()
-	if err != nil {
-		return nil, err
-	}
-	for _, entry := range files {
-		if entry.(*Object).remote == name {
-			return entry.(*Object), nil
-		}
-	}
-	return nil, fs.ErrorObjectNotFound
-}
-
 // List the objects and directories in dir into entries.  The
 // entries can be returned in any order but should be for a
 // complete directory.
@@ -701,13 +484,6 @@ func (f *Fs) findSharedFile(name string) (o *Object, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	if f.opt.SharedFiles {
-		return f.listReceivedFiles()
-	}
-	if f.opt.SharedFolders {
-		return f.listSharedFolders()
-	}
-
 	root := f.slashRoot
 	if dir != "" {
 		root += "/" + dir
@@ -771,7 +547,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 			leaf := f.opt.Enc.ToStandardName(path.Base(entryPath))
 			remote := path.Join(dir, leaf)
 			if folderInfo != nil {
-				d := fs.NewDir(remote, time.Now()).SetID(folderInfo.Id)
+				d := fs.NewDir(remote, time.Now())
 				entries = append(entries, d)
 			} else if fileInfo != nil {
 				o, err := f.newObjectWithInfo(remote, fileInfo)
@@ -794,9 +570,6 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	if f.opt.SharedFiles || f.opt.SharedFolders {
-		return nil, fserrors.NoRetryError(errors.New("not support in shared files mode"))
-	}
 	// Temporary Object under construction
 	o := &Object{
 		fs:     f,
@@ -812,9 +585,6 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	if f.opt.SharedFiles || f.opt.SharedFolders {
-		return fserrors.NoRetryError(errors.New("not support in shared files mode"))
-	}
 	root := path.Join(f.slashRoot, dir)
 
 	// can't create or run metadata on root
@@ -841,9 +611,10 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return err
 }
 
-// purgeCheck removes the root directory, if check is set then it
-// refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) {
+// Rmdir deletes the container
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	root := path.Join(f.slashRoot, dir)
 
 	// can't remove root
@@ -851,33 +622,31 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 		return errors.New("can't remove root directory")
 	}
 
-	if check {
-		// check directory exists
-		_, err = f.getDirMetadata(root)
-		if err != nil {
-			return errors.Wrap(err, "Rmdir")
-		}
+	// check directory exists
+	_, err := f.getDirMetadata(root)
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
 
-		root = f.opt.Enc.FromStandardPath(root)
-		// check directory empty
-		arg := files.ListFolderArg{
-			Path:      root,
-			Recursive: false,
-		}
-		if root == "/" {
-			arg.Path = "" // Specify root folder as empty string
-		}
-		var res *files.ListFolderResult
-		err = f.pacer.Call(func() (bool, error) {
-			res, err = f.srv.ListFolder(&arg)
-			return shouldRetry(err)
-		})
-		if err != nil {
-			return errors.Wrap(err, "Rmdir")
-		}
-		if len(res.Entries) != 0 {
-			return errors.New("directory not empty")
-		}
+	root = f.opt.Enc.FromStandardPath(root)
+	// check directory empty
+	arg := files.ListFolderArg{
+		Path:      root,
+		Recursive: false,
+	}
+	if root == "/" {
+		arg.Path = "" // Specify root folder as empty string
+	}
+	var res *files.ListFolderResult
+	err = f.pacer.Call(func() (bool, error) {
+		res, err = f.srv.ListFolder(&arg)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	if len(res.Entries) != 0 {
+		return errors.New("directory not empty")
 	}
 
 	// remove it
@@ -888,16 +657,6 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	return err
 }
 
-// Rmdir deletes the container
-//
-// Returns an error if it isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	if f.opt.SharedFiles || f.opt.SharedFolders {
-		return fserrors.NoRetryError(errors.New("not support in shared files mode"))
-	}
-	return f.purgeCheck(ctx, dir, true)
-}
-
 // Precision returns the precision
 func (f *Fs) Precision() time.Duration {
 	return time.Second
@@ -960,8 +719,15 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) (err error) {
+	// Let dropbox delete the filesystem tree
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.srv.DeleteV2(&files.DeleteArg{
+			Path: f.opt.Enc.FromStandardPath(f.slashRoot),
+		})
+		return shouldRetry(err)
+	})
+	return err
 }
 
 // Move src to this remote using server side move operations.
@@ -1166,16 +932,8 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// ID returns the object id
-func (o *Object) ID() string {
-	return o.id
-}
-
 // Hash returns the dropbox special hash
 func (o *Object) Hash(ctx context.Context, t hash.Type) (string, error) {
-	if o.fs.opt.SharedFiles || o.fs.opt.SharedFolders {
-		return "", fserrors.NoRetryError(errors.New("not support in shared files mode"))
-	}
 	if t != DbHashType {
 		return "", hash.ErrUnsupported
 	}
@@ -1193,9 +951,8 @@ func (o *Object) Size() int64 {
 
 // setMetadataFromEntry sets the fs data from a files.FileMetadata
 //
-// This isn't a complete set of metadata and has an inaccurate date
+// This isn't a complete set of metadata and has an inacurate date
 func (o *Object) setMetadataFromEntry(info *files.FileMetadata) error {
-	o.id = info.Id
 	o.bytes = int64(info.Size)
 	o.modTime = info.ClientModified
 	o.hash = info.ContentHash
@@ -1264,27 +1021,10 @@ func (o *Object) Storable() bool {
 
 // Open an object for read
 func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	if o.fs.opt.SharedFiles {
-		if len(options) != 0 {
-			return nil, errors.New("OpenOptions not supported for shared files")
-		}
-		arg := sharing.GetSharedLinkMetadataArg{
-			Url: o.url,
-		}
-		err = o.fs.pacer.Call(func() (bool, error) {
-			_, in, err = o.fs.sharing.GetSharedLinkFile(&arg)
-			return shouldRetry(err)
-		})
-		if err != nil {
-			return nil, err
-		}
-		return
-	}
-
 	fs.FixRangeOption(options, o.bytes)
 	headers := fs.OpenOptionHeaders(options)
 	arg := files.DownloadArg{
-		Path:         o.id,
+		Path:         o.fs.opt.Enc.FromStandardPath(o.remotePath()),
 		ExtraHeaders: headers,
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
@@ -1418,9 +1158,6 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 //
 // The new object may have been created if an error is returned
 func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	if o.fs.opt.SharedFiles || o.fs.opt.SharedFolders {
-		return fserrors.NoRetryError(errors.New("not support in shared files mode"))
-	}
 	remote := o.remotePath()
 	if ignoredFiles.MatchString(remote) {
 		return fserrors.NoRetryError(errors.Errorf("file name %q is disallowed - not uploading", path.Base(remote)))
@@ -1449,9 +1186,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 // Remove an object
 func (o *Object) Remove(ctx context.Context) (err error) {
-	if o.fs.opt.SharedFiles || o.fs.opt.SharedFolders {
-		return fserrors.NoRetryError(errors.New("not support in shared files mode"))
-	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, err = o.fs.srv.DeleteV2(&files.DeleteArg{
 			Path: o.fs.opt.Enc.FromStandardPath(o.remotePath()),
@@ -1472,5 +1206,4 @@ var (
 	_ fs.DirMover     = (*Fs)(nil)
 	_ fs.Abouter      = (*Fs)(nil)
 	_ fs.Object       = (*Object)(nil)
-	_ fs.IDer         = (*Object)(nil)
 )

backend/fichier/api.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -29,20 +28,6 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	// Detect this error which the integration tests provoke
-	// error HTTP error 403 (403 Forbidden) returned body: "{\"message\":\"Flood detected: IP Locked #374\",\"status\":\"KO\"}"
-	//
-	// https://1fichier.com/api.html
-	//
-	// file/ls.cgi is limited :
-	//
-	// Warning (can be changed in case of abuses) :
-	// List all files of the account is limited to 1 request per hour.
-	// List folders is limited to 5 000 results and 1 request per folder per 30s.
-	if err != nil && strings.Contains(err.Error(), "Flood detected") {
-		fs.Debugf(nil, "Sleeping for 30 seconds due to: %v", err)
-		time.Sleep(30 * time.Second)
-	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 

backend/fichier/fichier.go

@@ -306,10 +306,10 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
 // will return the object and the error, otherwise will return
 // nil and the error
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	existingObj, err := f.NewObject(ctx, src.Remote())
+	exisitingObj, err := f.NewObject(ctx, src.Remote())
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(ctx, in, src, options...)
+		return exisitingObj, exisitingObj.Update(ctx, in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(ctx, in, src, options...)
@@ -323,7 +323,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size int64, options ...fs.OpenOption) (fs.Object, error) {
-	if size > int64(300e9) {
+	if size > int64(100e9) {
 		return nil, errors.New("File too big, cant upload")
 	} else if size == 0 {
 		return nil, fs.ErrorCantUploadEmptyFiles

backend/ftp/ftp.go

@@ -6,9 +6,8 @@ import (
 	"crypto/tls"
 	"io"
 	"net/textproto"
+	"os"
 	"path"
-	"runtime"
-	"strings"
 	"sync"
 	"time"
 
@@ -21,15 +20,10 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/hash"
 	"github.com/rclone/rclone/lib/encoder"
-	"github.com/rclone/rclone/lib/env"
 	"github.com/rclone/rclone/lib/pacer"
 	"github.com/rclone/rclone/lib/readers"
 )
 
-var (
-	currentUser = env.CurrentUser()
-)
-
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -46,7 +40,7 @@ func init() {
 			}},
 		}, {
 			Name: "user",
-			Help: "FTP username, leave blank for current username, " + currentUser,
+			Help: "FTP username, leave blank for current username, " + os.Getenv("USER"),
 		}, {
 			Name: "port",
 			Help: "FTP port, leave blank to use default (21)",
@@ -166,43 +160,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Enable debugging output
-type debugLog struct {
-	mu   sync.Mutex
-	auth bool
-}
-
-// Write writes len(p) bytes from p to the underlying data stream. It returns
-// the number of bytes written from p (0 <= n <= len(p)) and any error
-// encountered that caused the write to stop early. Write must return a non-nil
-// error if it returns n < len(p). Write must not modify the slice data, even
-// temporarily.
-//
-// Implementations must not retain p.
-//
-// This writes debug info to the log
-func (dl *debugLog) Write(p []byte) (n int, err error) {
-	dl.mu.Lock()
-	defer dl.mu.Unlock()
-	_, file, _, ok := runtime.Caller(1)
-	direction := "FTP Rx"
-	if ok && strings.Contains(file, "multi") {
-		direction = "FTP Tx"
-	}
-	lines := strings.Split(string(p), "\r\n")
-	if lines[len(lines)-1] == "" {
-		lines = lines[:len(lines)-1]
-	}
-	for _, line := range lines {
-		if !dl.auth && strings.HasPrefix(line, "PASS") {
-			fs.Debugf(direction, "PASS *****")
-			continue
-		}
-		fs.Debugf(direction, "%q", line)
-	}
-	return len(p), nil
-}
-
 // Open a new connection to the FTP server.
 func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	fs.Debugf(f, "Connecting to FTP server")
@@ -226,9 +183,6 @@ func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
 	if f.opt.DisableEPSV {
 		ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
 	}
-	if fs.Config.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
-		ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: fs.Config.Dump&fs.DumpAuth != 0}))
-	}
 	c, err := ftp.Dial(f.dialAddr, ftpConfig...)
 	if err != nil {
 		fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
@@ -315,7 +269,7 @@ func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	}
 	user := opt.User
 	if user == "" {
-		user = currentUser
+		user = os.Getenv("USER")
 	}
 	port := opt.Port
 	if port == "" {

backend/googlecloudstorage/googlecloudstorage.go

@@ -79,8 +79,7 @@ func init() {
 		Config: func(name string, m configmap.Mapper) {
 			saFile, _ := m.Get("service_account_file")
 			saCreds, _ := m.Get("service_account_credentials")
-			anonymous, _ := m.Get("anonymous")
-			if saFile != "" || saCreds != "" || anonymous == "true" {
+			if saFile != "" || saCreds != "" {
 				return
 			}
 			err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
@@ -88,7 +87,13 @@ func init() {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Google Application Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Google Application Client Secret\nLeave blank normally.",
+		}, {
 			Name: "project_number",
 			Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",
 		}, {
@@ -98,10 +103,6 @@ func init() {
 			Name: "service_account_credentials",
 			Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
 			Hide: fs.OptionHideBoth,
-		}, {
-			Name:    "anonymous",
-			Help:    "Access public buckets and objects without credentials\nSet to 'true' if you just want to download files and don't configure credentials.",
-			Default: false,
 		}, {
 			Name: "object_acl",
 			Help: "Access Control List for new objects.",
@@ -255,7 +256,7 @@ Docs: https://cloud.google.com/storage/docs/bucket-policy-only
 			Default: (encoder.Base |
 				encoder.EncodeCrLf |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
@@ -264,7 +265,6 @@ type Options struct {
 	ProjectNumber             string               `config:"project_number"`
 	ServiceAccountFile        string               `config:"service_account_file"`
 	ServiceAccountCredentials string               `config:"service_account_credentials"`
-	Anonymous                 bool                 `config:"anonymous"`
 	ObjectACL                 string               `config:"object_acl"`
 	BucketACL                 string               `config:"bucket_acl"`
 	BucketPolicyOnly          bool                 `config:"bucket_policy_only"`
@@ -411,9 +411,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		opt.ServiceAccountCredentials = string(loadedCreds)
 	}
-	if opt.Anonymous {
-		oAuthClient = &http.Client{}
-	} else if opt.ServiceAccountCredentials != "" {
+	if opt.ServiceAccountCredentials != "" {
 		oAuthClient, err = getServiceAccountClient([]byte(opt.ServiceAccountCredentials))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")

backend/googlephotos/googlephotos.go

@@ -21,6 +21,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/googlephotos/api"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -109,7 +110,13 @@ func init() {
 `)
 
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Google Application Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Google Application Client Secret\nLeave blank normally.",
+		}, {
 			Name:    "read_only",
 			Default: false,
 			Help: `Set to make the Google Photos backend read only.
@@ -132,7 +139,7 @@ you want to read the media.`,
 			Default:  2000,
 			Help:     `Year limits the photos to be downloaded to those which are uploaded after the given year`,
 			Advanced: true,
-		}}...),
+		}},
 	})
 }
 

backend/googlephotos/googlephotos_test.go

@@ -115,7 +115,7 @@ func TestIntegration(t *testing.T) {
 				assert.Equal(t, "2013-07-26 08:57:21 +0000 UTC", entries[0].ModTime(ctx).String())
 			})
 
-			// Check it is there in the date/month/year hierarchy
+			// Check it is there in the date/month/year heirachy
 			// 2013-07-13 is the creation date of the folder
 			checkPresent := func(t *testing.T, objPath string) {
 				entries, err := f.List(ctx, objPath)

backend/hubic/hubic.go

@@ -4,7 +4,7 @@ package hubic
 
 // This uses the normal swift mechanism to update the credentials and
 // ignores the expires field returned by the Hubic API.  This may need
-// to be revisited after some actual experience.
+// to be revisted after some actual experience.
 
 import (
 	"context"
@@ -20,6 +20,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/rclone/rclone/backend/swift"
 	"github.com/rclone/rclone/fs"
+	"github.com/rclone/rclone/fs/config"
 	"github.com/rclone/rclone/fs/config/configmap"
 	"github.com/rclone/rclone/fs/config/configstruct"
 	"github.com/rclone/rclone/fs/config/obscure"
@@ -62,7 +63,13 @@ func init() {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
 		},
-		Options: append(oauthutil.SharedOptions, swift.SharedOptions...),
+		Options: append([]fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Hubic Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Hubic Client Secret\nLeave blank normally.",
+		}}, swift.SharedOptions...),
 	})
 }
 

backend/jottacloud/api/types.go

@@ -153,9 +153,9 @@ type CustomerInfo struct {
 	AccountType       string      `json:"account_type"`
 	SubscriptionType  string      `json:"subscription_type"`
 	Usage             int64       `json:"usage"`
-	Quota             int64       `json:"quota"`
+	Qouta             int64       `json:"quota"`
 	BusinessUsage     int64       `json:"business_usage"`
-	BusinessQuota     int64       `json:"business_quota"`
+	BusinessQouta     int64       `json:"business_quota"`
 	WriteLocked       bool        `json:"write_locked"`
 	ReadLocked        bool        `json:"read_locked"`
 	LockedCause       interface{} `json:"locked_cause"`
@@ -386,7 +386,7 @@ type Error struct {
 	Cause      string `xml:"cause"`
 }
 
-// Error returns a string for the error and satisfies the error interface
+// Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("error %d", e.StatusCode)
 	if e.Message != "" {

backend/jottacloud/jottacloud.go

@@ -107,7 +107,7 @@ func init() {
 				}
 			}
 
-			fmt.Printf("Use legacy authentication?.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n")
+			fmt.Printf("Use legacy authentification?.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n")
 			if config.Confirm(false) {
 				v1config(ctx, name, m)
 			} else {
@@ -230,7 +230,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// v1config configure a jottacloud backend using legacy authentication
+// v1config configure a jottacloud backend using legacy authentification
 func v1config(ctx context.Context, name string, m configmap.Mapper) {
 	srv := rest.NewClient(fshttp.NewClient(fs.Config))
 
@@ -323,7 +323,7 @@ func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegis
 	return deviceRegistration, err
 }
 
-// doAuthV1 runs the actual token request for V1 authentication
+// doAuthV1 runs the actual token request for V1 authentification
 func doAuthV1(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
 	// prepare out token request with username and password
 	values := url.Values{}
@@ -353,7 +353,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
 				authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
 				opts.ExtraHeaders = make(map[string]string)
 				opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
-				_, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
+				resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
 			}
 		}
 	}
@@ -365,7 +365,7 @@ func doAuthV1(ctx context.Context, srv *rest.Client, username, password string)
 	return token, err
 }
 
-// v2config configure a jottacloud backend using the modern JottaCli token based authentication
+// v2config configure a jottacloud backend using the modern JottaCli token based authentification
 func v2config(ctx context.Context, name string, m configmap.Mapper) {
 	srv := rest.NewClient(fshttp.NewClient(fs.Config))
 
@@ -373,9 +373,6 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
 	fmt.Printf("Login Token> ")
 	loginToken := config.ReadLine()
 
-	m.Set(configClientID, "jottacli")
-	m.Set(configClientSecret, "")
-
 	token, err := doAuthV2(ctx, srv, loginToken, m)
 	if err != nil {
 		log.Fatalf("Failed to get oauth token: %s", err)
@@ -387,6 +384,7 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
 
 	fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
 	if config.Confirm(false) {
+		oauthConfig.ClientID = "jottacli"
 		oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
 		if err != nil {
 			log.Fatalf("Failed to load oAuthClient: %s", err)
@@ -405,7 +403,7 @@ func v2config(ctx context.Context, name string, m configmap.Mapper) {
 	m.Set("configVersion", strconv.Itoa(configVersion))
 }
 
-// doAuthV2 runs the actual token request for V2 authentication
+// doAuthV2 runs the actual token request for V2 authentification
 func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
 	loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
 	if err != nil {
@@ -553,7 +551,7 @@ func (f *Fs) setEndpointURL() {
 	if f.opt.Mountpoint == "" {
 		f.opt.Mountpoint = defaultMountpoint
 	}
-	f.endpointURL = path.Join(f.user, f.opt.Device, f.opt.Mountpoint)
+	f.endpointURL = urlPathEscape(path.Join(f.user, f.opt.Device, f.opt.Mountpoint))
 }
 
 // readMetaDataForPath reads the metadata from the path
@@ -1072,8 +1070,8 @@ func (f *Fs) Precision() time.Duration {
 }
 
 // Purge deletes all the files and the container
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // copyOrMoves copies or moves directories or files depending on the method parameter
@@ -1089,7 +1087,8 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
-		return shouldRetry(resp, err)
+		retry, _ := shouldRetry(resp, err)
+		return (retry && resp.StatusCode != 500), err
 	})
 	if err != nil {
 		return nil, err
@@ -1193,6 +1192,18 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 	_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
 
+	// surprise! jottacloud fucked up dirmove - the api spits out an error but
+	// dir gets moved regardless
+	if apiErr, ok := err.(*api.Error); ok {
+		if apiErr.StatusCode == 500 {
+			_, err := f.NewObject(ctx, dstRemote)
+			if err == fs.ErrorNotAFile {
+				log.Printf("FIXME: ignoring DirMove error - move succeeded anyway\n")
+				return nil
+			}
+			return err
+		}
+	}
 	if err != nil {
 		return errors.Wrap(err, "couldn't move directory")
 	}
@@ -1512,7 +1523,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// If the file state is INCOMPLETE and CORRUPT, try to upload a then
+	// If the file state is INCOMPLETE and CORRPUT, try to upload a then
 	if response.State != "COMPLETED" {
 		// how much do we still have to upload?
 		remainingBytes := size - response.ResumePos

backend/local/about_unix.go

@@ -4,7 +4,6 @@ package local
 
 import (
 	"context"
-	"os"
 	"syscall"
 
 	"github.com/pkg/errors"
@@ -16,9 +15,6 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	var s syscall.Statfs_t
 	err := syscall.Statfs(f.root, &s)
 	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, fs.ErrorDirNotFound
-		}
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
 	bs := int64(s.Bsize) // nolint: unconvert

backend/local/lchtimes.go

@@ -1,4 +1,4 @@
-// +build windows plan9 js
+// +build windows plan9
 
 package local
 

backend/local/lchtimes_unix.go

@@ -1,4 +1,4 @@
-// +build !windows,!plan9,!js
+// +build !windows,!plan9
 
 package local
 

backend/local/local.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"runtime"
 	"strings"
 	"sync"
@@ -144,17 +145,6 @@ the OS zeros the file. However sparse files may be undesirable as they
 cause disk fragmentation and can be slow to work with.`,
 			Default:  false,
 			Advanced: true,
-		}, {
-			Name: "no_set_modtime",
-			Help: `Disable setting modtime
-
-Normally rclone updates modification time of files after they are done
-uploading. This can cause permissions issues on Linux platforms when 
-the user rclone is running as does not own the file uploaded, such as
-when copying to a CIFS mount owned by another user. If this option is 
-enabled, rclone will no longer update the modtime after copying a file.`,
-			Default:  false,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -177,7 +167,6 @@ type Options struct {
 	CaseSensitive     bool                 `config:"case_sensitive"`
 	CaseInsensitive   bool                 `config:"case_insensitive"`
 	NoSparse          bool                 `config:"no_sparse"`
-	NoSetModTime      bool                 `config:"no_set_modtime"`
 	Enc               encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -194,22 +183,20 @@ type Fs struct {
 	warned      map[string]struct{} // whether we have warned about this string
 
 	// do os.Lstat or os.Stat
-	lstat        func(name string) (os.FileInfo, error)
-	objectMetaMu sync.RWMutex // global lock for Object metadata
+	lstat          func(name string) (os.FileInfo, error)
+	objectHashesMu sync.Mutex // global lock for Object.hashes
 }
 
 // Object represents a local filesystem object
 type Object struct {
-	fs     *Fs    // The Fs this object is part of
-	remote string // The remote path (encoded path)
-	path   string // The local path (OS path)
-	// When using these items the fs.objectMetaMu must be held
-	size    int64 // file metadata - always present
-	mode    os.FileMode
-	modTime time.Time
-	hashes  map[hash.Type]string // Hashes
-	// these are read only and don't need the mutex held
-	translatedLink bool // Is this object a translated link
+	fs             *Fs    // The Fs this object is part of
+	remote         string // The remote path (encoded path)
+	path           string // The local path (OS path)
+	size           int64  // file metadata - always present
+	mode           os.FileMode
+	modTime        time.Time
+	hashes         map[hash.Type]string // Hashes
+	translatedLink bool                 // Is this object a translated link
 }
 
 // ------------------------------------------------------------
@@ -244,7 +231,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		CaseInsensitive:         f.caseInsensitive(),
 		CanHaveEmptyDirectories: true,
 		IsLocal:                 true,
-		SlowHash:                true,
 	}).Fill(f)
 	if opt.FollowSymlinks {
 		f.lstat = os.Stat
@@ -554,10 +540,6 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 
 // Precision of the file system
 func (f *Fs) Precision() (precision time.Duration) {
-	if f.opt.NoSetModTime {
-		return fs.ModTimeNotSupported
-	}
-
 	f.precisionOk.Do(func() {
 		f.precision = f.readPrecision()
 	})
@@ -616,25 +598,20 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 	return
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and directories
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	dir = f.localPath(dir)
-	fi, err := f.lstat(dir)
+func (f *Fs) Purge(ctx context.Context) error {
+	fi, err := f.lstat(f.root)
 	if err != nil {
-		// already purged
-		if os.IsNotExist(err) {
-			return fs.ErrorDirNotFound
-		}
 		return err
 	}
 	if !fi.Mode().IsDir() {
-		return errors.Errorf("can't purge non directory: %q", dir)
+		return errors.Errorf("can't purge non directory: %q", f.root)
 	}
-	return os.RemoveAll(dir)
+	return os.RemoveAll(f.root)
 }
 
 // Move src to this remote using server side move operations.
@@ -655,9 +632,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 
 	// Temporary Object under construction
 	dstObj := f.newObject(remote)
-	dstObj.fs.objectMetaMu.RLock()
-	dstObjMode := dstObj.mode
-	dstObj.fs.objectMetaMu.RUnlock()
 
 	// Check it is a file if it exists
 	err := dstObj.lstat()
@@ -665,7 +639,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.fs.isRegular(dstObjMode) {
+	} else if !dstObj.fs.isRegular(dstObj.mode) {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -819,10 +793,8 @@ func (o *Object) Remote() string {
 // Hash returns the requested hash of a file as a lowercase hex string
 func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 	// Check that the underlying file hasn't changed
-	o.fs.objectMetaMu.RLock()
 	oldtime := o.modTime
 	oldsize := o.size
-	o.fs.objectMetaMu.RUnlock()
 	err := o.lstat()
 	var changed bool
 	if err != nil {
@@ -834,16 +806,15 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 			return "", errors.Wrap(err, "hash: failed to stat")
 		}
 	} else {
-		o.fs.objectMetaMu.RLock()
 		changed = !o.modTime.Equal(oldtime) || oldsize != o.size
-		o.fs.objectMetaMu.RUnlock()
 	}
 
-	o.fs.objectMetaMu.RLock()
+	o.fs.objectHashesMu.Lock()
+	hashes := o.hashes
 	hashValue, hashFound := o.hashes[r]
-	o.fs.objectMetaMu.RUnlock()
+	o.fs.objectHashesMu.Unlock()
 
-	if changed || !hashFound {
+	if changed || hashes == nil || !hashFound {
 		var in io.ReadCloser
 
 		if !o.translatedLink {
@@ -862,7 +833,6 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
-		var hashes map[hash.Type]string
 		hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
 		closeErr := in.Close()
 		if err != nil {
@@ -872,36 +842,29 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
 			return "", errors.Wrap(closeErr, "hash: failed to close")
 		}
 		hashValue = hashes[r]
-		o.fs.objectMetaMu.Lock()
+		o.fs.objectHashesMu.Lock()
 		if o.hashes == nil {
 			o.hashes = hashes
 		} else {
 			o.hashes[r] = hashValue
 		}
-		o.fs.objectMetaMu.Unlock()
+		o.fs.objectHashesMu.Unlock()
 	}
 	return hashValue, nil
 }
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	o.fs.objectMetaMu.RLock()
-	defer o.fs.objectMetaMu.RUnlock()
 	return o.size
 }
 
 // ModTime returns the modification time of the object
 func (o *Object) ModTime(ctx context.Context) time.Time {
-	o.fs.objectMetaMu.RLock()
-	defer o.fs.objectMetaMu.RUnlock()
 	return o.modTime
 }
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
-	if o.fs.opt.NoSetModTime {
-		return nil
-	}
 	var err error
 	if o.translatedLink {
 		err = lChtimes(o.path, modTime, modTime)
@@ -917,9 +880,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
 
 // Storable returns a boolean showing if this object is storable
 func (o *Object) Storable() bool {
-	o.fs.objectMetaMu.RLock()
 	mode := o.mode
-	o.fs.objectMetaMu.RUnlock()
 	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
@@ -952,15 +913,11 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
 		if err != nil {
 			return 0, errors.Wrap(err, "can't read status of source file while transferring")
 		}
-		file.o.fs.objectMetaMu.RLock()
-		oldtime := file.o.modTime
-		oldsize := file.o.size
-		file.o.fs.objectMetaMu.RUnlock()
-		if oldsize != fi.Size() {
-			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", oldsize, fi.Size()))
+		if file.o.size != fi.Size() {
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size()))
 		}
-		if !oldtime.Equal(fi.ModTime()) {
-			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", oldtime, fi.ModTime()))
+		if !file.o.modTime.Equal(fi.ModTime()) {
+			return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime()))
 		}
 	}
 
@@ -977,9 +934,9 @@ func (file *localOpenFile) Close() (err error) {
 	err = file.in.Close()
 	if err == nil {
 		if file.hash.Size() == file.o.Size() {
-			file.o.fs.objectMetaMu.Lock()
+			file.o.fs.objectHashesMu.Lock()
 			file.o.hashes = file.hash.Sums()
-			file.o.fs.objectMetaMu.Unlock()
+			file.o.fs.objectHashesMu.Unlock()
 		}
 	}
 	return err
@@ -1004,7 +961,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		case *fs.SeekOption:
 			offset = x.Offset
 		case *fs.RangeOption:
-			offset, limit = x.Decode(o.Size())
+			offset, limit = x.Decode(o.size)
 		case *fs.HashesOption:
 			if x.Hashes.Count() > 0 {
 				hasher, err = hash.NewMultiHasherTypes(x.Hashes)
@@ -1162,9 +1119,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// All successful so update the hashes
 	if hasher != nil {
-		o.fs.objectMetaMu.Lock()
+		o.fs.objectHashesMu.Lock()
 		o.hashes = hasher.Sums()
-		o.fs.objectMetaMu.Unlock()
+		o.fs.objectHashesMu.Unlock()
 	}
 
 	// Set the mtime
@@ -1213,7 +1170,7 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
 		// Set the file to be a sparse file (important on Windows)
 		err = file.SetSparse(out)
 		if err != nil {
-			fs.Errorf(o, "Failed to set sparse: %v", err)
+			fs.Debugf(o, "Failed to set sparse: %v", err)
 		}
 	}
 
@@ -1226,19 +1183,16 @@ func (o *Object) setMetadata(info os.FileInfo) {
 	if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
 		return
 	}
-	o.fs.objectMetaMu.Lock()
-	o.size = info.Size()
-	o.modTime = info.ModTime()
-	o.mode = info.Mode()
-	o.fs.objectMetaMu.Unlock()
-	// On Windows links read as 0 size so set the correct size here
-	if runtime.GOOS == "windows" && o.translatedLink {
-		linkdst, err := os.Readlink(o.path)
-		if err != nil {
-			fs.Errorf(o, "Failed to read link size: %v", err)
-		} else {
-			o.size = int64(len(linkdst))
-		}
+	// Don't overwrite the info if we don't need to
+	// this avoids upsetting the race detector
+	if o.size != info.Size() {
+		o.size = info.Size()
+	}
+	if !o.modTime.Equal(info.ModTime()) {
+		o.modTime = info.ModTime()
+	}
+	if o.mode != info.Mode() {
+		o.mode = info.Mode()
 	}
 }
 
@@ -1271,7 +1225,7 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 
 		if !noUNC {
 			// Convert to UNC
-			s = file.UNCPath(s)
+			s = uncPath(s)
 		}
 		return s
 	}
@@ -1285,6 +1239,28 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
 	return s
 }
 
+// Pattern to match a windows absolute path: "c:\" and similar
+var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
+
+// uncPath converts an absolute Windows path
+// to a UNC long path.
+func uncPath(l string) string {
+	// If prefix is "\\", we already have a UNC path or server.
+	if strings.HasPrefix(l, `\\`) {
+		// If already long path, just keep it
+		if strings.HasPrefix(l, `\\?\`) {
+			return l
+		}
+
+		// Trim "\\" from path and add UNC prefix.
+		return `\\?\UNC\` + strings.TrimPrefix(l, `\\`)
+	}
+	if isAbsWinDrive.MatchString(l) {
+		return `\\?\` + l
+	}
+	return l
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs             = &Fs{}

backend/local/local_internal_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -88,6 +89,9 @@ func TestSymlink(t *testing.T) {
 
 	// Object viewed as symlink
 	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
+	if runtime.GOOS == "windows" {
+		file2.Size = 0 // symlinks are 0 length under Windows
+	}
 
 	// Object viewed as destination
 	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
@@ -117,6 +121,9 @@ func TestSymlink(t *testing.T) {
 	// Create a symlink
 	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
 	file3 := r.WriteObjectTo(ctx, r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
+	if runtime.GOOS == "windows" {
+		file3.Size = 0 // symlinks are 0 length under Windows
+	}
 	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
 	if haveLChtimes {
 		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
@@ -135,7 +142,9 @@ func TestSymlink(t *testing.T) {
 	o, err := r.Flocal.NewObject(ctx, "symlink2.txt"+linkSuffix)
 	require.NoError(t, err)
 	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
-	assert.Equal(t, int64(8), o.Size())
+	if runtime.GOOS != "windows" {
+		assert.Equal(t, int64(8), o.Size())
+	}
 
 	// Check that NewObject doesn't see the non suffixed version
 	_, err = r.Flocal.NewObject(ctx, "symlink2.txt")

backend/local/tests_test.go

@@ -5,6 +5,49 @@ import (
 	"testing"
 )
 
+var uncTestPaths = []string{
+	`C:\Ba*d\P|a?t<h>\Windows\Folder`,
+	`C:\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\server\share\Desktop`,
+	`\\?\UNC\\share\folder\Desktop`,
+	`\\server\share`,
+}
+
+var uncTestPathsResults = []string{
+	`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\C:\Windows\Folder`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\UNC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
+	`\\?\UNC\server\share\Desktop`,
+	`\\?\UNC\\share\folder\Desktop`,
+	`\\?\UNC\server\share`,
+}
+
+// Test that UNC paths are converted.
+func TestUncPaths(t *testing.T) {
+	for i, p := range uncTestPaths {
+		unc := uncPath(p)
+		if unc != uncTestPathsResults[i] {
+			t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
+		}
+		// Test we don't add more.
+		unc = uncPath(unc)
+		if unc != uncTestPathsResults[i] {
+			t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
+		}
+	}
+}
+
 // Test Windows character replacements
 var testsWindows = [][2]string{
 	{`c:\temp`, `c:\temp`},

backend/mailru/api/m1.go

@@ -117,7 +117,7 @@ type ListItem struct {
 	Name      string `json:"name"`
 	Home      string `json:"home"`
 	Size      int64  `json:"size"`
-	Mtime     uint64 `json:"mtime,omitempty"`
+	Mtime     int64  `json:"mtime,omitempty"`
 	Hash      string `json:"hash,omitempty"`
 	VirusScan string `json:"virus_scan,omitempty"`
 	Tree      string `json:"tree,omitempty"`
@@ -159,6 +159,71 @@ type FolderInfoResponse struct {
 	Email  string `json:"email"`
 }
 
+// ShardInfoResponse ...
+type ShardInfoResponse struct {
+	Email string `json:"email"`
+	Body  struct {
+		Video []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"video"`
+		ViewDirect []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"view_direct"`
+		WeblinkView []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_view"`
+		WeblinkVideo []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_video"`
+		WeblinkGet []struct {
+			Count int    `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_get"`
+		Stock []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"stock"`
+		WeblinkThumbnails []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"weblink_thumbnails"`
+		PublicUpload []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"public_upload"`
+		Auth []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"auth"`
+		Web []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"web"`
+		View []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"view"`
+		Upload []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"upload"`
+		Get []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"get"`
+		Thumbnails []struct {
+			Count string `json:"count"`
+			URL   string `json:"url"`
+		} `json:"thumbnails"`
+	} `json:"body"`
+	Time   int64 `json:"time"`
+	Status int   `json:"status"`
+}
+
 // CleanupResponse ...
 type CleanupResponse struct {
 	Email     string `json:"email"`

backend/mailru/mailru.go

@@ -37,7 +37,6 @@ import (
 	"github.com/rclone/rclone/lib/encoder"
 	"github.com/rclone/rclone/lib/oauthutil"
 	"github.com/rclone/rclone/lib/pacer"
-	"github.com/rclone/rclone/lib/readers"
 	"github.com/rclone/rclone/lib/rest"
 
 	"github.com/pkg/errors"
@@ -192,7 +191,7 @@ This option must not be used by an ordinary user. It is intended only to
 facilitate remote troubleshooting of backend issues. Strict meaning of
 flags is not documented and not guaranteed to persist between releases.
 Quirks will be removed when the backend grows stable.
-Supported quirks: atomicmkdir binlist`,
+Supported quirks: atomicmkdir binlist gzip insecure retry400`,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -238,6 +237,9 @@ func shouldRetry(res *http.Response, err error, f *Fs, opts *rest.Opts) (bool, e
 		reAuthErr := f.reAuthorize(opts, err)
 		return reAuthErr == nil, err // return an original error
 	}
+	if res != nil && res.StatusCode == 400 && f.quirks.retry400 {
+		return true, err
+	}
 	return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(res, retryErrorCodes), err
 }
 
@@ -273,7 +275,7 @@ type Fs struct {
 	root         string             // root path
 	opt          Options            // parsed options
 	speedupGlobs []string           // list of file name patterns eligible for speedup
-	speedupAny   bool               // true if all file names are eligible for speedup
+	speedupAny   bool               // true if all file names are aligible for speedup
 	features     *fs.Features       // optional features
 	srv          *rest.Client       // REST API client
 	cli          *http.Client       // underlying HTTP client (for authorize)
@@ -339,7 +341,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if opt.UserAgent != "" {
 		clientConfig.UserAgent = opt.UserAgent
 	}
-	clientConfig.NoGzip = true // Mimic official client, skip sending "Accept-Encoding: gzip"
+	clientConfig.NoGzip = !f.quirks.gzip // Send not "Accept-Encoding: gzip" like official client
 	f.cli = fshttp.NewClient(&clientConfig)
 
 	f.srv = rest.NewClient(f.cli)
@@ -347,6 +349,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f.srv.SetHeader("Accept", "*/*") // Send "Accept: */*" with every request like official client
 	f.srv.SetErrorHandler(errorHandler)
 
+	if f.quirks.insecure {
+		transport := f.cli.Transport.(*fshttp.Transport).Transport
+		transport.TLSClientConfig.InsecureSkipVerify = true
+		transport.ProxyConnectHeader = http.Header{"User-Agent": {clientConfig.UserAgent}}
+	}
+
 	if err = f.authorize(ctx, false); err != nil {
 		return nil, err
 	}
@@ -379,13 +387,30 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 // Internal maintenance flags (to be removed when the backend matures).
 // Primarily intended to facilitate remote support and troubleshooting.
 type quirks struct {
+	gzip        bool
+	insecure    bool
 	binlist     bool
 	atomicmkdir bool
+	retry400    bool
 }
 
 func (q *quirks) parseQuirks(option string) {
 	for _, flag := range strings.Split(option, ",") {
 		switch strings.ToLower(strings.TrimSpace(flag)) {
+		case "gzip":
+			// This backend mimics the official client which never sends the
+			// "Accept-Encoding: gzip" header. However, enabling compression
+			// might be good for performance.
+			// Use this quirk to investigate the performance impact.
+			// Remove this quirk if performance does not improve.
+			q.gzip = true
+		case "insecure":
+			// The mailru disk-o protocol is not documented. To compare HTTP
+			// stream against the official client one can use Telerik Fiddler,
+			// which introduces a self-signed certificate. This quirk forces
+			// the Go http layer to accept it.
+			// Remove this quirk when the backend reaches maturity.
+			q.insecure = true
 		case "binlist":
 			// The official client sometimes uses a so called "bin" protocol,
 			// implemented in the listBin file system method below. This method
@@ -398,11 +423,18 @@ func (q *quirks) parseQuirks(option string) {
 		case "atomicmkdir":
 			// At the moment rclone requires Mkdir to return success if the
 			// directory already exists. However, such programs as borgbackup
-			// use mkdir as a locking primitive and depend on its atomicity.
-			// Remove this quirk when the above issue is investigated.
+			// or restic use mkdir as a locking primitive and depend on its
+			// atomicity. This quirk is a workaround. It can be removed
+			// when the above issue is investigated.
 			q.atomicmkdir = true
+		case "retry400":
+			// This quirk will help in troubleshooting a very rare "Error 400"
+			// issue. It can be removed if the problem does not show up
+			// for a year or so. See the below issue:
+			// https://github.com/ivandeex/rclone/issues/14
+			q.retry400 = true
 		default:
-			// Ignore unknown flags
+			// Just ignore all unknown flags
 		}
 	}
 }
@@ -623,14 +655,9 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
 	if err != nil {
 		return nil, -1, err
 	}
-	mTime := int64(item.Mtime)
-	if mTime < 0 {
-		fs.Debugf(f, "Fixing invalid timestamp %d on mailru file %q", mTime, remote)
-		mTime = 0
-	}
 	switch item.Kind {
 	case "folder":
-		dir := fs.NewDir(remote, time.Unix(mTime, 0)).SetSize(item.Size)
+		dir := fs.NewDir(remote, time.Unix(item.Mtime, 0)).SetSize(item.Size)
 		dirSize := item.Count.Files + item.Count.Folders
 		return dir, dirSize, nil
 	case "file":
@@ -644,7 +671,7 @@ func (f *Fs) itemToDirEntry(ctx context.Context, item *api.ListItem) (entry fs.D
 			hasMetaData: true,
 			size:        item.Size,
 			mrHash:      binHash,
-			modTime:     time.Unix(mTime, 0),
+			modTime:     time.Unix(item.Mtime, 0),
 		}
 		return file, -1, nil
 	default:
@@ -1135,12 +1162,12 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return f.purgeWithCheck(ctx, dir, true, "rmdir")
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the root directory
 // Optional interface: Only implement this if you have a way of deleting
 // all the files quicker than just running Remove() on the result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
 	// fs.Debugf(f, ">>> Purge")
-	return f.purgeWithCheck(ctx, dir, false, "purge")
+	return f.purgeWithCheck(ctx, "", false, "purge")
 }
 
 // purgeWithCheck() removes the root directory.
@@ -1834,30 +1861,30 @@ func (f *Fs) uploadShard(ctx context.Context) (string, error) {
 		return f.shardURL, nil
 	}
 
-	opts := rest.Opts{
-		RootURL: api.DispatchServerURL,
-		Method:  "GET",
-		Path:    "/u",
-	}
-
-	var (
-		res *http.Response
-		url string
-		err error
-	)
-	err = f.pacer.Call(func() (bool, error) {
-		res, err = f.srv.Call(ctx, &opts)
-		if err == nil {
-			url, err = readBodyWord(res)
-		}
-		return fserrors.ShouldRetry(err), err
-	})
+	token, err := f.accessToken()
 	if err != nil {
-		closeBody(res)
 		return "", err
 	}
 
-	f.shardURL = url
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "/api/m1/dispatcher",
+		Parameters: url.Values{
+			"client_id":    {api.OAuthClientID},
+			"access_token": {token},
+		},
+	}
+
+	var info api.ShardInfoResponse
+	err = f.pacer.Call(func() (bool, error) {
+		res, err := f.srv.CallJSON(ctx, &opts, nil, &info)
+		return shouldRetry(res, err, f, &opts)
+	})
+	if err != nil {
+		return "", err
+	}
+
+	f.shardURL = info.Body.Upload[0].URL
 	f.shardExpiry = time.Now().Add(shardExpirySec * time.Second)
 	fs.Debugf(f, "new upload shard: %s", f.shardURL)
 
@@ -2089,18 +2116,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, err
 	}
 
-	start, end, partialRequest := getTransferRange(o.size, options...)
-
-	headers := map[string]string{
-		"Accept":       "*/*",
-		"Content-Type": "application/octet-stream",
-	}
-	if partialRequest {
-		rangeStr := fmt.Sprintf("bytes=%d-%d", start, end-1)
-		headers["Range"] = rangeStr
-		// headers["Content-Range"] = rangeStr
-		headers["Accept-Ranges"] = "bytes"
-	}
+	start, end, partial := getTransferRange(o.size, options...)
 
 	// TODO: set custom timeouts
 	opts := rest.Opts{
@@ -2111,7 +2127,10 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 			"client_id": {api.OAuthClientID},
 			"token":     {token},
 		},
-		ExtraHeaders: headers,
+		ExtraHeaders: map[string]string{
+			"Accept": "*/*",
+			"Range":  fmt.Sprintf("bytes=%d-%d", start, end-1),
+		},
 	}
 
 	var res *http.Response
@@ -2132,36 +2151,18 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
 		return nil, err
 	}
 
-	// Server should respond with Status 206 and Content-Range header to a range
-	// request. Status 200 (and no Content-Range) means a full-content response.
-	partialResponse := res.StatusCode == 206
-
-	var (
-		hasher     gohash.Hash
-		wrapStream io.ReadCloser
-	)
-	if !partialResponse {
+	var hasher gohash.Hash
+	if !partial {
 		// Cannot check hash of partial download
 		hasher = mrhash.New()
 	}
-	wrapStream = &endHandler{
+	wrapStream := &endHandler{
 		ctx:    ctx,
 		stream: res.Body,
 		hasher: hasher,
 		o:      o,
 		server: server,
 	}
-	if partialRequest && !partialResponse {
-		fs.Debugf(o, "Server returned full content instead of range")
-		if start > 0 {
-			// Discard the beginning of the data
-			_, err = io.CopyN(ioutil.Discard, wrapStream, start)
-			if err != nil {
-				return nil, err
-			}
-		}
-		wrapStream = readers.NewLimitedReadCloser(wrapStream, end-start)
-	}
 	return wrapStream, nil
 }
 
@@ -2214,7 +2215,7 @@ func (e *endHandler) handle(err error) error {
 	return io.EOF
 }
 
-// serverPool backs server dispatcher
+// serverPool backs server dispacher
 type serverPool struct {
 	pool      pendingServerMap
 	mu        sync.Mutex

backend/mega/mega.go

@@ -669,13 +669,13 @@ func (f *Fs) Precision() time.Duration {
 	return fs.ModTimeNotSupported
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck("", false)
 }
 
 // move a file or folder (srcFs, srcRemote, info) to (f, dstRemote)

backend/memory/memory.go

@@ -221,7 +221,7 @@ func (f *Fs) setRoot(root string) {
 	f.rootBucket, f.rootDirectory = bucket.Split(f.root)
 }
 
-// NewFs constructs an Fs from the path, bucket:path
+// NewFs contstructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)

backend/onedrive/api/types.go

@@ -254,7 +254,7 @@ type MoveItemRequest struct {
 //Always Type:view and Scope:anonymous for public sharing
 type CreateShareLinkRequest struct {
 	Type  string `json:"type"`            //Link type in View, Edit or Embed
-	Scope string `json:"scope,omitempty"` //Optional. Scope in anonymous, organization
+	Scope string `json:"scope,omitempty"` //Optional. Scope in anonymousi, organization
 }
 
 //CreateShareLinkResponse is the response from CreateShareLinkRequest
@@ -410,28 +410,3 @@ func (i *Item) GetParentReference() *ItemReference {
 func (i *Item) IsRemote() bool {
 	return i.RemoteItem != nil
 }
-
-// User details for each version
-type User struct {
-	Email       string `json:"email"`
-	ID          string `json:"id"`
-	DisplayName string `json:"displayName"`
-}
-
-// LastModifiedBy for each version
-type LastModifiedBy struct {
-	User User `json:"user"`
-}
-
-// Version info
-type Version struct {
-	ID                   string         `json:"id"`
-	LastModifiedDateTime time.Time      `json:"lastModifiedDateTime"`
-	Size                 int            `json:"size"`
-	LastModifiedBy       LastModifiedBy `json:"lastModifiedBy"`
-}
-
-// VersionsResponse is returned from /versions
-type VersionsResponse struct {
-	Versions []Version `json:"value"`
-}

backend/onedrive/onedrive.go

@@ -14,7 +14,6 @@ import (
 	"path"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -27,8 +26,6 @@ import (
 	"github.com/rclone/rclone/fs/config/obscure"
 	"github.com/rclone/rclone/fs/fserrors"
 	"github.com/rclone/rclone/fs/hash"
-	"github.com/rclone/rclone/fs/operations"
-	"github.com/rclone/rclone/fs/walk"
 	"github.com/rclone/rclone/lib/atexit"
 	"github.com/rclone/rclone/lib/dircache"
 	"github.com/rclone/rclone/lib/encoder"
@@ -241,7 +238,13 @@ func init() {
 			m.Set(configDriveType, rootItem.ParentReference.DriveType)
 			config.SaveConfig()
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Microsoft App Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Microsoft App Client Secret\nLeave blank normally.",
+		}, {
 			Name: "chunk_size",
 			Help: `Chunk size to upload files with - must be multiple of 320k (327,680 bytes).
 
@@ -281,23 +284,6 @@ different Onedrives.  Note that this isn't enabled by default
 because it isn't easy to tell if it will work between any two
 configurations.`,
 			Advanced: true,
-		}, {
-			Name:    "no_versions",
-			Default: false,
-			Help: `Remove all versions on modifying operations
-
-Onedrive for business creates versions when rclone uploads new files
-overwriting an existing one and when it sets the modification time.
-
-These versions take up space out of the quota.
-
-This flag checks for versions after file upload and setting
-modification time and removes all but the last version.
-
-**NB** Onedrive personal can't currently delete versions so don't use
-this flag there.
-`,
-			Advanced: true,
 		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
@@ -344,7 +330,7 @@ this flag there.
 				encoder.EncodeRightSpace |
 				encoder.EncodeWin |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
@@ -355,7 +341,6 @@ type Options struct {
 	DriveType               string               `config:"drive_type"`
 	ExposeOneNoteFiles      bool                 `config:"expose_onenote_files"`
 	ServerSideAcrossConfigs bool                 `config:"server_side_across_configs"`
-	NoVersions              bool                 `config:"no_versions"`
 	Enc                     encoder.MultiEncoder `config:"encoding"`
 }
 
@@ -1088,13 +1073,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return dstObj, nil
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // Move src to this remote using server side move operations.
@@ -1122,10 +1107,9 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	id, dstDriveID, _ := parseNormalizedID(directoryID)
 	_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
 
-	if f.canonicalDriveID(dstDriveID) != srcObj.fs.canonicalDriveID(srcObjDriveID) {
+	if dstDriveID != srcObjDriveID {
 		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
 		// "Items cannot be moved between Drives using this request."
-		fs.Debugf(f, "Can't move files between drives (%q != %q)", dstDriveID, srcObjDriveID)
 		return nil, fs.ErrorCantMove
 	}
 
@@ -1184,10 +1168,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
 	_, srcDriveID, _ := parseNormalizedID(srcID)
 
-	if f.canonicalDriveID(dstDriveID) != srcFs.canonicalDriveID(srcDriveID) {
+	if dstDriveID != srcDriveID {
 		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
 		// "Items cannot be moved between Drives using this request."
-		fs.Debugf(f, "Can't move directories between drives (%q != %q)", dstDriveID, srcDriveID)
 		return fs.ErrorCantDirMove
 	}
 
@@ -1247,10 +1230,6 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
 		return nil, errors.Wrap(err, "about failed")
 	}
 	q := drive.Quota
-	// On (some?) Onedrive sharepoints these are all 0 so return unknown in that case
-	if q.Total == 0 && q.Used == 0 && q.Deleted == 0 && q.Remaining == 0 {
-		return &fs.Usage{}, nil
-	}
 	usage = &fs.Usage{
 		Total:   fs.NewUsageValue(q.Total),     // quota of bytes that can be used
 		Used:    fs.NewUsageValue(q.Used),      // bytes in use
@@ -1294,73 +1273,6 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
 	return result.Link.WebURL, nil
 }
 
-// CleanUp deletes all the hidden files.
-func (f *Fs) CleanUp(ctx context.Context) error {
-	token := make(chan struct{}, fs.Config.Checkers)
-	var wg sync.WaitGroup
-	err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error {
-		err = entries.ForObjectError(func(obj fs.Object) error {
-			o, ok := obj.(*Object)
-			if !ok {
-				return errors.New("internal error: not a onedrive object")
-			}
-			wg.Add(1)
-			token <- struct{}{}
-			go func() {
-				defer func() {
-					<-token
-					wg.Done()
-				}()
-				err := o.deleteVersions(ctx)
-				if err != nil {
-					fs.Errorf(o, "Failed to remove versions: %v", err)
-				}
-			}()
-			return nil
-		})
-		wg.Wait()
-		return err
-	})
-	return err
-}
-
-// Finds and removes any old versions for o
-func (o *Object) deleteVersions(ctx context.Context) error {
-	opts := newOptsCall(o.id, "GET", "/versions")
-	var versions api.VersionsResponse
-	err := o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions)
-		return shouldRetry(resp, err)
-	})
-	if err != nil {
-		return err
-	}
-	if len(versions.Versions) < 2 {
-		return nil
-	}
-	for _, version := range versions.Versions[1:] {
-		err = o.deleteVersion(ctx, version.ID)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// Finds and removes any old versions for o
-func (o *Object) deleteVersion(ctx context.Context, ID string) error {
-	if operations.SkipDestructive(ctx, fmt.Sprintf("%s of %s", ID, o.remote), "delete version") {
-		return nil
-	}
-	fs.Infof(o, "removing version %q", ID)
-	opts := newOptsCall(o.id, "DELETE", "/versions/"+ID)
-	opts.NoResponse = true
-	return o.fs.pacer.Call(func() (bool, error) {
-		resp, err := o.fs.srv.Call(ctx, &opts)
-		return shouldRetry(resp, err)
-	})
-}
-
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -1524,13 +1436,6 @@ func (o *Object) setModTime(ctx context.Context, modTime time.Time) (*api.Item,
 		resp, err := o.fs.srv.CallJSON(ctx, &opts, &update, &info)
 		return shouldRetry(resp, err)
 	})
-	// Remove versions if required
-	if o.fs.opt.NoVersions {
-		err := o.deleteVersions(ctx)
-		if err != nil {
-			fs.Errorf(o, "Failed to remove versions: %v", err)
-		}
-	}
 	return info, err
 }
 
@@ -1837,14 +1742,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		return err
 	}
 
-	// If updating the file then remove versions
-	if o.fs.opt.NoVersions && o.hasMetaData {
-		err = o.deleteVersions(ctx)
-		if err != nil {
-			fs.Errorf(o, "Failed to remove versions: %v", err)
-		}
-	}
-
 	return o.setMetaData(info)
 }
 
@@ -1890,17 +1787,6 @@ func parseNormalizedID(ID string) (string, string, string) {
 	return ID, "", ""
 }
 
-// Returns the canonical form of the driveID
-func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
-	if driveID == "" {
-		canonicalDriveID = f.opt.DriveID
-	} else {
-		canonicalDriveID = driveID
-	}
-	canonicalDriveID = strings.ToLower(canonicalDriveID)
-	return canonicalDriveID
-}
-
 // getRelativePathInsideBase checks if `target` is inside `base`. If so, it
 // returns a relative path for `target` based on `base` and a boolean `true`.
 // Otherwise returns "", false.
@@ -1941,7 +1827,6 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.PublicLinker    = (*Fs)(nil)
-	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = &Object{}
 	_ fs.IDer            = &Object{}

backend/opendrive/opendrive.go

@@ -506,13 +506,13 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 	return nil
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // Return an Object from a path
@@ -646,6 +646,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
 
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
+	400, // Bad request (seen in "Next token is expired")
 	401, // Unauthorized (seen in "Token has expired")
 	408, // Request Timeout
 	423, // Locked - get this on folders sometimes

backend/pcloud/pcloud.go

@@ -42,7 +42,7 @@ const (
 	minSleep                    = 10 * time.Millisecond
 	maxSleep                    = 2 * time.Second
 	decayConstant               = 2 // bigger for slower decay, exponential
-	defaultHostname             = "api.pcloud.com"
+	rootURL                     = "https://api.pcloud.com"
 )
 
 // Globals
@@ -51,8 +51,8 @@ var (
 	oauthConfig = &oauth2.Config{
 		Scopes: nil,
 		Endpoint: oauth2.Endpoint{
-			AuthURL: "https://my.pcloud.com/oauth2/authorize",
-			// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
+			AuthURL:  "https://my.pcloud.com/oauth2/authorize",
+			TokenURL: "https://api.pcloud.com/oauth2_token",
 		},
 		ClientID:     rcloneClientID,
 		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
@@ -60,50 +60,28 @@ var (
 	}
 )
 
-// Update the TokenURL with the actual hostname
-func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
-	oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
-}
-
 // Register with Fs
 func init() {
-	updateTokenURL(oauthConfig, defaultHostname)
 	fs.Register(&fs.RegInfo{
 		Name:        "pcloud",
 		Description: "Pcloud",
 		NewFs:       NewFs,
 		Config: func(name string, m configmap.Mapper) {
-			optc := new(Options)
-			err := configstruct.Set(m, optc)
-			if err != nil {
-				fs.Errorf(nil, "Failed to read config: %v", err)
-			}
-			updateTokenURL(oauthConfig, optc.Hostname)
-			checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
-				if auth == nil || auth.Form == nil {
-					return errors.New("form not found in response")
-				}
-				hostname := auth.Form.Get("hostname")
-				if hostname == "" {
-					hostname = defaultHostname
-				}
-				// Save the hostname in the config
-				m.Set("hostname", hostname)
-				// Update the token URL
-				updateTokenURL(oauthConfig, hostname)
-				fs.Debugf(nil, "pcloud: got hostname %q", hostname)
-				return nil
-			}
 			opt := oauthutil.Options{
-				CheckAuth:    checkAuth,
 				StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
 			}
-			err = oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
+			err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
 			if err != nil {
 				log.Fatalf("Failed to configure token: %v", err)
 			}
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Pcloud App Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Pcloud App Client Secret\nLeave blank normally.",
+		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
@@ -118,24 +96,7 @@ func init() {
 			Help:     "Fill in for rclone to use a non root folder as its starting point.",
 			Default:  "d0",
 			Advanced: true,
-		}, {
-			Name: "hostname",
-			Help: `Hostname to connect to.
-
-This is normally set when rclone initially does the oauth connection,
-however you will need to set it by hand if you are using remote config
-with rclone authorize.
-`,
-			Default:  defaultHostname,
-			Advanced: true,
-			Examples: []fs.OptionExample{{
-				Value: defaultHostname,
-				Help:  "Original/US region",
-			}, {
-				Value: "eapi.pcloud.com",
-				Help:  "EU region",
-			}},
-		}}...),
+		}},
 	})
 }
 
@@ -143,7 +104,6 @@ with rclone authorize.
 type Options struct {
 	Enc          encoder.MultiEncoder `config:"encoding"`
 	RootFolderID string               `config:"root_folder_id"`
-	Hostname     string               `config:"hostname"`
 }
 
 // Fs represents a remote pcloud
@@ -293,13 +253,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to configure Pcloud")
 	}
-	updateTokenURL(oauthConfig, opt.Hostname)
 
 	f := &Fs{
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		srv:   rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),
+		srv:   rest.NewClient(oAuthClient).SetRoot(rootURL),
 		pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
 	}
 	f.features = (&fs.Features{
@@ -675,13 +634,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return dstObj, nil
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // CleanUp empties the trash
@@ -824,19 +783,14 @@ func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (str
 }
 
 func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (string, error) {
-	obj, err := f.NewObject(ctx, path)
-	if err != nil {
-		return "", err
-	}
-	o := obj.(*Object)
 	opts := rest.Opts{
 		Method:     "POST",
 		Path:       "/getfilepublink",
 		Parameters: url.Values{},
 	}
 	var result api.PubLinkResult
-	opts.Parameters.Set("fileid", fileIDtoNumber(o.id))
-	err = f.pacer.Call(func() (bool, error) {
+	opts.Parameters.Set("path", path)
+	err := f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
 		err = result.Error.Update(err)
 		return shouldRetry(resp, err)
@@ -849,6 +803,11 @@ func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (str
 
 // PublicLink adds a "readable by anyone with link" permission on the given file or folder.
 func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
+	err := f.dirCache.FindRoot(ctx, false)
+	if err != nil {
+		return "", err
+	}
+
 	dirID, err := f.dirCache.FindDir(ctx, remote, false)
 	if err == fs.ErrorDirNotFound {
 		return f.linkFile(ctx, remote, expire)
@@ -1125,7 +1084,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 
 	// Special treatment for a 0 length upload.  This doesn't work
 	// with PUT even with Content-Length set (by setting
-	// opts.Body=0), so upload it as a multipart form POST with
+	// opts.Body=0), so upload it as a multpart form POST with
 	// Content-Length set.
 	if size == 0 {
 		formReader, contentType, overhead, err := rest.MultipartUpload(in, opts.Parameters, "content", leaf)

backend/premiumizeme/premiumizeme.go

@@ -609,13 +609,13 @@ func (f *Fs) Precision() time.Duration {
 	return fs.ModTimeNotSupported
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // move a file or folder

backend/putio/fs.go

@@ -236,10 +236,10 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // The new object may have been created if an error is returned
 func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (o fs.Object, err error) {
 	// defer log.Trace(f, "src=%+v", src)("o=%+v, err=%v", &o, &err)
-	existingObj, err := f.NewObject(ctx, src.Remote())
+	exisitingObj, err := f.NewObject(ctx, src.Remote())
 	switch err {
 	case nil:
-		return existingObj, existingObj.Update(ctx, in, src, options...)
+		return exisitingObj, exisitingObj.Update(ctx, in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(ctx, in, src, options...)
@@ -458,9 +458,10 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
 	return err
 }
 
-// purgeCheck removes the root directory, if check is set then it
-// refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error) {
+// Rmdir deletes the container
+//
+// Returns an error if it isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
 	// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
 
 	root := strings.Trim(path.Join(f.root, dir), "/")
@@ -477,20 +478,18 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	}
 	dirID := atoi(directoryID)
 
-	if check {
-		// check directory empty
-		var children []putio.File
-		err = f.pacer.Call(func() (bool, error) {
-			// fs.Debugf(f, "listing files: %d", dirID)
-			children, _, err = f.client.Files.List(ctx, dirID)
-			return shouldRetry(err)
-		})
-		if err != nil {
-			return errors.Wrap(err, "Rmdir")
-		}
-		if len(children) != 0 {
-			return errors.New("directory not empty")
-		}
+	// check directory empty
+	var children []putio.File
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "listing files: %d", dirID)
+		children, _, err = f.client.Files.List(ctx, dirID)
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	if len(children) != 0 {
+		return errors.New("directory not empty")
 	}
 
 	// remove it
@@ -503,26 +502,35 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) (err error)
 	return err
 }
 
-// Rmdir deletes the container
-//
-// Returns an error if it isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) (err error) {
-	return f.purgeCheck(ctx, dir, true)
-}
-
 // Precision returns the precision
 func (f *Fs) Precision() time.Duration {
 	return time.Second
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) (err error) {
+func (f *Fs) Purge(ctx context.Context) (err error) {
 	// defer log.Trace(f, "")("err=%v", &err)
-	return f.purgeCheck(ctx, dir, false)
+
+	if f.root == "" {
+		return errors.New("can't purge root directory")
+	}
+	rootIDs, err := f.dirCache.RootID(ctx, false)
+	if err != nil {
+		return err
+	}
+	rootID := atoi(rootIDs)
+	// Let putio delete the filesystem tree
+	err = f.pacer.Call(func() (bool, error) {
+		// fs.Debugf(f, "deleting file: %d", rootID)
+		err = f.client.Files.Delete(ctx, rootID)
+		return shouldRetry(err)
+	})
+	f.dirCache.ResetRoot()
+	return err
 }
 
 // Copy src to this remote using server side copy operations.

backend/putio/object.go

@@ -115,7 +115,7 @@ func (o *Object) MimeType(ctx context.Context) string {
 
 // setMetadataFromEntry sets the fs data from a putio.File
 //
-// This isn't a complete set of metadata and has an inaccurate date
+// This isn't a complete set of metadata and has an inacurate date
 func (o *Object) setMetadataFromEntry(info putio.File) error {
 	o.file = &info
 	o.modtime = info.UpdatedAt.Time

backend/qingstor/qingstor.go

@@ -1,7 +1,7 @@
 // Package qingstor provides an interface to QingStor object storage
 // Home: https://www.qingcloud.com/
 
-// +build !plan9,!js
+// +build !plan9
 
 package qingstor
 
@@ -104,7 +104,7 @@ enough memory, then increasing this will speed up the transfers.`,
 This is the number of chunks of the same file that are uploaded
 concurrently.
 
-NB if you set this to > 1 then the checksums of multipart uploads
+NB if you set this to > 1 then the checksums of multpart uploads
 become corrupted (the uploads themselves are not corrupted though).
 
 If you are uploading small numbers of large file over high speed link
@@ -356,7 +356,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		WriteMimeType:     true,
 		BucketBased:       true,
 		BucketBasedRootOK: true,
-		SlowModTime:       true,
 	}).Fill(f)
 
 	if f.rootBucket != "" && f.rootDirectory != "" {

backend/qingstor/qingstor_test.go

@@ -1,6 +1,6 @@
 // Test QingStor filesystem interface
 
-// +build !plan9,!js
+// +build !plan9
 
 package qingstor
 

backend/qingstor/qingstor_unsupported.go

@@ -1,6 +1,6 @@
 // Build for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 js
+// +build plan9
 
 package qingstor

backend/qingstor/upload.go

@@ -1,6 +1,6 @@
 // Upload object to QingStor
 
-// +build !plan9,!js
+// +build !plan9
 
 package qingstor
 

backend/s3/v2sign.go

@@ -9,10 +9,7 @@ import (
 	"net/http"
 	"sort"
 	"strings"
-	"sync"
 	"time"
-
-	"github.com/rclone/rclone/fs"
 )
 
 // URL parameters that need to be added to the signature
@@ -36,20 +33,12 @@ var s3ParamsToSign = map[string]struct{}{
 	"response-cache-control":       {},
 	"response-content-disposition": {},
 	"response-content-encoding":    {},
-	"lifecycle":                    {},
-	"website":                      {},
-	"delete":                       {},
-	"cors":                         {},
-	"restore":                      {},
 }
 
-// Warn once about empty endpoint
-var warnEmptyEndpointOnce sync.Once
-
 // sign signs requests using v2 auth
 //
 // Cobbled together from goamz and aws-sdk-go
-func v2sign(opt *Options, req *http.Request) {
+func sign(AccessKey, SecretKey string, req *http.Request) {
 	// Set date
 	date := time.Now().UTC().Format(time.RFC1123)
 	req.Header.Set("Date", date)
@@ -59,26 +48,11 @@ func v2sign(opt *Options, req *http.Request) {
 	if uri == "" {
 		uri = "/"
 	}
-	// If not using path style then need to stick the bucket on
-	// the start of the requests if doing a bucket based query
-	if !opt.ForcePathStyle {
-		if opt.Endpoint == "" {
-			warnEmptyEndpointOnce.Do(func() {
-				fs.Logf(nil, "If using v2 auth with AWS and force_path_style=false, endpoint must be set in the config")
-			})
-		} else if req.URL.Host != opt.Endpoint {
-			// read the bucket off the start of the hostname
-			i := strings.IndexRune(req.URL.Host, '.')
-			if i >= 0 {
-				uri = "/" + req.URL.Host[:i] + uri
-			}
-		}
-	}
 
 	// Look through headers of interest
 	var md5 string
 	var contentType string
-	var headersToSign [][2]string // slice of key, value pairs
+	var headersToSign []string
 	for k, v := range req.Header {
 		k = strings.ToLower(k)
 		switch k {
@@ -89,26 +63,15 @@ func v2sign(opt *Options, req *http.Request) {
 		default:
 			if strings.HasPrefix(k, "x-amz-") {
 				vall := strings.Join(v, ",")
-				headersToSign = append(headersToSign, [2]string{k, vall})
+				headersToSign = append(headersToSign, k+":"+vall)
 			}
 		}
 	}
 	// Make headers of interest into canonical string
 	var joinedHeadersToSign string
 	if len(headersToSign) > 0 {
-		// sort by keys
-		sort.Slice(headersToSign, func(i, j int) bool {
-			return headersToSign[i][0] < headersToSign[j][0]
-		})
-		// join into key:value\n
-		var out strings.Builder
-		for _, kv := range headersToSign {
-			out.WriteString(kv[0])
-			out.WriteRune(':')
-			out.WriteString(kv[1])
-			out.WriteRune('\n')
-		}
-		joinedHeadersToSign = out.String()
+		sort.StringSlice(headersToSign).Sort()
+		joinedHeadersToSign = strings.Join(headersToSign, "\n") + "\n"
 	}
 
 	// Look for query parameters which need to be added to the signature
@@ -133,11 +96,11 @@ func v2sign(opt *Options, req *http.Request) {
 
 	// Make signature
 	payload := req.Method + "\n" + md5 + "\n" + contentType + "\n" + date + "\n" + joinedHeadersToSign + uri
-	hash := hmac.New(sha1.New, []byte(opt.SecretAccessKey))
+	hash := hmac.New(sha1.New, []byte(SecretKey))
 	_, _ = hash.Write([]byte(payload))
 	signature := make([]byte, base64.StdEncoding.EncodedLen(hash.Size()))
 	base64.StdEncoding.Encode(signature, hash.Sum(nil))
 
 	// Set signature in request
-	req.Header.Set("Authorization", "AWS "+opt.AccessKeyID+":"+string(signature))
+	req.Header.Set("Authorization", "AWS "+AccessKey+":"+string(signature))
 }

backend/seafile/api/types.go

@@ -46,7 +46,7 @@ type Library struct {
 	Encrypted bool   `json:"encrypted"`
 	Owner     string `json:"owner"`
 	ID        string `json:"id"`
-	Size      int64  `json:"size"`
+	Size      int    `json:"size"`
 	Name      string `json:"name"`
 	Modified  int64  `json:"mtime"`
 }

backend/seafile/seafile.go

@@ -584,38 +584,29 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return nil
 }
 
-// purgeCheck removes the root directory, if check is set then it
-// refuses to do so if it has anything in
-func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
+// Rmdir removes the directory or library if empty
+//
+// Return an error if it doesn't exist or isn't empty
+func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	libraryName, dirPath := f.splitPath(dir)
 	libraryID, err := f.getLibraryID(ctx, libraryName)
 	if err != nil {
 		return err
 	}
 
-	if check {
-		directoryEntries, err := f.getDirectoryEntries(ctx, libraryID, dirPath, false)
-		if err != nil {
-			return err
-		}
-		if len(directoryEntries) > 0 {
-			return fs.ErrorDirectoryNotEmpty
-		}
+	directoryEntries, err := f.getDirectoryEntries(ctx, libraryID, dirPath, false)
+	if err != nil {
+		return err
+	}
+	if len(directoryEntries) > 0 {
+		return fs.ErrorDirectoryNotEmpty
 	}
-
 	if dirPath == "" || dirPath == "/" {
 		return f.deleteLibrary(ctx, libraryID)
 	}
 	return f.deleteDir(ctx, libraryID, dirPath)
 }
 
-// Rmdir removes the directory or library if empty
-//
-// Return an error if it doesn't exist or isn't empty
-func (f *Fs) Rmdir(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, true)
-}
-
 // ==================== Optional Interface fs.ListRer ====================
 
 // ListR lists the objects and directories of the Fs starting
@@ -902,14 +893,33 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 
 // ==================== Optional Interface fs.Purger ====================
 
-// Purge all files in the directory
+// Purge all files in the root and the root directory
 //
 // Implement this if you have a way of deleting all the files
 // quicker than just running Remove() on the result of List()
 //
 // Return an error if it doesn't exist
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	if f.libraryName == "" {
+		return errors.New("Cannot delete from the root of the server. Please select a library")
+	}
+	libraryID, err := f.getLibraryID(ctx, f.libraryName)
+	if err != nil {
+		return err
+	}
+	if f.rootDirectory == "" {
+		// Delete library
+		err = f.deleteLibrary(ctx, libraryID)
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+	err = f.deleteDir(ctx, libraryID, f.rootDirectory)
+	if err != nil {
+		return err
+	}
+	return nil
 }
 
 // ==================== Optional Interface fs.CleanUpper ====================
@@ -1004,7 +1014,7 @@ func (f *Fs) listLibraries(ctx context.Context) (entries fs.DirEntries, err erro
 
 	for _, library := range libraries {
 		d := fs.NewDir(library.Name, time.Unix(library.Modified, 0))
-		d.SetSize(library.Size)
+		d.SetSize(int64(library.Size))
 		entries = append(entries, d)
 	}
 

backend/sftp/sftp.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"os/user"
 	"path"
 	"regexp"
 	"strconv"
@@ -32,7 +33,6 @@ import (
 	"github.com/rclone/rclone/lib/readers"
 	sshagent "github.com/xanzy/ssh-agent"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/crypto/ssh/knownhosts"
 )
 
 const (
@@ -43,7 +43,7 @@ const (
 )
 
 var (
-	currentUser = env.CurrentUser()
+	currentUser = readCurrentUser()
 )
 
 func init() {
@@ -82,21 +82,6 @@ func init() {
 Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
 in the new OpenSSH format can't be used.`,
 			IsPassword: true,
-		}, {
-			Name: "pubkey_file",
-			Help: `Optional path to public key file.
-
-Set this if you have a signed certificate you want to use for authentication.` + env.ShellExpandHelp,
-		}, {
-			Name: "known_hosts_file",
-			Help: `Optional path to known_hosts file.
-
-Set this value to enable server host key validation.` + env.ShellExpandHelp,
-			Advanced: true,
-			Examples: []fs.OptionExample{{
-				Value: "~/.ssh/known_hosts",
-				Help:  "Use OpenSSH's known_hosts file",
-			}},
 		}, {
 			Name: "key_use_agent",
 			Help: `When set forces the usage of the ssh-agent.
@@ -179,18 +164,6 @@ Home directory can be found in a shared folder called "home"
 			Default:  false,
 			Help:     "Set to skip any symlinks and any other non regular files.",
 			Advanced: true,
-		}, {
-			Name:     "subsystem",
-			Default:  "sftp",
-			Help:     "Specifies the SSH2 subsystem on the remote host.",
-			Advanced: true,
-		}, {
-			Name:    "server_command",
-			Default: "",
-			Help: `Specifies the path or command to run a sftp server on the remote host.
-
-The subsystem option is ignored when server_command is defined.`,
-			Advanced: true,
 		}},
 	}
 	fs.Register(fsi)
@@ -205,8 +178,6 @@ type Options struct {
 	KeyPem            string `config:"key_pem"`
 	KeyFile           string `config:"key_file"`
 	KeyFilePass       string `config:"key_file_pass"`
-	PubKeyFile        string `config:"pubkey_file"`
-	KnownHostsFile    string `config:"known_hosts_file"`
 	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
@@ -216,15 +187,12 @@ type Options struct {
 	Md5sumCommand     string `config:"md5sum_command"`
 	Sha1sumCommand    string `config:"sha1sum_command"`
 	SkipLinks         bool   `config:"skip_links"`
-	Subsystem         string `config:"subsystem"`
-	ServerCommand     string `config:"server_command"`
 }
 
 // Fs stores the interface to the remote SFTP files
 type Fs struct {
 	name         string
 	root         string
-	absRoot      string
 	opt          Options          // parsed options
 	m            configmap.Mapper // config
 	features     *fs.Features     // optional features
@@ -235,7 +203,6 @@ type Fs struct {
 	poolMu       sync.Mutex
 	pool         []*conn
 	pacer        *fs.Pacer // pacer for operations
-	savedpswd    string
 }
 
 // Object is a remote SFTP file that has been stat'd (so it exists, but is not necessarily open for reading)
@@ -249,6 +216,20 @@ type Object struct {
 	sha1sum *string     // Cached SHA1 checksum
 }
 
+// readCurrentUser finds the current user name or "" if not found
+func readCurrentUser() (userName string) {
+	usr, err := user.Current()
+	if err == nil {
+		return usr.Username
+	}
+	// Fall back to reading $USER then $LOGNAME
+	userName = os.Getenv("USER")
+	if userName != "" {
+		return userName
+	}
+	return os.Getenv("LOGNAME")
+}
+
 // dial starts a client connection to the given SSH server. It is a
 // convenience function that connects to the given network address,
 // initiates the SSH handshake, and then sets up a Client.
@@ -308,7 +289,7 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't connect SSH")
 	}
-	c.sftpClient, err = f.newSftpClient(c.sshClient)
+	c.sftpClient, err = sftp.NewClient(c.sshClient)
 	if err != nil {
 		_ = c.sshClient.Close()
 		return nil, errors.Wrap(err, "couldn't initialise SFTP")
@@ -317,35 +298,6 @@ func (f *Fs) sftpConnection() (c *conn, err error) {
 	return c, nil
 }
 
-// Creates a new SFTP client on conn, using the specified subsystem
-// or sftp server, and zero or more option functions
-func (f *Fs) newSftpClient(conn *ssh.Client, opts ...sftp.ClientOption) (*sftp.Client, error) {
-	s, err := conn.NewSession()
-	if err != nil {
-		return nil, err
-	}
-	pw, err := s.StdinPipe()
-	if err != nil {
-		return nil, err
-	}
-	pr, err := s.StdoutPipe()
-	if err != nil {
-		return nil, err
-	}
-
-	if f.opt.ServerCommand != "" {
-		if err := s.Start(f.opt.ServerCommand); err != nil {
-			return nil, err
-		}
-	} else {
-		if err := s.RequestSubsystem(f.opt.Subsystem); err != nil {
-			return nil, err
-		}
-	}
-
-	return sftp.NewClientPipe(pr, pw, opts...)
-}
-
 // Get an SFTP connection from the pool, or open a new one
 func (f *Fs) getSftpConnection() (c *conn, err error) {
 	f.poolMu.Lock()
@@ -414,10 +366,6 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
-	// This will hold the Fs object.  We need to create it here
-	// so we can refer to it in the SSH callback, but it's populated
-	// in NewFsWithConnection
-	f := &Fs{}
 	ctx := context.Background()
 	// Parse config into Options struct
 	opt := new(Options)
@@ -431,7 +379,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if opt.Port == "" {
 		opt.Port = "22"
 	}
-
 	sshConfig := &ssh.ClientConfig{
 		User:            opt.User,
 		Auth:            []ssh.AuthMethod{},
@@ -440,14 +387,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ClientVersion:   "SSH-2.0-" + fs.Config.UserAgent,
 	}
 
-	if opt.KnownHostsFile != "" {
-		hostcallback, err := knownhosts.New(opt.KnownHostsFile)
-		if err != nil {
-			return nil, errors.Wrap(err, "couldn't parse known_hosts_file")
-		}
-		sshConfig.HostKeyCallback = hostcallback
-	}
-
 	if opt.UseInsecureCipher {
 		sshConfig.Config.SetDefaults()
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc", "aes192-cbc", "aes256-cbc", "3des-cbc")
@@ -455,7 +394,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	keyFile := env.ShellExpand(opt.KeyFile)
-	pubkeyFile := env.ShellExpand(opt.PubKeyFile)
 	//keyPem := env.ShellExpand(opt.KeyPem)
 	// Add ssh agent-auth if no password or file or key PEM specified
 	if (opt.Pass == "" && keyFile == "" && !opt.AskPassword && opt.KeyPem == "") || opt.KeyUseAgent {
@@ -525,38 +463,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse private key file")
 		}
-
-		// If a public key has been specified then use that
-		if pubkeyFile != "" {
-			certfile, err := ioutil.ReadFile(pubkeyFile)
-			if err != nil {
-				return nil, errors.Wrap(err, "unable to read cert file")
-			}
-
-			pk, _, _, _, err := ssh.ParseAuthorizedKey(certfile)
-			if err != nil {
-				return nil, errors.Wrap(err, "unable to parse cert file")
-			}
-
-			// And the signer for this, which includes the private key signer
-			// This is what we'll pass to the ssh client.
-			// Normally the ssh client will use the public key built
-			// into the private key, but we need to tell it to use the user
-			// specified public key cert.  This signer is specific to the
-			// cert and will include the private key signer.  Now ssh
-			// knows everything it needs.
-			cert, ok := pk.(*ssh.Certificate)
-			if !ok {
-				return nil, errors.New("public key file is not a certificate file: " + pubkeyFile)
-			}
-			pubsigner, err := ssh.NewCertSigner(cert, signer)
-			if err != nil {
-				return nil, errors.Wrap(err, "error generating cert signer")
-			}
-			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(pubsigner))
-		} else {
-			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
-		}
+		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signer))
 	}
 
 	// Auth from password if specified
@@ -568,68 +475,42 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
 
-	// Config for password if none was defined and we're allowed to
-	// We don't ask now; we ask if the ssh connection succeeds
+	// Ask for password if none was defined and we're allowed to
 	if opt.Pass == "" && opt.AskPassword {
-		sshConfig.Auth = append(sshConfig.Auth, ssh.PasswordCallback(f.getPass))
-	}
-
-	return NewFsWithConnection(ctx, f, name, root, m, opt, sshConfig)
-}
-
-// If we're in password mode and ssh connection succeeds then this
-// callback is called.  First time around we ask the user, and then
-// save it so on reconnection we give back the previous string.
-// This removes the ability to let the user correct a mistaken entry,
-// but means that reconnects are transparent.
-// We'll re-use config.Pass for this, 'cos we know it's not been
-// specified.
-func (f *Fs) getPass() (string, error) {
-	for f.savedpswd == "" {
 		_, _ = fmt.Fprint(os.Stderr, "Enter SFTP password: ")
-		f.savedpswd = config.ReadPassword()
+		clearpass := config.ReadPassword()
+		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
-	return f.savedpswd, nil
+
+	return NewFsWithConnection(ctx, name, root, m, opt, sshConfig)
 }
 
 // NewFsWithConnection creates a new Fs object from the name and root and an ssh.ClientConfig. It connects to
 // the host specified in the ssh.ClientConfig
-func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m configmap.Mapper, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
-	// Populate the Filesystem Object
-	f.name = name
-	f.root = root
-	f.absRoot = root
-	f.opt = *opt
-	f.m = m
-	f.config = sshConfig
-	f.url = "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root
-	f.mkdirLock = newStringLock()
-	f.pacer = fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant)))
-	f.savedpswd = ""
-
+func NewFsWithConnection(ctx context.Context, name string, root string, m configmap.Mapper, opt *Options, sshConfig *ssh.ClientConfig) (fs.Fs, error) {
+	f := &Fs{
+		name:      name,
+		root:      root,
+		opt:       *opt,
+		m:         m,
+		config:    sshConfig,
+		url:       "sftp://" + opt.User + "@" + opt.Host + ":" + opt.Port + "/" + root,
+		mkdirLock: newStringLock(),
+		pacer:     fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
+	}
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
-		SlowHash:                true,
 	}).Fill(f)
 	// Make a connection and pool it to return errors early
 	c, err := f.getSftpConnection()
 	if err != nil {
 		return nil, errors.Wrap(err, "NewFs")
 	}
-	cwd, err := c.sftpClient.Getwd()
 	f.putSftpConnection(&c, nil)
-	if err != nil {
-		fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
-	} else if !path.IsAbs(f.root) {
-		f.absRoot = path.Join(cwd, f.root)
-		fs.Debugf(f, "Using absolute root directory %q", f.absRoot)
-	}
 	if root != "" {
 		// Check to see if the root actually an existing file
-		oldAbsRoot := f.absRoot
 		remote := path.Base(root)
 		f.root = path.Dir(root)
-		f.absRoot = path.Dir(f.absRoot)
 		if f.root == "." {
 			f.root = ""
 		}
@@ -638,7 +519,6 @@ func NewFsWithConnection(ctx context.Context, f *Fs, name string, root string, m
 			if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
 				// File doesn't exist so return old f
 				f.root = root
-				f.absRoot = oldAbsRoot
 				return f, nil
 			}
 			return nil, err
@@ -721,7 +601,7 @@ func (f *Fs) dirExists(dir string) (bool, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	root := path.Join(f.absRoot, dir)
+	root := path.Join(f.root, dir)
 	ok, err := f.dirExists(root)
 	if err != nil {
 		return nil, errors.Wrap(err, "List failed")
@@ -802,7 +682,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 // directories above that
 func (f *Fs) mkParentDir(remote string) error {
 	parent := path.Dir(remote)
-	return f.mkdir(path.Join(f.absRoot, parent))
+	return f.mkdir(path.Join(f.root, parent))
 }
 
 // mkdir makes the directory and parents using native paths
@@ -838,7 +718,7 @@ func (f *Fs) mkdir(dirPath string) error {
 
 // Mkdir makes the root directory of the Fs object
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
-	root := path.Join(f.absRoot, dir)
+	root := path.Join(f.root, dir)
 	return f.mkdir(root)
 }
 
@@ -854,7 +734,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 		return fs.ErrorDirectoryNotEmpty
 	}
 	// Remove the directory
-	root := path.Join(f.absRoot, dir)
+	root := path.Join(f.root, dir)
 	c, err := f.getSftpConnection()
 	if err != nil {
 		return errors.Wrap(err, "Rmdir")
@@ -881,7 +761,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	}
 	err = c.sftpClient.Rename(
 		srcObj.path(),
-		path.Join(f.absRoot, remote),
+		path.Join(f.root, remote),
 	)
 	f.putSftpConnection(&c, err)
 	if err != nil {
@@ -908,8 +788,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
 		fs.Debugf(srcFs, "Can't move directory - not same remote type")
 		return fs.ErrorCantDirMove
 	}
-	srcPath := path.Join(srcFs.absRoot, srcRemote)
-	dstPath := path.Join(f.absRoot, dstRemote)
+	srcPath := path.Join(srcFs.root, srcRemote)
+	dstPath := path.Join(f.root, dstRemote)
 
 	// Check if destination exists
 	ok, err := f.dirExists(dstPath)
@@ -952,7 +832,7 @@ func (f *Fs) run(cmd string) ([]byte, error) {
 
 	session, err := c.sshClient.NewSession()
 	if err != nil {
-		return nil, errors.Wrap(err, "run: get SFTP session")
+		return nil, errors.Wrap(err, "run: get SFTP sessiion")
 	}
 	defer func() {
 		_ = session.Close()
@@ -1151,7 +1031,7 @@ func shellEscape(str string) string {
 func parseHash(bytes []byte) string {
 	// For strings with backslash *sum writes a leading \
 	// https://unix.stackexchange.com/q/313733/94054
-	return strings.ToLower(strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0]) // Split at hash / filename separator / all convert to lowercase
+	return strings.Split(strings.TrimLeft(string(bytes), "\\"), " ")[0] // Split at hash / filename separator
 }
 
 // Parses the byte array output from the SSH session
@@ -1195,7 +1075,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
 
 // path returns the native path of the object
 func (o *Object) path() string {
-	return path.Join(o.fs.absRoot, o.remote)
+	return path.Join(o.fs.root, o.remote)
 }
 
 // setMetadata updates the info in the object from the stat result passed in
@@ -1211,7 +1091,7 @@ func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "stat")
 	}
-	absPath := path.Join(f.absRoot, remote)
+	absPath := path.Join(f.root, remote)
 	info, err = c.sftpClient.Stat(absPath)
 	f.putSftpConnection(&c, err)
 	return info, err

backend/sharefile/api/types.go

@@ -95,7 +95,7 @@ type UploadSpecification struct {
 	ChunkURI           string `json:"ChunkUri"`           // Specifies the URI the client must send the file data to
 	FinishURI          string `json:"FinishUri"`          // If provided, specifies the final call the client must perform to finish the upload process
 	ProgressData       string `json:"ProgressData"`       // Allows the client to check progress of standard uploads
-	IsResume           bool   `json:"IsResume"`           // Specifies a Resumable upload is supported.
+	IsResume           bool   `json:"IsResume"`           // Specifies a Resumable upload is supproted.
 	ResumeIndex        int64  `json:"ResumeIndex"`        // Specifies the initial index for resuming, if IsResume is true.
 	ResumeOffset       int64  `json:"ResumeOffset"`       // Specifies the initial file offset by bytes, if IsResume is true
 	ResumeFileHash     string `json:"ResumeFileHash"`     // Specifies the MD5 hash of the first ResumeOffset bytes of the partial file found at the server

backend/sharefile/sharefile.go

@@ -853,8 +853,8 @@ func (f *Fs) Precision() time.Duration {
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // updateItem patches a file or folder
@@ -1090,7 +1090,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (dst fs.Obj
 		} else if err != nil {
 			return nil, errors.Wrap(err, "copy: failed to examine destination dir")
 		} else {
-			// otherwise need to copy via a temporary directory
+			// otherwise need to copy via a temporary directlry
 		}
 	}
 

backend/sugarsync/sugarsync.go

@@ -350,7 +350,7 @@ func (f *Fs) getAuth(req *http.Request) (err error) {
 	// if have auth, check it is in date
 	if f.opt.Authorization == "" || f.opt.User == "" || f.authExpiry.IsZero() || time.Until(f.authExpiry) < expiryLeeway {
 		// Get the auth token
-		f.srv.SetSigner(nil) // temporarily remove the signer so we don't infinitely recurse
+		f.srv.SetSigner(nil) // temporariliy remove the signer so we don't infinitely recurse
 		err = f.getAuthToken(ctx)
 		f.srv.SetSigner(f.getAuth) // replace signer
 		if err != nil {
@@ -895,12 +895,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return dstObj, nil
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
 	// Caution: Deleting a folder may orphan objects. It's important
 	// to remove the contents of the folder before you delete the
 	// folder. That's because removing a folder using DELETE does not
@@ -920,7 +920,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	if f.opt.HardDelete {
 		return fs.ErrorCantPurge
 	}
-	return f.purgeCheck(ctx, dir, false)
+	return f.purgeCheck(ctx, "", false)
 }
 
 // moveFile moves a file server side

backend/swift/swift.go

@@ -447,7 +447,6 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		WriteMimeType:     true,
 		BucketBased:       true,
 		BucketBasedRootOK: true,
-		SlowModTime:       true,
 	}).Fill(f)
 	if f.rootContainer != "" && f.rootDirectory != "" {
 		// Check to see if the object exists - ignoring directory markers
@@ -505,15 +504,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, er
 	// making sure we read the full metadata for all 0 byte files.
 	// We don't read the metadata for directory marker objects.
 	if info != nil && info.Bytes == 0 && info.ContentType != "application/directory" {
-		err := o.readMetaData() // reads info and headers, returning an error
-		if err == fs.ErrorObjectNotFound {
-			// We have a dangling large object here so just return the original metadata
-			fs.Errorf(o, "dangling large object with no contents")
-		} else if err != nil {
-			return nil, err
-		} else {
-			return o, nil
-		}
+		info = nil
 	}
 	if info != nil {
 		// Set info but not headers
@@ -545,7 +536,7 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error
 // container to the start.
 //
 // Set recurse to read sub directories
-func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn listFn) error {
+func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, fn listFn) error {
 	if prefix != "" && !strings.HasSuffix(prefix, "/") {
 		prefix += "/"
 	}
@@ -579,7 +570,7 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
 					fs.Logf(f, "Odd name received %q", remote)
 					continue
 				}
-				if !includeDirMarkers && remote == prefix {
+				if remote == prefix {
 					// If we have zero length directory markers ending in / then swift
 					// will return them in the listing for the directory which causes
 					// duplicate directories.  Ignore them here.
@@ -602,8 +593,8 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
 type addEntryFn func(fs.DirEntry) error
 
 // list the objects into the function supplied
-func (f *Fs) list(container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn addEntryFn) error {
-	err := f.listContainerRoot(container, directory, prefix, addContainer, recurse, includeDirMarkers, func(remote string, object *swift.Object, isDirectory bool) (err error) {
+func (f *Fs) list(container, directory, prefix string, addContainer bool, recurse bool, fn addEntryFn) error {
+	err := f.listContainerRoot(container, directory, prefix, addContainer, recurse, func(remote string, object *swift.Object, isDirectory bool) (err error) {
 		if isDirectory {
 			remote = strings.TrimRight(remote, "/")
 			d := fs.NewDir(remote, time.Time{}).SetSize(object.Bytes)
@@ -615,7 +606,7 @@ func (f *Fs) list(container, directory, prefix string, addContainer bool, recurs
 			if err != nil {
 				return err
 			}
-			if includeDirMarkers || o.Storable() {
+			if o.Storable() {
 				err = fn(o)
 			}
 		}
@@ -633,7 +624,7 @@ func (f *Fs) listDir(container, directory, prefix string, addContainer bool) (en
 		return nil, fs.ErrorListBucketRequired
 	}
 	// List the objects
-	err = f.list(container, directory, prefix, addContainer, false, false, func(entry fs.DirEntry) error {
+	err = f.list(container, directory, prefix, addContainer, false, func(entry fs.DirEntry) error {
 		entries = append(entries, entry)
 		return nil
 	})
@@ -703,7 +694,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 	container, directory := f.split(dir)
 	list := walk.NewListRHelper(callback)
 	listR := func(container, directory, prefix string, addContainer bool) error {
-		return f.list(container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error {
+		return f.list(container, directory, prefix, addContainer, true, func(entry fs.DirEntry) error {
 			return list.Add(entry)
 		})
 	}
@@ -840,21 +831,17 @@ func (f *Fs) Precision() time.Duration {
 	return time.Nanosecond
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and directories
 //
 // Implemented here so we can make sure we delete directory markers
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	container, directory := f.split(dir)
-	if container == "" {
-		return fs.ErrorListBucketRequired
-	}
+func (f *Fs) Purge(ctx context.Context) error {
 	// Delete all the files including the directory markers
 	toBeDeleted := make(chan fs.Object, fs.Config.Transfers)
 	delErr := make(chan error, 1)
 	go func() {
 		delErr <- operations.DeleteFiles(ctx, toBeDeleted)
 	}()
-	err := f.list(container, directory, f.rootDirectory, false, true, true, func(entry fs.DirEntry) error {
+	err := f.list(f.rootContainer, f.rootDirectory, f.rootDirectory, f.rootContainer == "", true, func(entry fs.DirEntry) error {
 		if o, ok := entry.(*Object); ok {
 			toBeDeleted <- o
 		}
@@ -868,7 +855,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	if err != nil {
 		return err
 	}
-	return f.Rmdir(ctx, dir)
+	return f.Rmdir(ctx, "")
 }
 
 // Copy src to this remote using server side copy operations.
@@ -1115,18 +1102,13 @@ func min(x, y int64) int64 {
 //
 // if except is passed in then segments with that prefix won't be deleted
 func (o *Object) removeSegments(except string) error {
-	segmentsContainer, _, err := o.getSegmentsDlo()
-	if err != nil {
-		return err
-	}
-	except = path.Join(o.remote, except)
-	// fs.Debugf(o, "segmentsContainer %q prefix %q", segmentsContainer, prefix)
-	err = o.fs.listContainerRoot(segmentsContainer, o.remote, "", false, true, true, func(remote string, object *swift.Object, isDirectory bool) error {
+	segmentsContainer, prefix, err := o.getSegmentsDlo()
+	err = o.fs.listContainerRoot(segmentsContainer, prefix, "", false, true, func(remote string, object *swift.Object, isDirectory bool) error {
 		if isDirectory {
 			return nil
 		}
 		if except != "" && strings.HasPrefix(remote, except) {
-			// fs.Debugf(o, "Ignoring current segment file %q in container %q", remote, segmentsContainer)
+			// fs.Debugf(o, "Ignoring current segment file %q in container %q", segmentsRoot+remote, segmentsContainer)
 			return nil
 		}
 		fs.Debugf(o, "Removing segment file %q in container %q", remote, segmentsContainer)
@@ -1335,7 +1317,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		// object has been safely uploaded
 		o.lastModified = modTime
 		o.size = size
-		o.md5 = rxHeaders["Etag"]
+		o.md5 = rxHeaders["ETag"]
 		o.contentType = contentType
 		o.headers = headers
 		if inCount != nil {

backend/tardigrade/fs.go

@@ -67,12 +67,12 @@ func init() {
 					log.Fatalf("Couldn't create access grant: %v", err)
 				}
 
-				serializedAccess, err := access.Serialize()
+				serialziedAccess, err := access.Serialize()
 				if err != nil {
 					log.Fatalf("Couldn't serialize access grant: %v", err)
 				}
 				configMapper.Set("satellite_address", satellite)
-				configMapper.Set("access_grant", serializedAccess)
+				configMapper.Set("access_grant", serialziedAccess)
 			} else if provider == existingProvider {
 				config.FileDeleteKey(name, "satellite_address")
 				config.FileDeleteKey(name, "api_key")

backend/union/entry.go

@@ -1,6 +1,7 @@
 package union
 
 import (
+	"bufio"
 	"context"
 	"io"
 	"sync"
@@ -66,9 +67,31 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 		obj := entries[0].(*upstream.Object)
 		return obj.Update(ctx, in, src, options...)
 	}
-	// Multi-threading
-	readers, errChan := multiReader(len(entries), in)
+	// Get multiple reader
+	readers := make([]io.Reader, len(entries))
+	writers := make([]io.Writer, len(entries))
 	errs := Errors(make([]error, len(entries)+1))
+	for i := range entries {
+		r, w := io.Pipe()
+		bw := bufio.NewWriter(w)
+		readers[i], writers[i] = r, bw
+		defer func() {
+			err := w.Close()
+			if err != nil {
+				panic(err)
+			}
+		}()
+	}
+	go func() {
+		mw := io.MultiWriter(writers...)
+		es := make([]error, len(writers)+1)
+		_, es[len(es)-1] = io.Copy(mw, in)
+		for i, bw := range writers {
+			es[i] = bw.(*bufio.Writer).Flush()
+		}
+		errs[len(entries)] = Errors(es).Err()
+	}()
+	// Multi-threading
 	multithread(len(entries), func(i int) {
 		if o, ok := entries[i].(*upstream.Object); ok {
 			err := o.Update(ctx, readers[i], src, options...)
@@ -77,7 +100,6 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
 			errs[i] = fs.ErrorNotAFile
 		}
 	})
-	errs[len(entries)] = <-errChan
 	return errs.Err()
 }
 

backend/union/policy/epall.go

@@ -61,7 +61,7 @@ func (p *EpAll) Action(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.epall(ctx, upstreams, path)
 }
 
-// ActionEntries is ACTION category policy but receiving a set of candidate entries
+// ActionEntries is ACTION category policy but receivng a set of candidate entries
 func (p *EpAll) ActionEntries(entries ...upstream.Entry) ([]upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/epmfs.go

@@ -106,7 +106,7 @@ func (p *EpMfs) Search(ctx context.Context, upstreams []*upstream.Fs, path strin
 	return p.mfs(upstreams)
 }
 
-// SearchEntries is SEARCH category policy but receiving a set of candidate entries
+// SearchEntries is SEARCH category policy but receivng a set of candidate entries
 func (p *EpMfs) SearchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	if len(entries) == 0 {
 		return nil, fs.ErrorObjectNotFound

backend/union/policy/ff.go

@@ -14,7 +14,7 @@ func init() {
 // FF stands for first found
 // Search category: same as epff.
 // Action category: same as epff.
-// Create category: Given the order of the candidates, act on the first one found.
+// Create category: Given the order of the candiates, act on the first one found.
 type FF struct {
 	EpFF
 }

backend/union/union.go

@@ -60,7 +60,7 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	Upstreams    fs.SpaceSepList `config:"upstreams"`
-	Remotes      fs.SpaceSepList `config:"remotes"` // Deprecated
+	Remotes      fs.SpaceSepList `config:"remotes"` // Depreated
 	ActionPolicy string          `config:"action_policy"`
 	CreatePolicy string          `config:"create_policy"`
 	SearchPolicy string          `config:"search_policy"`
@@ -145,16 +145,11 @@ func (f *Fs) Hashes() hash.Set {
 // Mkdir makes the root directory of the Fs object
 func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	upstreams, err := f.create(ctx, dir)
-	if err == fs.ErrorObjectNotFound {
-		if dir != parentDir(dir) {
-			if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
-				return err
-			}
-			upstreams, err = f.create(ctx, dir)
-		} else if dir == "" {
-			// If root dirs not created then create them
-			upstreams, err = f.upstreams, nil
+	if err == fs.ErrorObjectNotFound && dir != parentDir(dir) {
+		if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
+			return err
 		}
+		upstreams, err = f.create(ctx, dir)
 	}
 	if err != nil {
 		return err
@@ -167,13 +162,13 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
 	return errs.Err()
 }
 
-// Purge all files in the directory
+// Purge all files in the root and the root directory
 //
 // Implement this if you have a way of deleting all the files
 // quicker than just running Remove() on the result of List()
 //
 // Return an error if it doesn't exist
-func (f *Fs) Purge(ctx context.Context, dir string) error {
+func (f *Fs) Purge(ctx context.Context) error {
 	for _, r := range f.upstreams {
 		if r.Features().Purge == nil {
 			return fs.ErrorCantPurge
@@ -185,10 +180,7 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
 	}
 	errs := Errors(make([]error, len(upstreams)))
 	multithread(len(upstreams), func(i int) {
-		err := upstreams[i].Features().Purge(ctx, dir)
-		if errors.Cause(err) == fs.ErrorDirNotFound {
-			err = nil
-		}
+		err := upstreams[i].Features().Purge(ctx)
 		errs[i] = errors.Wrap(err, upstreams[i].Name())
 	})
 	return errs.Err()
@@ -390,37 +382,6 @@ func (f *Fs) DirCacheFlush() {
 	})
 }
 
-// Tee in into n outputs
-//
-// When finished read the error from the channel
-func multiReader(n int, in io.Reader) ([]io.Reader, <-chan error) {
-	readers := make([]io.Reader, n)
-	pipeWriters := make([]*io.PipeWriter, n)
-	writers := make([]io.Writer, n)
-	errChan := make(chan error, 1)
-	for i := range writers {
-		r, w := io.Pipe()
-		bw := bufio.NewWriter(w)
-		readers[i], pipeWriters[i], writers[i] = r, w, bw
-	}
-	go func() {
-		mw := io.MultiWriter(writers...)
-		es := make([]error, 2*n+1)
-		_, copyErr := io.Copy(mw, in)
-		es[2*n] = copyErr
-		// Flush the buffers
-		for i, bw := range writers {
-			es[i] = bw.(*bufio.Writer).Flush()
-		}
-		// Close the underlying pipes
-		for i, pw := range pipeWriters {
-			es[2*i] = pw.CloseWithError(copyErr)
-		}
-		errChan <- Errors(es).Err()
-	}()
-	return readers, errChan
-}
-
 func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bool, options ...fs.OpenOption) (fs.Object, error) {
 	srcPath := src.Remote()
 	upstreams, err := f.create(ctx, srcPath)
@@ -448,9 +409,31 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 		e, err := f.wrapEntries(u.WrapObject(o))
 		return e.(*Object), err
 	}
-	// Multi-threading
-	readers, errChan := multiReader(len(upstreams), in)
 	errs := Errors(make([]error, len(upstreams)+1))
+	// Get multiple reader
+	readers := make([]io.Reader, len(upstreams))
+	writers := make([]io.Writer, len(upstreams))
+	for i := range writers {
+		r, w := io.Pipe()
+		bw := bufio.NewWriter(w)
+		readers[i], writers[i] = r, bw
+		defer func() {
+			err := w.Close()
+			if err != nil {
+				panic(err)
+			}
+		}()
+	}
+	go func() {
+		mw := io.MultiWriter(writers...)
+		es := make([]error, len(writers)+1)
+		_, es[len(es)-1] = io.Copy(mw, in)
+		for i, bw := range writers {
+			es[i] = bw.(*bufio.Writer).Flush()
+		}
+		errs[len(upstreams)] = Errors(es).Err()
+	}()
+	// Multi-threading
 	objs := make([]upstream.Entry, len(upstreams))
 	multithread(len(upstreams), func(i int) {
 		u := upstreams[i]
@@ -467,7 +450,6 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
 		}
 		objs[i] = u.WrapObject(o)
 	})
-	errs[len(upstreams)] = <-errChan
 	err = errs.Err()
 	if err != nil {
 		return nil, err
@@ -522,9 +504,6 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 	}
 	for _, u := range f.upstreams {
 		usg, err := u.About(ctx)
-		if errors.Cause(err) == fs.ErrorDirNotFound {
-			continue
-		}
 		if err != nil {
 			return nil, err
 		}
@@ -572,7 +551,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
-	entriesList := make([][]upstream.Entry, len(f.upstreams))
+	entriess := make([][]upstream.Entry, len(f.upstreams))
 	errs := Errors(make([]error, len(f.upstreams)))
 	multithread(len(f.upstreams), func(i int) {
 		u := f.upstreams[i]
@@ -585,7 +564,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		for j, e := range entries {
 			uEntries[j], _ = u.WrapEntry(e)
 		}
-		entriesList[i] = uEntries
+		entriess[i] = uEntries
 	})
 	if len(errs) == len(errs.FilterNil()) {
 		errs = errs.Map(func(e error) error {
@@ -599,7 +578,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 		}
 		return nil, errs.Err()
 	}
-	return f.mergeDirEntries(entriesList)
+	return f.mergeDirEntries(entriess)
 }
 
 // ListR lists the objects and directories of the Fs starting
@@ -619,7 +598,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
 // Don't implement this unless you have a more efficient way
 // of listing recursively that doing a directory traversal.
 func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
-	var entriesList [][]upstream.Entry
+	var entriess [][]upstream.Entry
 	errs := Errors(make([]error, len(f.upstreams)))
 	var mutex sync.Mutex
 	multithread(len(f.upstreams), func(i int) {
@@ -631,7 +610,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 				uEntries[j], _ = u.WrapEntry(e)
 			}
 			mutex.Lock()
-			entriesList = append(entriesList, uEntries)
+			entriess = append(entriess, uEntries)
 			mutex.Unlock()
 			return nil
 		}
@@ -658,7 +637,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
 		}
 		return errs.Err()
 	}
-	entries, err := f.mergeDirEntries(entriesList)
+	entries, err := f.mergeDirEntries(entriess)
 	if err != nil {
 		return err
 	}
@@ -729,9 +708,9 @@ func (f *Fs) searchEntries(entries ...upstream.Entry) (upstream.Entry, error) {
 	return f.searchPolicy.SearchEntries(entries...)
 }
 
-func (f *Fs) mergeDirEntries(entriesList [][]upstream.Entry) (fs.DirEntries, error) {
+func (f *Fs) mergeDirEntries(entriess [][]upstream.Entry) (fs.DirEntries, error) {
 	entryMap := make(map[string]([]upstream.Entry))
-	for _, en := range entriesList {
+	for _, en := range entriess {
 		if en == nil {
 			continue
 		}
@@ -823,7 +802,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
-	fs.Debugf(f, "actionPolicy = %T, createPolicy = %T, searchPolicy = %T", f.actionPolicy, f.createPolicy, f.searchPolicy)
 	var features = (&fs.Features{
 		CaseInsensitive:         true,
 		DuplicateFiles:          false,

backend/union/union_test.go

@@ -153,29 +153,3 @@ func TestPolicy2(t *testing.T) {
 		UnimplementableObjectMethods: []string{"MimeType"},
 	})
 }
-
-func TestPolicy3(t *testing.T) {
-	if *fstest.RemoteName != "" {
-		t.Skip("Skipping as -remote set")
-	}
-	tempdir1 := filepath.Join(os.TempDir(), "rclone-union-test-policy31")
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-union-test-policy32")
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-union-test-policy33")
-	require.NoError(t, os.MkdirAll(tempdir1, 0744))
-	require.NoError(t, os.MkdirAll(tempdir2, 0744))
-	require.NoError(t, os.MkdirAll(tempdir3, 0744))
-	upstreams := tempdir1 + " " + tempdir2 + " " + tempdir3
-	name := "TestUnionPolicy3"
-	fstests.Run(t, &fstests.Opt{
-		RemoteName: name + ":",
-		ExtraConfig: []fstests.ExtraConfigItem{
-			{Name: name, Key: "type", Value: "union"},
-			{Name: name, Key: "upstreams", Value: upstreams},
-			{Name: name, Key: "action_policy", Value: "all"},
-			{Name: name, Key: "create_policy", Value: "all"},
-			{Name: name, Key: "search_policy", Value: "all"},
-		},
-		UnimplementableFsMethods:     []string{"OpenWriterAt", "DuplicateFiles"},
-		UnimplementableObjectMethods: []string{"MimeType"},
-	})
-}

backend/union/upstream/upstream.go

@@ -52,7 +52,7 @@ type Object struct {
 	f *Fs
 }
 
-// Entry describe a wrapped fs.DirEntry interface with the
+// Entry describe a warpped fs.DirEntry interface with the
 // information of upstream Fs
 type Entry interface {
 	fs.DirEntry
@@ -97,7 +97,6 @@ func New(remote, root string, cacheTime time.Duration) (*Fs, error) {
 		return nil, err
 	}
 	f.Fs = myFs
-	cache.PinUntilFinalized(f.Fs, f)
 	return f, err
 }
 
@@ -336,9 +335,6 @@ func (f *Fs) updateUsageCore(lock bool) error {
 	usage, err := f.RootFs.Features().About(ctx)
 	if err != nil {
 		f.cacheUpdate = false
-		if errors.Cause(err) == fs.ErrorDirNotFound {
-			err = nil
-		}
 		return err
 	}
 	if lock {

backend/webdav/webdav.go

@@ -686,8 +686,8 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
 
 // mkParentDir makes the parent of the native path dirPath if
 // necessary and any directories above that
-func (f *Fs) mkParentDir(ctx context.Context, dirPath string) (err error) {
-	// defer log.Trace(dirPath, "")("err=%v", &err)
+func (f *Fs) mkParentDir(ctx context.Context, dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
 	// chop off trailing / if it exists
 	if strings.HasSuffix(dirPath, "/") {
 		dirPath = dirPath[:len(dirPath)-1]
@@ -699,27 +699,6 @@ func (f *Fs) mkParentDir(ctx context.Context, dirPath string) (err error) {
 	return f.mkdir(ctx, parent)
 }
 
-// _dirExists - list dirPath to see if it exists
-//
-// dirPath should be a native path ending in a /
-func (f *Fs) _dirExists(ctx context.Context, dirPath string) (exists bool) {
-	opts := rest.Opts{
-		Method: "PROPFIND",
-		Path:   dirPath,
-		ExtraHeaders: map[string]string{
-			"Depth": "0",
-		},
-	}
-	var result api.Multistatus
-	var resp *http.Response
-	var err error
-	err = f.pacer.Call(func() (bool, error) {
-		resp, err = f.srv.CallXML(ctx, &opts, nil, &result)
-		return f.shouldRetry(resp, err)
-	})
-	return err == nil
-}
-
 // low level mkdir, only makes the directory, doesn't attempt to create parents
 func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
 	// We assume the root is already created
@@ -740,29 +719,19 @@ func (f *Fs) _mkdir(ctx context.Context, dirPath string) error {
 		return f.shouldRetry(resp, err)
 	})
 	if apiErr, ok := err.(*api.Error); ok {
-		// Check if it already exists. The response code for this isn't
-		// defined in the RFC so the implementations vary wildly.
-		//
+		// already exists
 		// owncloud returns 423/StatusLocked if the create is already in progress
 		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
 			return nil
 		}
-		// 4shared returns a 409/StatusConflict here which clashes
-		// horribly with the intermediate paths don't exist meaning. So
-		// check to see if actually exists. This will correct other
-		// error codes too.
-		if f._dirExists(ctx, dirPath) {
-			return nil
-		}
-
 	}
 	return err
 }
 
 // mkdir makes the directory and parents using native paths
-func (f *Fs) mkdir(ctx context.Context, dirPath string) (err error) {
-	// defer log.Trace(dirPath, "")("err=%v", &err)
-	err = f._mkdir(ctx, dirPath)
+func (f *Fs) mkdir(ctx context.Context, dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
+	err := f._mkdir(ctx, dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
 		// parent does not exist so create it first then try again
 		if apiErr.StatusCode == http.StatusConflict {
@@ -899,13 +868,13 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
 	return f.copyOrMove(ctx, src, remote, "COPY")
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // Move src to this remote using server side move operations.

backend/webdav/webdav_test.go

@@ -12,7 +12,7 @@ import (
 // TestIntegration runs integration tests against the remote
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
-		RemoteName: "TestWebdavNextcloud:",
+		RemoteName: "TestWebdavNexcloud:",
 		NilObject:  (*webdav.Object)(nil),
 	})
 }

backend/yandex/api/types.go

@@ -20,7 +20,7 @@ type ResourceInfoRequestOptions struct {
 	Fields   []string
 }
 
-//ResourceInfoResponse struct is returned by the API for metadata requests.
+//ResourceInfoResponse struct is returned by the API for metedata requests.
 type ResourceInfoResponse struct {
 	PublicKey        string                 `json:"public_key"`
 	Name             string                 `json:"name"`

backend/yandex/yandex.go

@@ -67,7 +67,13 @@ func init() {
 				return
 			}
 		},
-		Options: append(oauthutil.SharedOptions, []fs.Option{{
+		Options: []fs.Option{{
+			Name: config.ConfigClientID,
+			Help: "Yandex Client Id\nLeave blank normally.",
+		}, {
+			Name: config.ConfigClientSecret,
+			Help: "Yandex Client Secret\nLeave blank normally.",
+		}, {
 			Name:     config.ConfigEncoding,
 			Help:     config.ConfigEncodingHelp,
 			Advanced: true,
@@ -75,7 +81,7 @@ func init() {
 			// it doesn't seem worth making an exception for this
 			Default: (encoder.Display |
 				encoder.EncodeInvalidUtf8),
-		}}...),
+		}},
 	})
 }
 
@@ -631,13 +637,13 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
 	return f.purgeCheck(ctx, dir, true)
 }
 
-// Purge deletes all the files in the directory
+// Purge deletes all the files and the container
 //
 // Optional interface: Only implement this if you have a way of
 // deleting all the files quicker than just running Remove() on the
 // result of List()
-func (f *Fs) Purge(ctx context.Context, dir string) error {
-	return f.purgeCheck(ctx, dir, false)
+func (f *Fs) Purge(ctx context.Context) error {
+	return f.purgeCheck(ctx, "", false)
 }
 
 // copyOrMoves copies or moves directories or files depending on the method parameter

bin/cross-compile.go

@@ -40,33 +40,28 @@ var (
 )
 
 // GOOS/GOARCH pairs we build for
-//
-// If the GOARCH contains a - it is a synthetic arch with more parameters
 var osarches = []string{
 	"windows/386",
 	"windows/amd64",
+	"darwin/386",
 	"darwin/amd64",
 	"linux/386",
 	"linux/amd64",
 	"linux/arm",
-	"linux/arm-v7",
 	"linux/arm64",
 	"linux/mips",
 	"linux/mipsle",
 	"freebsd/386",
 	"freebsd/amd64",
 	"freebsd/arm",
-	"freebsd/arm-v7",
 	"netbsd/386",
 	"netbsd/amd64",
 	"netbsd/arm",
-	"netbsd/arm-v7",
 	"openbsd/386",
 	"openbsd/amd64",
 	"plan9/386",
 	"plan9/amd64",
 	"solaris/amd64",
-	"js/wasm",
 }
 
 // Special environment flags for a given arch
@@ -74,7 +69,6 @@ var archFlags = map[string][]string{
 	"386":    {"GO386=387"},
 	"mips":   {"GOMIPS=softfloat"},
 	"mipsle": {"GOMIPS=softfloat"},
-	"arm-v7": {"GOARM=7"},
 }
 
 // runEnv - run a shell command with env
@@ -269,18 +263,9 @@ func cleanupResourceSyso(sysoFilePath string) {
 	}
 }
 
-// Trip a version suffix off the arch if present
-func stripVersion(goarch string) string {
-	i := strings.Index(goarch, "-")
-	if i < 0 {
-		return goarch
-	}
-	return goarch[:i]
-}
-
 // build the binary in dir returning success or failure
 func compileArch(version, goos, goarch, dir string) bool {
-	log.Printf("Compiling %s/%s into %s", goos, goarch, dir)
+	log.Printf("Compiling %s/%s", goos, goarch)
 	output := filepath.Join(dir, "rclone")
 	if goos == "windows" {
 		output += ".exe"
@@ -298,13 +283,14 @@ func compileArch(version, goos, goarch, dir string) bool {
 		"go", "build",
 		"--ldflags", "-s -X github.com/rclone/rclone/fs.Version=" + version,
 		"-trimpath",
+		"-i",
 		"-o", output,
 		"-tags", *tags,
 		"..",
 	}
 	env := []string{
 		"GOOS=" + goos,
-		"GOARCH=" + stripVersion(goarch),
+		"GOARCH=" + goarch,
 	}
 	if !*cgo {
 		env = append(env, "CGO_ENABLED=0")
@@ -320,16 +306,14 @@ func compileArch(version, goos, goarch, dir string) bool {
 		return false
 	}
 	if !*compileOnly {
-		if goos != "js" {
-			artifacts := []string{buildZip(dir)}
-			// build a .deb and .rpm if appropriate
-			if goos == "linux" {
-				artifacts = append(artifacts, buildDebAndRpm(dir, version, stripVersion(goarch))...)
-			}
-			if *copyAs != "" {
-				for _, artifact := range artifacts {
-					run("ln", artifact, strings.Replace(artifact, "-"+version, "-"+*copyAs, 1))
-				}
+		artifacts := []string{buildZip(dir)}
+		// build a .deb and .rpm if appropriate
+		if goos == "linux" {
+			artifacts = append(artifacts, buildDebAndRpm(dir, version, goarch)...)
+		}
+		if *copyAs != "" {
+			for _, artifact := range artifacts {
+				run("ln", artifact, strings.Replace(artifact, "-"+version, "-"+*copyAs, 1))
 			}
 		}
 		// tidy up

bin/get-github-release.go

@@ -374,13 +374,16 @@ func untar(srcFile, fileName, extractDir string) {
 				if err != nil {
 					log.Fatalf("Couldn't open output file: %v", err)
 				}
+				defer func() {
+					err := out.Close()
+					if err != nil {
+						log.Fatalf("Couldn't close output: %v", err)
+					}
+				}()
 				n, err := io.Copy(out, tarReader)
 				if err != nil {
 					log.Fatalf("Couldn't write output file: %v", err)
 				}
-				if err = out.Close(); err != nil {
-					log.Fatalf("Couldn't close output: %v", err)
-				}
 				log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
 			}
 		}

bin/make_changelog.py

@@ -141,7 +141,7 @@ def main():
     for name in sorted(bugfixes.keys()):
         out(name)
 
-    # Read old changelog and split
+    # Read old changlog and split
     with open("docs/content/changelog.md") as fd:
         old_changelog = fd.read()
     heading = "# Changelog"

bin/make_manual.py

@@ -6,7 +6,6 @@ conversion into man pages etc.
 
 import os
 import re
-import time
 from datetime import datetime
 
 docpath = "docs/content"
@@ -157,19 +156,17 @@ def read_commands(docpath):
         if command != "rclone.md":
             docs.append(read_command(command))
     return "\n".join(docs)
-
+    
 def main():
     check_docs(docpath)
     command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
-    build_date = datetime.utcfromtimestamp(
-            int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
     with open(outfile, "w") as out:
         out.write("""\
 %% rclone(1) User Manual
 %% Nick Craig-Wood
 %% %s
 
-""" % build_date.strftime("%b %d, %Y"))
+""" % datetime.now().strftime("%b %d, %Y"))
         for doc in docs:
             contents = read_doc(doc)
             # Substitute the commands into doc.md

bin/make_test_files.go

@@ -19,7 +19,7 @@ var (
 	// Flags
 	numberOfFiles            = flag.Int("n", 1000, "Number of files to create")
 	averageFilesPerDirectory = flag.Int("files-per-directory", 10, "Average number of files per directory")
-	maxDepth                 = flag.Int("max-depth", 10, "Maximum depth of directory hierarchy")
+	maxDepth                 = flag.Int("max-depth", 10, "Maximum depth of directory heirachy")
 	minFileSize              = flag.Int64("min-size", 0, "Minimum size of file to create")
 	maxFileSize              = flag.Int64("max-size", 100, "Maximum size of files to create")
 	minFileNameLength        = flag.Int("min-name-length", 4, "Minimum size of file to create")
@@ -61,7 +61,7 @@ func fileName() (name string) {
 	return name
 }
 
-// dir is a directory in the directory hierarchy being built up
+// dir is a directory in the directory heirachy being built up
 type dir struct {
 	name     string
 	depth    int
@@ -69,7 +69,7 @@ type dir struct {
 	parent   *dir
 }
 
-// Create a random directory hierarchy under d
+// Create a random directory heirachy under d
 func (d *dir) createDirectories() {
 	for totalDirectories < directoriesToCreate {
 		newDir := &dir{
@@ -91,7 +91,7 @@ func (d *dir) createDirectories() {
 	return
 }
 
-// list the directory hierarchy
+// list the directory heirachy
 func (d *dir) list(path string, output []string) []string {
 	dirPath := filepath.Join(path, d.name)
 	output = append(output, dirPath)

bin/not-in-stable.go

@@ -10,8 +10,6 @@ import (
 	"os"
 	"os/exec"
 	"regexp"
-
-	"github.com/coreos/go-semver/semver"
 )
 
 // version=$(sed <VERSION -e 's/\.[0-9]+*$//g')
@@ -30,7 +28,7 @@ func readCommits(from, to string) (logMap map[string]string, logs []string) {
 	cmd := exec.Command("git", "log", "--oneline", from+".."+to)
 	out, err := cmd.Output()
 	if err != nil {
-		log.Fatalf("failed to run git log %s: %v", from+".."+to, err)
+		log.Fatalf("failed to run git log: %v", err)
 	}
 	logMap = map[string]string{}
 	logs = []string{}
@@ -55,20 +53,15 @@ func main() {
 	if len(args) != 0 {
 		log.Fatalf("Syntax: %s", os.Args[0])
 	}
-	// v1.54.0
 	versionBytes, err := ioutil.ReadFile("VERSION")
 	if err != nil {
 		log.Fatalf("Failed to read version: %v", err)
 	}
-	if versionBytes[0] == 'v' {
-		versionBytes = versionBytes[1:]
-	}
-	versionBytes = bytes.TrimSpace(versionBytes)
-	semver := semver.New(string(versionBytes))
-	stable := fmt.Sprintf("v%d.%d", semver.Major, semver.Minor-1)
-	log.Printf("Finding commits in %v not in stable %s", semver, stable)
-	masterMap, masterLogs := readCommits(stable+".0", "master")
-	stableMap, _ := readCommits(stable+".0", stable+"-stable")
+	i := bytes.LastIndexByte(versionBytes, '.')
+	version := string(versionBytes[:i])
+	log.Printf("Finding commits not in stable %s", version)
+	masterMap, masterLogs := readCommits(version+".0", "master")
+	stableMap, _ := readCommits(version+".0", version+"-stable")
 	for _, logMessage := range masterLogs {
 		// Commit found in stable already
 		if _, found := stableMap[logMessage]; found {

bin/test-repeat-vfs.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Thrash the VFS tests
-
-set -e
-
-# Optionally set the iterations with the first parameter
-iterations=${1:-100}
-
-base=$(dirname $(dirname $(realpath "$0")))
-echo ${base}
-run=${base}/bin/test-repeat.sh
-echo ${run}
-
-testdirs="
-vfs
-vfs/vfscache
-vfs/vfscache/writeback
-vfs/vfscache/downloaders
-cmd/cmount
-"
-
-for testdir in ${testdirs}; do
-    echo "Testing ${testdir} with ${iterations} iterations"
-    cd ${base}/${testdir}
-    ${run} -i=${iterations} -race -tags=cmount
-done

cmd/cachestats/cachestats.go

@@ -1,4 +1,4 @@
-// +build !plan9,!js
+// +build !plan9
 
 package cachestats