Version v1.46

Thanks to:
- Christopher Peterson (@cspeterson) for the original script
- Danil Semelenov (@sgtpep) for many refinements

.circleci/config.yml

@@ -1,3 +1,4 @@
+---
 version: 2
 
 jobs:
@@ -13,10 +14,10 @@ jobs:
     - run:
         name: Cross-compile rclone
         command: |
-            docker pull billziss/xgo-cgofuse
+            docker pull rclone/xgo-cgofuse
             go get -v github.com/karalabe/xgo
             xgo \
-                --image=billziss/xgo-cgofuse \
+                --image=rclone/xgo-cgofuse \
                 --targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
                 -tags cmount \
                 .
@@ -29,6 +30,21 @@ jobs:
         command: |
             mkdir -p /tmp/rclone.dist
             cp -R rclone-* /tmp/rclone.dist
+            mkdir build
+            cp -R rclone-* build/
+
+    - run:
+        name: Build rclone
+        command: |
+            go version
+            go build
+
+    - run:
+        name: Upload artifacts
+        command: |
+          if [[ $CIRCLE_PULL_REQUEST != "" ]]; then
+              make circleci_upload
+          fi
 
     - store_artifacts:
         path: /tmp/rclone.dist

.golangci.yml

@@ -0,0 +1,27 @@
+# golangci-lint configuration options
+
+run:
+  build-tags:
+    - cmount
+
+linters:
+  enable:
+    - deadcode
+    - errcheck
+    - goimports
+    - golint
+    - ineffassign
+    - structcheck
+    - varcheck
+    - govet
+    - unconvert
+    #- prealloc
+    #- maligned
+  disable-all: true
+
+issues:
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0

.gometalinter.json

@@ -1,14 +0,0 @@
-{
-    "Enable": [
-        "deadcode",
-        "errcheck",
-        "goimports",
-        "golint",
-        "ineffassign",
-        "structcheck",
-        "varcheck",
-        "vet"
-    ],
-    "EnableGC": true,
-    "Vendor": true
-}

.travis.yml

@@ -24,6 +24,7 @@ script:
 env:
   global:
     - GOTAGS=cmount
+    - GO111MODULE=off
     - secure: gU8gCV9R8Kv/Gn0SmCP37edpfIbPoSvsub48GK7qxJdTU628H0KOMiZW/T0gtV5d67XJZ4eKnhJYlxwwxgSgfejO32Rh5GlYEKT/FuVoH0BD72dM1GDFLSrUiUYOdoHvf/BKIFA3dJFT4lk2ASy4Zh7SEoXHG6goBlqUpYx8hVA=
     - secure: AMjrMAksDy3QwqGqnvtUg8FL/GNVgNqTqhntLF9HSU0njHhX6YurGGnfKdD9vNHlajPQOewvmBjwNLcDWGn2WObdvmh9Ohep0EmOjZ63kliaRaSSQueSd8y0idfqMQAxep0SObOYbEDVmQh0RCAE9wOVKRaPgw98XvgqWGDq5Tw=
     - secure: Uaiveq+/rvQjO03GzvQZV2J6pZfedoFuhdXrLVhhHSeP4ZBca0olw7xaqkabUyP3LkVYXMDSX8EbyeuQT1jfEe5wp5sBdfaDtuYW6heFyjiHIIIbVyBfGXon6db4ETBjOaX/Xt8uktrgNge6qFlj+kpnmpFGxf0jmDLw1zgg7tk=

CONTRIBUTING.md

@@ -351,6 +351,12 @@ Unit tests
 Integration tests
 
   * Add your backend to `fstest/test_all/config.yaml`
+      * Once you've done that then you can use the integration test framework from the project root:
+      * go install ./...
+      * test_all -backend remote
+
+Or if you want to run the integration tests manually:
+
   * Make sure integration tests pass with
       * `cd fs/operations`
       * `go test -v -remote TestRemote:`
@@ -372,4 +378,3 @@ Add your fs to the docs - you'll need to pick an icon for it from [fontawesome](
   * `docs/content/about.md` - front page of rclone.org
   * `docs/layouts/chrome/navbar.html` - add it to the website navigation
   * `bin/make_manual.py` - add the page to the `docs` constant
-  * `cmd/cmd.go` - the main help for rclone

MAINTAINERS.md

@@ -1,14 +1,17 @@
 # Maintainers guide for rclone #
 
-Current active maintainers of rclone are
+Current active maintainers of rclone are:
 
-  * Nick Craig-Wood @ncw
-  * Stefan Breunig @breunigs
-  * Ishuah Kariuki @ishuah
-  * Remus Bunduc @remusb - cache subsystem maintainer
-  * Fabian Möller @B4dM4n
-  * Alex Chen @Cnly
-  * Sandeep Ummadi @sandeepkru
+| Name             | GitHub ID   | Specific Responsibilities    |
+| :--------------- | :---------- | :--------------------------  |
+| Nick Craig-Wood  | @ncw        | overall project health       |
+| Stefan Breunig   | @breunigs   |                              |
+| Ishuah Kariuki   | @ishuah     |                              |
+| Remus Bunduc     | @remusb     | cache backend                |
+| Fabian Möller    | @B4dM4n     |                              |
+| Alex Chen        | @Cnly       | onedrive backend             |
+| Sandeep Ummadi   | @sandeepkru | azureblob backend            |
+| Sebastian Bünger | @buengese   | jottacloud & yandex backends |
 
 **This is a work in progress Draft**
 

Makefile

@@ -64,30 +64,20 @@ endif
 # Do source code quality checks
 check:	rclone
 ifdef FULL_TESTS
-	go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
-	errcheck $(BUILDTAGS) ./...
-	find . -name \*.go | grep -v /vendor/ | xargs goimports -d | grep . ; test $$? -eq 1
-	go list ./... | xargs -n1 golint | grep -E -v '(StorageUrl|CdnUrl)' ; test $$? -eq 1
+	@# we still run go vet for -printfuncs which golangci-lint doesn't do yet
+	@# see: https://github.com/golangci/golangci-lint/issues/204
+	@echo "-- START CODE QUALITY REPORT -------------------------------"
+	@go vet $(BUILDTAGS) -printfuncs Debugf,Infof,Logf,Errorf ./...
+	@golangci-lint run ./...
+	@echo "-- END CODE QUALITY REPORT ---------------------------------"
 else
 	@echo Skipping source quality tests as version of go too old
 endif
 
-gometalinter_install:
-	go get -u github.com/alecthomas/gometalinter
-	gometalinter --install --update
-
-# We aren't using gometalinter as the default linter yet because
-# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
-# 2. can't get -printfuncs working with the vet linter
-gometalinter:
-	gometalinter ./...
-
 # Get the build dependencies
 build_dep:
 ifdef FULL_TESTS
-	go get -u github.com/kisielk/errcheck
-	go get -u golang.org/x/tools/cmd/goimports
-	go get -u golang.org/x/lint/golint
+	go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
 endif
 
 # Get the release dependencies
@@ -116,7 +106,7 @@ MANUAL.txt:	MANUAL.md
 	pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
 
 commanddocs: rclone
-	rclone gendocs docs/content/commands/
+	XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/commands/
 
 backenddocs: rclone bin/make_backend_docs.py
 	./bin/make_backend_docs.py
@@ -185,6 +175,13 @@ ifndef BRANCH_PATH
 endif
 	@echo Beta release ready at $(BETA_URL)
 
+circleci_upload:
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD)/testbuilds
+ifndef BRANCH_PATH
+	./rclone --config bin/travis.rclone.conf -v copy build/ $(BETA_UPLOAD_ROOT)/test/testbuilds-latest
+endif
+	@echo Beta release ready at $(BETA_URL)/testbuilds
+
 BUILD_FLAGS := -exclude "^(windows|darwin)/"
 ifeq ($(TRAVIS_OS_NAME),osx)
 	BUILD_FLAGS := -include "^darwin/" -cgo
@@ -192,7 +189,7 @@ endif
 
 travis_beta:
 ifeq ($(TRAVIS_OS_NAME),linux)
-	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
 endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
 	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)

README.md

@@ -20,6 +20,7 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
 
 ## Storage providers
 
+  * Alibaba Cloud (Aliyun) Object Storage System (OSS) [:page_facing_up:](https://rclone.org/s3/#alibaba-oss)
   * Amazon Drive [:page_facing_up:](https://rclone.org/amazonclouddrive/) ([See note](https://rclone.org/amazonclouddrive/#status))
   * Amazon S3 [:page_facing_up:](https://rclone.org/s3/)
   * Backblaze B2 [:page_facing_up:](https://rclone.org/b2/)
@@ -43,13 +44,14 @@ Rclone *("rsync for cloud storage")* is a command line program to sync files and
   * Nextcloud [:page_facing_up:](https://rclone.org/webdav/#nextcloud)
   * OVH [:page_facing_up:](https://rclone.org/swift/)
   * OpenDrive [:page_facing_up:](https://rclone.org/opendrive/)
-  * Openstack Swift [:page_facing_up:](https://rclone.org/swift/)
+  * OpenStack Swift [:page_facing_up:](https://rclone.org/swift/)
   * Oracle Cloud Storage [:page_facing_up:](https://rclone.org/swift/)
   * ownCloud [:page_facing_up:](https://rclone.org/webdav/#owncloud)
   * pCloud [:page_facing_up:](https://rclone.org/pcloud/)
   * put.io [:page_facing_up:](https://rclone.org/webdav/#put-io)
   * QingStor [:page_facing_up:](https://rclone.org/qingstor/)
   * Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
+  * Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)
   * SFTP [:page_facing_up:](https://rclone.org/sftp/)
   * Wasabi [:page_facing_up:](https://rclone.org/s3/#wasabi)
   * WebDAV [:page_facing_up:](https://rclone.org/webdav/)
@@ -60,13 +62,13 @@ Please see [the full list of all storage providers and their features](https://r
 
 ## Features
 
-  * MD5/SHA1 hashes checked at all times for file integrity
+  * MD5/SHA-1 hashes checked at all times for file integrity
   * Timestamps preserved on files
   * Partial syncs supported on a whole file basis
   * [Copy](https://rclone.org/commands/rclone_copy/) mode to just copy new/changed files
   * [Sync](https://rclone.org/commands/rclone_sync/) (one way) mode to make a directory identical
   * [Check](https://rclone.org/commands/rclone_check/) mode to check for file hash equality
-  * Can sync to and from network, eg two different cloud accounts
+  * Can sync to and from network, e.g. two different cloud accounts
   * Optional encryption ([Crypt](https://rclone.org/crypt/))
   * Optional cache ([Cache](https://rclone.org/cache/))
   * Optional FUSE mount ([rclone mount](https://rclone.org/commands/rclone_mount/))
@@ -91,4 +93,4 @@ License
 -------
 
 This is free software under the terms of MIT the license (check the
-[COPYING file](/rclone/COPYING) included in this package).
+[COPYING file](/COPYING) included in this package).

backend/alias/alias.go

@@ -30,7 +30,7 @@ type Options struct {
 	Remote string `config:"remote"`
 }
 
-// NewFs contstructs an Fs from the path.
+// NewFs constructs an Fs from the path.
 //
 // The returned Fs is the actual Fs, referenced by remote in the config
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {

backend/alias/alias_internal_test.go

@@ -80,7 +80,7 @@ func TestNewFS(t *testing.T) {
 			wantEntry := test.entries[i]
 
 			require.Equal(t, wantEntry.remote, gotEntry.Remote(), what)
-			require.Equal(t, wantEntry.size, int64(gotEntry.Size()), what)
+			require.Equal(t, wantEntry.size, gotEntry.Size(), what)
 			_, isDir := gotEntry.(fs.Directory)
 			require.Equal(t, wantEntry.isDir, isDir, what)
 		}

backend/amazonclouddrive/amazonclouddrive.go

@@ -21,7 +21,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/ncw/go-acd"
+	acd "github.com/ncw/go-acd"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/configmap"
@@ -264,7 +264,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, acdConfig, baseClient)
 	if err != nil {
-		log.Fatalf("Failed to configure Amazon Drive: %v", err)
+		return nil, errors.Wrap(err, "failed to configure Amazon Drive")
 	}
 
 	c := acd.NewClient(oAuthClient)

backend/azureblob/azureblob.go

@@ -1,6 +1,6 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
 
-// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 
@@ -22,12 +22,14 @@ import (
 	"sync"
 	"time"
 
+	"github.com/Azure/azure-pipeline-go/pipeline"
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/accounting"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/fserrors"
+	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/pacer"
@@ -50,6 +52,7 @@ const (
 	defaultUploadCutoff = 256 * fs.MebiByte
 	maxUploadCutoff     = 256 * fs.MebiByte
 	defaultAccessTier   = azblob.AccessTierNone
+	maxTryTimeout       = time.Hour * 24 * 365 //max time of an azure web request response window (whether or not data is flowing)
 )
 
 // Register with Fs
@@ -74,7 +77,7 @@ func init() {
 		}, {
 			Name:     "upload_cutoff",
 			Help:     "Cutoff for switching to chunked upload (<= 256MB).",
-			Default:  fs.SizeSuffix(defaultUploadCutoff),
+			Default:  defaultUploadCutoff,
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -82,7 +85,7 @@ func init() {
 
 Note that this is stored in memory and there may be up to
 "--transfers" chunks stored at once in memory.`,
-			Default:  fs.SizeSuffix(defaultChunkSize),
+			Default:  defaultChunkSize,
 			Advanced: true,
 		}, {
 			Name: "list_chunk",
@@ -134,6 +137,7 @@ type Fs struct {
 	root             string                // the path we are working on if any
 	opt              Options               // parsed config options
 	features         *fs.Features          // optional features
+	client           *http.Client          // http client we are using
 	svcURL           *azblob.ServiceURL    // reference to serviceURL
 	cntURL           *azblob.ContainerURL  // reference to containerURL
 	container        string                // the container we are working on
@@ -271,7 +275,39 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// httpClientFactory creates a Factory object that sends HTTP requests
+// to a rclone's http.Client.
+//
+// copied from azblob.newDefaultHTTPClientFactory
+func httpClientFactory(client *http.Client) pipeline.Factory {
+	return pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
+		return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
+			r, err := client.Do(request.WithContext(ctx))
+			if err != nil {
+				err = pipeline.NewError(err, "HTTP request failed")
+			}
+			return pipeline.NewHTTPResponse(r), err
+		}
+	})
+}
+
+// newPipeline creates a Pipeline using the specified credentials and options.
+//
+// this code was copied from azblob.NewPipeline
+func (f *Fs) newPipeline(c azblob.Credential, o azblob.PipelineOptions) pipeline.Pipeline {
+	// Closest to API goes first; closest to the wire goes last
+	factories := []pipeline.Factory{
+		azblob.NewTelemetryPolicyFactory(o.Telemetry),
+		azblob.NewUniqueRequestIDPolicyFactory(),
+		azblob.NewRetryPolicyFactory(o.Retry),
+		c,
+		pipeline.MethodFactoryMarker(), // indicates at what stage in the pipeline the method factory is invoked
+		azblob.NewRequestLogPolicyFactory(o.RequestLog),
+	}
+	return pipeline.NewPipeline(factories, pipeline.Options{HTTPSender: httpClientFactory(f.client), Log: o.Log})
+}
+
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -306,6 +342,23 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			string(azblob.AccessTierHot), string(azblob.AccessTierCool), string(azblob.AccessTierArchive))
 	}
 
+	f := &Fs{
+		name:        name,
+		opt:         *opt,
+		container:   container,
+		root:        directory,
+		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant).SetPacer(pacer.S3Pacer),
+		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
+		client:      fshttp.NewClient(fs.Config),
+	}
+	f.features = (&fs.Features{
+		ReadMimeType:  true,
+		WriteMimeType: true,
+		BucketBased:   true,
+		SetTier:       true,
+		GetTier:       true,
+	}).Fill(f)
+
 	var (
 		u            *url.URL
 		serviceURL   azblob.ServiceURL
@@ -322,7 +375,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to make azure storage url from account and endpoint")
 		}
-		pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
+		pipeline := f.newPipeline(credential, azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
 		serviceURL = azblob.NewServiceURL(*u, pipeline)
 		containerURL = serviceURL.NewContainerURL(container)
 	case opt.SASURL != "":
@@ -331,7 +384,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 			return nil, errors.Wrapf(err, "failed to parse SAS URL")
 		}
 		// use anonymous credentials in case of sas url
-		pipeline := azblob.NewPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{})
+		pipeline := f.newPipeline(azblob.NewAnonymousCredential(), azblob.PipelineOptions{Retry: azblob.RetryOptions{TryTimeout: maxTryTimeout}})
 		// Check if we have container level SAS or account level sas
 		parts := azblob.NewBlobURLParts(*u)
 		if parts.ContainerName != "" {
@@ -339,7 +392,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 				return nil, errors.New("Container name in SAS URL and container provided in command do not match")
 			}
 
-			container = parts.ContainerName
 			containerURL = azblob.NewContainerURL(*u, pipeline)
 		} else {
 			serviceURL = azblob.NewServiceURL(*u, pipeline)
@@ -348,24 +400,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	default:
 		return nil, errors.New("Need account+key or connectionString or sasURL")
 	}
+	f.svcURL = &serviceURL
+	f.cntURL = &containerURL
 
-	f := &Fs{
-		name:        name,
-		opt:         *opt,
-		container:   container,
-		root:        directory,
-		svcURL:      &serviceURL,
-		cntURL:      &containerURL,
-		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
-		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
-	}
-	f.features = (&fs.Features{
-		ReadMimeType:  true,
-		WriteMimeType: true,
-		BucketBased:   true,
-		SetTier:       true,
-		GetTier:       true,
-	}).Fill(f)
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the (container,directory) is actually an existing file
@@ -379,8 +416,8 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		_, err := f.NewObject(remote)
 		if err != nil {
-			if err == fs.ErrorObjectNotFound {
-				// File doesn't exist so return old f
+			if err == fs.ErrorObjectNotFound || err == fs.ErrorNotAFile {
+				// File doesn't exist or is a directory so return old f
 				f.root = oldRoot
 				return f, nil
 			}
@@ -436,6 +473,21 @@ func (o *Object) updateMetadataWithModTime(modTime time.Time) {
 	o.meta[modTimeKey] = modTime.Format(timeFormatOut)
 }
 
+// Returns whether file is a directory marker or not
+func isDirectoryMarker(size int64, metadata azblob.Metadata, remote string) bool {
+	// Directory markers are 0 length
+	if size == 0 {
+		// Note that metadata with hdi_isfolder = true seems to be a
+		// defacto standard for marking blobs as directories.
+		endsWithSlash := strings.HasSuffix(remote, "/")
+		if endsWithSlash || remote == "" || metadata["hdi_isfolder"] == "true" {
+			return true
+		}
+
+	}
+	return false
+}
+
 // listFn is called from list to handle an object
 type listFn func(remote string, object *azblob.BlobItem, isDirectory bool) error
 
@@ -471,6 +523,7 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
 		MaxResults: int32(maxResults),
 	}
 	ctx := context.Background()
+	directoryMarkers := map[string]struct{}{}
 	for marker := (azblob.Marker{}); marker.NotDone(); {
 		var response *azblob.ListBlobsHierarchySegmentResponse
 		err := f.pacer.Call(func() (bool, error) {
@@ -500,13 +553,23 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
 				continue
 			}
 			remote := file.Name[len(f.root):]
-			// Check for directory
-			isDirectory := strings.HasSuffix(remote, "/")
-			if isDirectory {
-				remote = remote[:len(remote)-1]
+			if isDirectoryMarker(*file.Properties.ContentLength, file.Metadata, remote) {
+				if strings.HasSuffix(remote, "/") {
+					remote = remote[:len(remote)-1]
+				}
+				err = fn(remote, file, true)
+				if err != nil {
+					return err
+				}
+				// Keep track of directory markers. If recursing then
+				// there will be no Prefixes so no need to keep track
+				if !recurse {
+					directoryMarkers[remote] = struct{}{}
+				}
+				continue // skip directory marker
 			}
 			// Send object
-			err = fn(remote, file, isDirectory)
+			err = fn(remote, file, false)
 			if err != nil {
 				return err
 			}
@@ -519,6 +582,10 @@ func (f *Fs) list(dir string, recurse bool, maxResults uint, fn listFn) error {
 				continue
 			}
 			remote = remote[len(f.root):]
+			// Don't send if already sent as a directory marker
+			if _, found := directoryMarkers[remote]; found {
+				continue
+			}
 			// Send object
 			err = fn(remote, nil, true)
 			if err != nil {
@@ -686,6 +753,35 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.
 	return fs, fs.Update(in, src, options...)
 }
 
+// Check if the container exists
+//
+// NB this can return incorrect results if called immediately after container deletion
+func (f *Fs) dirExists() (bool, error) {
+	options := azblob.ListBlobsSegmentOptions{
+		Details: azblob.BlobListingDetails{
+			Copy:             false,
+			Metadata:         false,
+			Snapshots:        false,
+			UncommittedBlobs: false,
+			Deleted:          false,
+		},
+		MaxResults: 1,
+	}
+	err := f.pacer.Call(func() (bool, error) {
+		ctx := context.Background()
+		_, err := f.cntURL.ListBlobsHierarchySegment(ctx, azblob.Marker{}, "", options)
+		return f.shouldRetry(err)
+	})
+	if err == nil {
+		return true, nil
+	}
+	// Check http error code along with service code, current SDK doesn't populate service code correctly sometimes
+	if storageErr, ok := err.(azblob.StorageError); ok && (storageErr.ServiceCode() == azblob.ServiceCodeContainerNotFound || storageErr.Response().StatusCode == http.StatusNotFound) {
+		return false, nil
+	}
+	return false, err
+}
+
 // Mkdir creates the container if it doesn't exist
 func (f *Fs) Mkdir(dir string) error {
 	f.containerOKMu.Lock()
@@ -693,6 +789,15 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
+	if !f.containerDeleted {
+		exists, err := f.dirExists()
+		if err == nil {
+			f.containerOK = exists
+		}
+		if err != nil || exists {
+			return err
+		}
+	}
 
 	// now try to create the container
 	err := f.pacer.Call(func() (bool, error) {
@@ -705,6 +810,11 @@ func (f *Fs) Mkdir(dir string) error {
 					f.containerOK = true
 					return false, nil
 				case azblob.ServiceCodeContainerBeingDeleted:
+					// From https://docs.microsoft.com/en-us/rest/api/storageservices/delete-container
+					// When a container is deleted, a container with the same name cannot be created
+					// for at least 30 seconds; the container may not be available for more than 30
+					// seconds if the service is still processing the request.
+					time.Sleep(6 * time.Second) // default 10 retries will be 60 seconds
 					f.containerDeleted = true
 					return true, err
 				}
@@ -722,7 +832,7 @@ func (f *Fs) Mkdir(dir string) error {
 // isEmpty checks to see if a given directory is empty and returns an error if not
 func (f *Fs) isEmpty(dir string) (err error) {
 	empty := true
-	err = f.list("", true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
+	err = f.list(dir, true, 1, func(remote string, object *azblob.BlobItem, isDirectory bool) error {
 		empty = false
 		return nil
 	})
@@ -917,27 +1027,37 @@ func (o *Object) setMetadata(metadata azblob.Metadata) {
 //  o.md5
 //  o.meta
 func (o *Object) decodeMetaDataFromPropertiesResponse(info *azblob.BlobGetPropertiesResponse) (err error) {
+	metadata := info.NewMetadata()
+	size := info.ContentLength()
+	if isDirectoryMarker(size, metadata, o.remote) {
+		return fs.ErrorNotAFile
+	}
 	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
 	// this as base64 encoded string.
 	o.md5 = base64.StdEncoding.EncodeToString(info.ContentMD5())
 	o.mimeType = info.ContentType()
-	o.size = info.ContentLength()
-	o.modTime = time.Time(info.LastModified())
+	o.size = size
+	o.modTime = info.LastModified()
 	o.accessTier = azblob.AccessTierType(info.AccessTier())
-	o.setMetadata(info.NewMetadata())
+	o.setMetadata(metadata)
 
 	return nil
 }
 
 func (o *Object) decodeMetaDataFromBlob(info *azblob.BlobItem) (err error) {
+	metadata := info.Metadata
+	size := *info.Properties.ContentLength
+	if isDirectoryMarker(size, metadata, o.remote) {
+		return fs.ErrorNotAFile
+	}
 	// NOTE - Client library always returns MD5 as base64 decoded string, Object needs to maintain
 	// this as base64 encoded string.
 	o.md5 = base64.StdEncoding.EncodeToString(info.Properties.ContentMD5)
 	o.mimeType = *info.Properties.ContentType
-	o.size = *info.Properties.ContentLength
+	o.size = size
 	o.modTime = info.Properties.LastModified
 	o.accessTier = info.Properties.AccessTier
-	o.setMetadata(info.Metadata)
+	o.setMetadata(metadata)
 	return nil
 }
 
@@ -983,12 +1103,6 @@ func (o *Object) readMetaData() (err error) {
 	return o.decodeMetaDataFromPropertiesResponse(blobProperties)
 }
 
-// timeString returns modTime as the number of milliseconds
-// elapsed since January 1, 1970 UTC as a decimal string.
-func timeString(modTime time.Time) string {
-	return strconv.FormatInt(modTime.UnixNano()/1E6, 10)
-}
-
 // parseTimeString converts a decimal string number of milliseconds
 // elapsed since January 1, 1970 UTC into a time.Time and stores it in
 // the modTime variable.

backend/azureblob/azureblob_internal_test.go

@@ -1,4 +1,4 @@
-// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 

backend/azureblob/azureblob_test.go

@@ -1,6 +1,6 @@
 // Test AzureBlob filesystem interface
 
-// +build !freebsd,!netbsd,!openbsd,!plan9,!solaris,go1.8
+// +build !plan9,!solaris,go1.8
 
 package azureblob
 

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +1,6 @@
 // Build for azureblob for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build freebsd netbsd openbsd plan9 solaris !go1.8
+// +build plan9 solaris !go1.8
 
 package azureblob

backend/b2/api/types.go

@@ -17,12 +17,12 @@ type Error struct {
 	Message string `json:"message"` // A human-readable message, in English, saying what went wrong.
 }
 
-// Error statisfies the error interface
+// Error satisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (%d %s)", e.Message, e.Status, e.Code)
 }
 
-// Fatal statisfies the Fatal interface
+// Fatal satisfies the Fatal interface
 //
 // It indicates which errors should be treated as fatal
 func (e *Error) Fatal() bool {
@@ -100,7 +100,7 @@ func RemoveVersion(remote string) (t Timestamp, newRemote string) {
 	return Timestamp(newT), base[:versionStart] + ext
 }
 
-// IsZero returns true if the timestamp is unitialised
+// IsZero returns true if the timestamp is uninitialized
 func (t Timestamp) IsZero() bool {
 	return time.Time(t).IsZero()
 }
@@ -136,6 +136,7 @@ type AuthorizeAccountResponse struct {
 	AccountID               string   `json:"accountId"`               // The identifier for the account.
 	Allowed                 struct { // An object (see below) containing the capabilities of this auth token, and any restrictions on using it.
 		BucketID     string      `json:"bucketId"`     // When present, access is restricted to one bucket.
+		BucketName   string      `json:"bucketName"`   // When present, name of bucket - may be empty
 		Capabilities []string    `json:"capabilities"` // A list of strings, each one naming a capability the key has.
 		NamePrefix   interface{} `json:"namePrefix"`   // When present, access is restricted to files whose names start with the prefix
 	} `json:"allowed"`

backend/b2/b2.go

@@ -108,7 +108,7 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
 Files above this size will be uploaded in chunks of "--b2-chunk-size".
 
 This value should be set no larger than 4.657GiB (== 5GB).`,
-			Default:  fs.SizeSuffix(defaultUploadCutoff),
+			Default:  defaultUploadCutoff,
 			Advanced: true,
 		}, {
 			Name: "chunk_size",
@@ -117,8 +117,13 @@ This value should be set no larger than 4.657GiB (== 5GB).`,
 When uploading large files, chunk the file into this size.  Note that
 these chunks are buffered in memory and there might a maximum of
 "--transfers" chunks in progress at once.  5,000,000 Bytes is the
-minimim size.`,
-			Default:  fs.SizeSuffix(defaultChunkSize),
+minimum size.`,
+			Default:  defaultChunkSize,
+			Advanced: true,
+		}, {
+			Name:     "disable_checksum",
+			Help:     `Disable checksums for large (> upload cutoff) files`,
+			Default:  false,
 			Advanced: true,
 		}},
 	})
@@ -126,14 +131,15 @@ minimim size.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	Account      string        `config:"account"`
-	Key          string        `config:"key"`
-	Endpoint     string        `config:"endpoint"`
-	TestMode     string        `config:"test_mode"`
-	Versions     bool          `config:"versions"`
-	HardDelete   bool          `config:"hard_delete"`
-	UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
-	ChunkSize    fs.SizeSuffix `config:"chunk_size"`
+	Account         string        `config:"account"`
+	Key             string        `config:"key"`
+	Endpoint        string        `config:"endpoint"`
+	TestMode        string        `config:"test_mode"`
+	Versions        bool          `config:"versions"`
+	HardDelete      bool          `config:"hard_delete"`
+	UploadCutoff    fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize       fs.SizeSuffix `config:"chunk_size"`
+	DisableCheckSum bool          `config:"disable_checksum"`
 }
 
 // Fs represents a remote b2 server
@@ -313,7 +319,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs contstructs an Fs from the path, bucket:path
+// NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -368,6 +374,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 	// If this is a key limited to a single bucket, it must exist already
 	if f.bucket != "" && f.info.Allowed.BucketID != "" {
+		allowedBucket := f.info.Allowed.BucketName
+		if allowedBucket == "" {
+			return nil, errors.New("bucket that application key is restricted to no longer exists")
+		}
+		if allowedBucket != f.bucket {
+			return nil, errors.Errorf("you must use bucket %q with this application key", allowedBucket)
+		}
 		f.markBucketOK()
 		f.setBucketID(f.info.Allowed.BucketID)
 	}
@@ -980,6 +993,12 @@ func (f *Fs) purge(oldOnly bool) error {
 			errReturn = err
 		}
 	}
+	var isUnfinishedUploadStale = func(timestamp api.Timestamp) bool {
+		if time.Since(time.Time(timestamp)).Hours() > 24 {
+			return true
+		}
+		return false
+	}
 
 	// Delete Config.Transfers in parallel
 	toBeDeleted := make(chan *api.File, fs.Config.Transfers)
@@ -1003,6 +1022,9 @@ func (f *Fs) purge(oldOnly bool) error {
 				if object.Action == "hide" {
 					fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)
 					toBeDeleted <- object
+				} else if object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {
+					fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())
+					toBeDeleted <- object
 				} else {
 					fs.Debugf(remote, "Not deleting current version (id %q) %q", object.ID, object.Action)
 				}
@@ -1437,7 +1459,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	// Content-Type b2/x-auto to automatically set the stored Content-Type
 	// post upload. In the case where a file extension is absent or the
 	// lookup fails, the Content-Type is set to application/octet-stream. The
-	// Content-Type mappings can be purused here.
+	// Content-Type mappings can be pursued here.
 	//
 	// X-Bz-Content-Sha1
 	// required
@@ -1484,11 +1506,6 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		},
 		ContentLength: &size,
 	}
-	// for go1.8 (see release notes) we must nil the Body if we want a
-	// "Content-Length: 0" header which b2 requires for all files.
-	if size == 0 {
-		opts.Body = nil
-	}
 	var response api.FileInfo
 	// Don't retry, return a retry error instead
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {

backend/b2/upload.go

@@ -116,8 +116,10 @@ func (f *Fs) newLargeUpload(o *Object, in io.Reader, src fs.ObjectInfo) (up *lar
 		},
 	}
 	// Set the SHA1 if known
-	if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
-		request.Info[sha1Key] = calculatedSha1
+	if !o.fs.opt.DisableCheckSum {
+		if calculatedSha1, err := src.Hash(hash.SHA1); err == nil && calculatedSha1 != "" {
+			request.Info[sha1Key] = calculatedSha1
+		}
 	}
 	var response api.StartLargeFileResponse
 	err = f.pacer.Call(func() (bool, error) {

backend/box/api/types.go

@@ -45,7 +45,7 @@ type Error struct {
 	RequestID   string `json:"request_id"`
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	out := fmt.Sprintf("Error %q (%d)", e.Code, e.Status)
 	if e.Message != "" {
@@ -57,7 +57,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error statisfies the error interface
+// Check Error satisfies the error interface
 var _ error = (*Error)(nil)
 
 // ItemFields are the fields needed for FileInfo

backend/box/box.go

@@ -171,13 +171,13 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRety := false
+	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRety = true
+		authRetry = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
 // substitute reserved characters for box
@@ -252,7 +252,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	root = parsePath(root)
 	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure Box: %v", err)
+		return nil, errors.Wrap(err, "failed to configure Box")
 	}
 
 	f := &Fs{
@@ -530,10 +530,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 //
 // The new object may have been created if an error is returned
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return exisitingObj, exisitingObj.Update(in, src, options...)
+		return existingObj, existingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)

backend/box/upload.go

@@ -211,8 +211,8 @@ outer:
 		}
 
 		reqSize := remaining
-		if reqSize >= int64(chunkSize) {
-			reqSize = int64(chunkSize)
+		if reqSize >= chunkSize {
+			reqSize = chunkSize
 		}
 
 		// Make a block of memory

backend/cache/cache.go

@@ -576,7 +576,7 @@ The slice indices are similar to Python slices: start[:end]
 
 start is the 0 based chunk number from the beginning of the file
 to fetch inclusive. end is 0 based chunk number from the beginning
-of the file to fetch exclisive.
+of the file to fetch exclusive.
 Both values can be negative, in which case they count from the back
 of the file. The value "-5:" represents the last 5 chunks of a file.
 
@@ -870,7 +870,7 @@ func (f *Fs) notifyChangeUpstream(remote string, entryType fs.EntryType) {
 	}
 }
 
-// ChangeNotify can subsribe multiple callers
+// ChangeNotify can subscribe multiple callers
 // this is coupled with the wrapped fs ChangeNotify (if it supports it)
 // and also notifies other caches (i.e VFS) to clear out whenever something changes
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollInterval <-chan time.Duration) {
@@ -1549,7 +1549,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 
 	if srcObj.isTempFile() {
-		// we check if the feature is stil active
+		// we check if the feature is still active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't copy - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantCopy
@@ -1625,7 +1625,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 
 	// if this is a temp object then we perform the changes locally
 	if srcObj.isTempFile() {
-		// we check if the feature is stil active
+		// we check if the feature is still active
 		if f.opt.TempWritePath == "" {
 			fs.Errorf(srcObj, "can't move - this is a local cached file but this feature is turned off this run")
 			return nil, fs.ErrorCantMove

backend/cache/cache_internal_test.go

@@ -387,10 +387,10 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
-	require.Equal(t, o.Size(), int64(testSize))
+	require.Equal(t, o.Size(), testSize)
 	time.Sleep(time.Second * 3)
 
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, testSize, false)
 	require.NoError(t, err)
 	require.Equal(t, int64(len(checkSample)), o.Size())
 
@@ -726,6 +726,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// Call the rc function
 	m, err := cacheExpire.Fn(rc.Params{"remote": "data.bin"})
+	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -735,18 +736,21 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
-	li1, err := runInstance.list(t, rootFs, "")
+	_, err = runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
 
 	// create some rand test data
 	testData2 := randStringBytes(int(chunkSize))
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
 
 	// list should have 1 item only
-	li1, err = runInstance.list(t, rootFs, "")
+	li1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
 	require.Len(t, li1, 1)
 
 	// Call the rc function
 	m, err = cacheExpire.Fn(rc.Params{"remote": "/"})
+	require.NoError(t, err)
 	require.Contains(t, m, "status")
 	require.Contains(t, m, "message")
 	require.Equal(t, "ok", m["status"])
@@ -754,6 +758,7 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 
 	// list should have 2 items now
 	li2, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
 	require.Len(t, li2, 2)
 }
 
@@ -1490,7 +1495,8 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 	var err error
 
 	if r.useMount {
-		f, err := os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+		var f *os.File
+		f, err = os.OpenFile(path.Join(runInstance.mntDir, src), os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
 		if err != nil {
 			return err
 		}
@@ -1500,7 +1506,8 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		}()
 		_, err = f.WriteString(data + append)
 	} else {
-		obj1, err := rootFs.NewObject(src)
+		var obj1 fs.Object
+		obj1, err = rootFs.NewObject(src)
 		if err != nil {
 			return err
 		}
@@ -1632,15 +1639,13 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	cfs, ok := f.(*cache.Fs)
 	if ok {
 		return cfs, nil
-	} else {
-		if f.Features().UnWrap != nil {
-			cfs, ok := f.Features().UnWrap().(*cache.Fs)
-			if ok {
-				return cfs, nil
-			}
+	}
+	if f.Features().UnWrap != nil {
+		cfs, ok := f.Features().UnWrap().(*cache.Fs)
+		if ok {
+			return cfs, nil
 		}
 	}
-
 	return nil, errors.New("didn't found a cache fs")
 }
 

backend/cache/plex.go

@@ -15,7 +15,7 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	"github.com/patrickmn/go-cache"
+	cache "github.com/patrickmn/go-cache"
 	"golang.org/x/net/websocket"
 )
 

backend/cache/storage_memory.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/ncw/rclone/fs"
-	"github.com/patrickmn/go-cache"
+	cache "github.com/patrickmn/go-cache"
 	"github.com/pkg/errors"
 )
 

backend/cache/storage_persistent.go

@@ -398,7 +398,7 @@ func (b *Persistent) AddObject(cachedObject *Object) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", cachedObject, err)
 		}
-		err = bucket.Put([]byte(cachedObject.Name), []byte(encoded))
+		err = bucket.Put([]byte(cachedObject.Name), encoded)
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", cachedObject, err)
 		}
@@ -809,7 +809,7 @@ func (b *Persistent) addPendingUpload(destPath string, started bool) error {
 		if err != nil {
 			return errors.Errorf("couldn't marshal object (%v) info: %v", destPath, err)
 		}
-		err = bucket.Put([]byte(destPath), []byte(encoded))
+		err = bucket.Put([]byte(destPath), encoded)
 		if err != nil {
 			return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 		}
@@ -1049,7 +1049,7 @@ func (b *Persistent) ReconcileTempUploads(cacheFs *Fs) error {
 			if err != nil {
 				return errors.Errorf("couldn't marshal object (%v) info: %v", queuedEntry, err)
 			}
-			err = bucket.Put([]byte(destPath), []byte(encoded))
+			err = bucket.Put([]byte(destPath), encoded)
 			if err != nil {
 				return errors.Errorf("couldn't cache object (%v) info: %v", destPath, err)
 			}

backend/crypt/cipher.go

@@ -41,6 +41,7 @@ var (
 	ErrorBadDecryptControlChar   = errors.New("bad decryption - contains control chars")
 	ErrorNotAMultipleOfBlocksize = errors.New("not a multiple of blocksize")
 	ErrorTooShortAfterDecode     = errors.New("too short after base32 decode")
+	ErrorTooLongAfterDecode      = errors.New("too long after base32 decode")
 	ErrorEncryptedFileTooShort   = errors.New("file is too short to be encrypted")
 	ErrorEncryptedFileBadHeader  = errors.New("file has truncated block header")
 	ErrorEncryptedBadMagic       = errors.New("not an encrypted file - bad magic string")
@@ -284,6 +285,9 @@ func (c *cipher) decryptSegment(ciphertext string) (string, error) {
 		// not possible if decodeFilename() working correctly
 		return "", ErrorTooShortAfterDecode
 	}
+	if len(rawCiphertext) > 2048 {
+		return "", ErrorTooLongAfterDecode
+	}
 	paddedPlaintext := eme.Transform(c.block, c.nameTweak[:], rawCiphertext, eme.DirectionDecrypt)
 	plaintext, err := pkcs7.Unpad(nameCipherBlockSize, paddedPlaintext)
 	if err != nil {
@@ -459,7 +463,7 @@ func (c *cipher) deobfuscateSegment(ciphertext string) (string, error) {
 			if int(newRune) < base {
 				newRune += 256
 			}
-			_, _ = result.WriteRune(rune(newRune))
+			_, _ = result.WriteRune(newRune)
 
 		default:
 			_, _ = result.WriteRune(runeValue)
@@ -744,7 +748,7 @@ func (c *cipher) newDecrypter(rc io.ReadCloser) (*decrypter, error) {
 	if !bytes.Equal(readBuf[:fileMagicSize], fileMagicBytes) {
 		return nil, fh.finishAndClose(ErrorEncryptedBadMagic)
 	}
-	// retreive the nonce
+	// retrieve the nonce
 	fh.nonce.fromBuf(readBuf[fileMagicSize:])
 	fh.initialNonce = fh.nonce
 	return fh, nil

backend/crypt/cipher_test.go

@@ -194,6 +194,10 @@ func TestEncryptSegment(t *testing.T) {
 
 func TestDecryptSegment(t *testing.T) {
 	// We've tested the forwards above, now concentrate on the errors
+	longName := make([]byte, 3328)
+	for i := range longName {
+		longName[i] = 'a'
+	}
 	c, _ := newCipher(NameEncryptionStandard, "", "", true)
 	for _, test := range []struct {
 		in          string
@@ -201,6 +205,7 @@ func TestDecryptSegment(t *testing.T) {
 	}{
 		{"64=", ErrorBadBase32Encoding},
 		{"!", base32.CorruptInputError(0)},
+		{string(longName), ErrorTooLongAfterDecode},
 		{encodeFileName([]byte("a")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef")), ErrorNotAMultipleOfBlocksize},
 		{encodeFileName([]byte("123456789abcdef0")), pkcs7.ErrorPaddingTooLong},

backend/crypt/crypt.go

@@ -122,7 +122,7 @@ func NewCipher(m configmap.Mapper) (Cipher, error) {
 	return newCipherForConfig(opt)
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -555,7 +555,7 @@ func (f *Fs) DecryptFileName(encryptedFileName string) (string, error) {
 }
 
 // ComputeHash takes the nonce from o, and encrypts the contents of
-// src with it, and calcuates the hash given by HashType on the fly
+// src with it, and calculates the hash given by HashType on the fly
 //
 // Note that we break lots of encapsulation in this function.
 func (f *Fs) ComputeHash(o *Object, src fs.Object, hashType hash.Type) (hashStr string, err error) {

backend/drive/drive.go

@@ -1,4 +1,7 @@
 // Package drive interfaces with the Google Drive object storage system
+
+// +build go1.9
+
 package drive
 
 // FIXME need to deal with some corner cases
@@ -18,6 +21,7 @@ import (
 	"net/url"
 	"os"
 	"path"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -36,6 +40,7 @@ import (
 	"github.com/ncw/rclone/lib/dircache"
 	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
+	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -51,7 +56,8 @@ const (
 	driveFolderType             = "application/vnd.google-apps.folder"
 	timeFormatIn                = time.RFC3339
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
-	minSleep                    = 10 * time.Millisecond
+	defaultMinSleep             = fs.Duration(100 * time.Millisecond)
+	defaultBurst                = 100
 	defaultExportExtensions     = "docx,xlsx,pptx,svg"
 	scopePrefix                 = "https://www.googleapis.com/auth/"
 	defaultScope                = "drive"
@@ -122,6 +128,29 @@ var (
 	_linkTemplates   map[string]*template.Template // available link types
 )
 
+// Parse the scopes option returning a slice of scopes
+func driveScopes(scopesString string) (scopes []string) {
+	if scopesString == "" {
+		scopesString = defaultScope
+	}
+	for _, scope := range strings.Split(scopesString, ",") {
+		scope = strings.TrimSpace(scope)
+		scopes = append(scopes, scopePrefix+scope)
+	}
+	return scopes
+}
+
+// Returns true if one of the scopes was "drive.appfolder"
+func driveScopesContainsAppFolder(scopes []string) bool {
+	for _, scope := range scopes {
+		if scope == scopePrefix+"drive.appfolder" {
+			return true
+		}
+
+	}
+	return false
+}
+
 // Register with Fs
 func init() {
 	fs.Register(&fs.RegInfo{
@@ -136,18 +165,14 @@ func init() {
 				fs.Errorf(nil, "Couldn't parse config into struct: %v", err)
 				return
 			}
+
 			// Fill in the scopes
-			if opt.Scope == "" {
-				opt.Scope = defaultScope
-			}
-			driveConfig.Scopes = nil
-			for _, scope := range strings.Split(opt.Scope, ",") {
-				driveConfig.Scopes = append(driveConfig.Scopes, scopePrefix+strings.TrimSpace(scope))
-				// Set the root_folder_id if using drive.appfolder
-				if scope == "drive.appfolder" {
-					m.Set("root_folder_id", "appDataFolder")
-				}
+			driveConfig.Scopes = driveScopes(opt.Scope)
+			// Set the root_folder_id if using drive.appfolder
+			if driveScopesContainsAppFolder(driveConfig.Scopes) {
+				m.Set("root_folder_id", "appDataFolder")
 			}
+
 			if opt.ServiceAccountFile == "" {
 				err = oauthutil.Config("drive", name, m, driveConfig)
 				if err != nil {
@@ -334,6 +359,16 @@ will download it anyway.`,
 			Default:  fs.SizeSuffix(-1),
 			Help:     "If Object's are greater, use drive v2 API to download.",
 			Advanced: true,
+		}, {
+			Name:     "pacer_min_sleep",
+			Default:  defaultMinSleep,
+			Help:     "Minimum time to sleep between API calls.",
+			Advanced: true,
+		}, {
+			Name:     "pacer_burst",
+			Default:  defaultBurst,
+			Help:     "Number of API calls to allow without sleeping.",
+			Advanced: true,
 		}},
 	})
 
@@ -376,6 +411,8 @@ type Options struct {
 	AcknowledgeAbuse          bool          `config:"acknowledge_abuse"`
 	KeepRevisionForever       bool          `config:"keep_revision_forever"`
 	V2DownloadMinSize         fs.SizeSuffix `config:"v2_download_min_size"`
+	PacerMinSleep             fs.Duration   `config:"pacer_min_sleep"`
+	PacerBurst                int           `config:"pacer_burst"`
 }
 
 // Fs represents a remote drive server
@@ -445,7 +482,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whehter a given err rates being retried
+// shouldRetry determines whether a given err rates being retried
 func shouldRetry(err error) (bool, error) {
 	if err == nil {
 		return false, nil
@@ -696,12 +733,16 @@ func parseExtensions(extensionsIn ...string) (extensions, mimeTypes []string, er
 
 // Figure out if the user wants to use a team drive
 func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
+	// Stop if we are running non-interactive config
+	if fs.Config.AutoConfirm {
+		return nil
+	}
 	if opt.TeamDriveID == "" {
 		fmt.Printf("Configure this as a team drive?\n")
 	} else {
 		fmt.Printf("Change current team drive ID %q?\n", opt.TeamDriveID)
 	}
-	if !config.ConfirmWithDefault(false) {
+	if !config.Confirm() {
 		return nil
 	}
 	client, err := createOAuthClient(opt, name, m)
@@ -718,7 +759,7 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 	listFailed := false
 	for {
 		var teamDrives *drive.TeamDriveList
-		err = newPacer().Call(func() (bool, error) {
+		err = newPacer(opt).Call(func() (bool, error) {
 			teamDrives, err = listTeamDrives.Do()
 			return shouldRetry(err)
 		})
@@ -748,12 +789,13 @@ func configTeamDrive(opt *Options, m configmap.Mapper, name string) error {
 }
 
 // newPacer makes a pacer configured for drive
-func newPacer() *pacer.Pacer {
-	return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
+func newPacer(opt *Options) *pacer.Pacer {
+	return pacer.New().SetMinSleep(time.Duration(opt.PacerMinSleep)).SetBurst(opt.PacerBurst).SetPacer(pacer.GoogleDrivePacer)
 }
 
 func getServiceAccountClient(opt *Options, credentialsData []byte) (*http.Client, error) {
-	conf, err := google.JWTConfigFromJSON(credentialsData, driveConfig.Scopes...)
+	scopes := driveScopes(opt.Scope)
+	conf, err := google.JWTConfigFromJSON(credentialsData, scopes...)
 	if err != nil {
 		return nil, errors.Wrap(err, "error processing credentials")
 	}
@@ -821,7 +863,7 @@ func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
 	return
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -852,7 +894,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
 		name:  name,
 		root:  root,
 		opt:   *opt,
-		pacer: newPacer(),
+		pacer: newPacer(opt),
 	}
 	f.isTeamDrive = opt.TeamDriveID != ""
 	f.features = (&fs.Features{
@@ -1298,17 +1340,46 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
+// listREntry is a task to be executed by a litRRunner
+type listREntry struct {
+	id, path string
+}
+
+// listRSlices is a helper struct to sort two slices at once
+type listRSlices struct {
+	dirs  []string
+	paths []string
+}
+
+func (s listRSlices) Sort() {
+	sort.Sort(s)
+}
+
+func (s listRSlices) Len() int {
+	return len(s.dirs)
+}
+
+func (s listRSlices) Swap(i, j int) {
+	s.dirs[i], s.dirs[j] = s.dirs[j], s.dirs[i]
+	s.paths[i], s.paths[j] = s.paths[j], s.paths[i]
+}
+
+func (s listRSlices) Less(i, j int) bool {
+	return s.dirs[i] < s.dirs[j]
+}
+
 // listRRunner will read dirIDs from the in channel, perform the file listing an call cb with each DirEntry.
 //
-// In each cycle, will wait up to 10ms to read up to grouping entries from the in channel.
+// In each cycle it will read up to grouping entries from the in channel without blocking.
 // If an error occurs it will be send to the out channel and then return. Once the in channel is closed,
 // nil is send to the out channel and the function returns.
-func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
+func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan listREntry, out chan<- error, cb func(fs.DirEntry) error, grouping int) {
 	var dirs []string
+	var paths []string
 
 	for dir := range in {
-		dirs = append(dirs[:0], dir)
-		wait := time.After(10 * time.Millisecond)
+		dirs = append(dirs[:0], dir.id)
+		paths = append(paths[:0], dir.path)
 	waitloop:
 		for i := 1; i < grouping; i++ {
 			select {
@@ -1316,31 +1387,32 @@ func (f *Fs) listRRunner(wg *sync.WaitGroup, in <-chan string, out chan<- error,
 				if !ok {
 					break waitloop
 				}
-				dirs = append(dirs, d)
-			case <-wait:
-				break waitloop
+				dirs = append(dirs, d.id)
+				paths = append(paths, d.path)
+			default:
 			}
 		}
+		listRSlices{dirs, paths}.Sort()
 		var iErr error
 		_, err := f.list(dirs, "", false, false, false, func(item *drive.File) bool {
-			parentPath := ""
-			if len(item.Parents) > 0 {
-				p, ok := f.dirCache.GetInv(item.Parents[0])
-				if ok {
-					parentPath = p
+			for _, parent := range item.Parents {
+				// only handle parents that are in the requested dirs list
+				i := sort.SearchStrings(dirs, parent)
+				if i == len(dirs) || dirs[i] != parent {
+					continue
+				}
+				remote := path.Join(paths[i], item.Name)
+				entry, err := f.itemToDirEntry(remote, item)
+				if err != nil {
+					iErr = err
+					return true
 				}
-			}
-			remote := path.Join(parentPath, item.Name)
-			entry, err := f.itemToDirEntry(remote, item)
-			if err != nil {
-				iErr = err
-				return true
-			}
 
-			err = cb(entry)
-			if err != nil {
-				iErr = err
-				return true
+				err = cb(entry)
+				if err != nil {
+					iErr = err
+					return true
+				}
 			}
 			return false
 		})
@@ -1391,30 +1463,44 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	if err != nil {
 		return err
 	}
+	if directoryID == "root" {
+		var info *drive.File
+		err = f.pacer.CallNoRetry(func() (bool, error) {
+			info, err = f.svc.Files.Get("root").
+				Fields("id").
+				SupportsTeamDrives(f.isTeamDrive).
+				Do()
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return err
+		}
+		directoryID = info.Id
+	}
 
 	mu := sync.Mutex{} // protects in and overflow
 	wg := sync.WaitGroup{}
-	in := make(chan string, inputBuffer)
+	in := make(chan listREntry, inputBuffer)
 	out := make(chan error, fs.Config.Checkers)
 	list := walk.NewListRHelper(callback)
-	overfflow := []string{}
+	overflow := []listREntry{}
 
 	cb := func(entry fs.DirEntry) error {
 		mu.Lock()
 		defer mu.Unlock()
 		if d, isDir := entry.(*fs.Dir); isDir && in != nil {
 			select {
-			case in <- d.ID():
+			case in <- listREntry{d.ID(), d.Remote()}:
 				wg.Add(1)
 			default:
-				overfflow = append(overfflow, d.ID())
+				overflow = append(overflow, listREntry{d.ID(), d.Remote()})
 			}
 		}
 		return list.Add(entry)
 	}
 
 	wg.Add(1)
-	in <- directoryID
+	in <- listREntry{directoryID, dir}
 
 	for i := 0; i < fs.Config.Checkers; i++ {
 		go f.listRRunner(&wg, in, out, cb, grouping)
@@ -1423,18 +1509,18 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 		// wait until the all directories are processed
 		wg.Wait()
 		// if the input channel overflowed add the collected entries to the channel now
-		for len(overfflow) > 0 {
+		for len(overflow) > 0 {
 			mu.Lock()
-			l := len(overfflow)
-			// only fill half of the channel to prevent entries beeing put into overfflow again
+			l := len(overflow)
+			// only fill half of the channel to prevent entries beeing put into overflow again
 			if l > inputBuffer/2 {
 				l = inputBuffer / 2
 			}
 			wg.Add(l)
-			for _, d := range overfflow[:l] {
+			for _, d := range overflow[:l] {
 				in <- d
 			}
-			overfflow = overfflow[l:]
+			overflow = overflow[l:]
 			mu.Unlock()
 
 			// wait again for the completion of all directories
@@ -1625,14 +1711,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.Name, srcDir)
+				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.Name, srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.rmdir(srcDir.ID(), true)
 		if err != nil {
-			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -2051,7 +2137,7 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // ChangeNotify calls the passed function with a path that has had changes.
 // If the implementation uses polling, it should adhere to the given interval.
 //
-// Automatically restarts itself in case of unexpected behaviour of the remote.
+// Automatically restarts itself in case of unexpected behavior of the remote.
 //
 // Close the returned channel to stop being notified.
 func (f *Fs) ChangeNotify(notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
@@ -2158,11 +2244,13 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), startPage
 
 				// translate the parent dir of this object
 				if len(change.File.Parents) > 0 {
-					if parentPath, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
-						// and append the drive file name to compute the full file name
-						newPath := path.Join(parentPath, change.File.Name)
-						// this will now clear the actual file too
-						pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
+					for _, parent := range change.File.Parents {
+						if parentPath, ok := f.dirCache.GetInv(parent); ok {
+							// and append the drive file name to compute the full file name
+							newPath := path.Join(parentPath, change.File.Name)
+							// this will now clear the actual file too
+							pathsToClear = append(pathsToClear, entryType{path: newPath, entryType: changeType})
+						}
 					}
 				} else { // a true root object that is changed
 					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
@@ -2454,16 +2542,32 @@ func (o *documentObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err e
 	// Update the size with what we are reading as it can change from
 	// the HEAD in the listing to this GET. This stops rclone marking
 	// the transfer as corrupted.
+	var offset, end int64 = 0, -1
+	var newOptions = options[:0]
 	for _, o := range options {
+		// Note that Range requests don't work on Google docs:
 		// https://developers.google.com/drive/v3/web/manage-downloads#partial_download
-		if _, ok := o.(*fs.RangeOption); ok {
-			return nil, errors.New("partial downloads are not supported while exporting Google Documents")
+		// So do a subset of them manually
+		switch x := o.(type) {
+		case *fs.RangeOption:
+			offset, end = x.Start, x.End
+		case *fs.SeekOption:
+			offset, end = x.Offset, -1
+		default:
+			newOptions = append(newOptions, o)
 		}
 	}
+	options = newOptions
+	if offset != 0 {
+		return nil, errors.New("partial downloads are not supported while exporting Google Documents")
+	}
 	in, err = o.baseObject.open(o.url, options...)
 	if in != nil {
 		in = &openDocumentFile{o: o, in: in}
 	}
+	if end >= 0 {
+		in = readers.NewLimitedReadCloser(in, end-offset+1)
+	}
 	return
 }
 func (o *linkObject) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
@@ -2529,6 +2633,9 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 	newO, err := o.fs.newObjectWithInfo(src.Remote(), info)
+	if err != nil {
+		return err
+	}
 	switch newO := newO.(type) {
 	case *Object:
 		*o = *newO
@@ -2567,6 +2674,9 @@ func (o *documentObject) Update(in io.Reader, src fs.ObjectInfo, options ...fs.O
 	remote = remote[:len(remote)-o.extLen]
 
 	newO, err := o.fs.newObjectWithInfo(remote, info)
+	if err != nil {
+		return err
+	}
 	switch newO := newO.(type) {
 	case *documentObject:
 		*o = *newO

backend/drive/drive_internal_test.go

@@ -1,3 +1,5 @@
+// +build go1.9
+
 package drive
 
 import (
@@ -20,6 +22,31 @@ import (
 	"google.golang.org/api/drive/v3"
 )
 
+func TestDriveScopes(t *testing.T) {
+	for _, test := range []struct {
+		in       string
+		want     []string
+		wantFlag bool
+	}{
+		{"", []string{
+			"https://www.googleapis.com/auth/drive",
+		}, false},
+		{" drive.file , drive.readonly", []string{
+			"https://www.googleapis.com/auth/drive.file",
+			"https://www.googleapis.com/auth/drive.readonly",
+		}, false},
+		{" drive.file , drive.appfolder", []string{
+			"https://www.googleapis.com/auth/drive.file",
+			"https://www.googleapis.com/auth/drive.appfolder",
+		}, true},
+	} {
+		got := driveScopes(test.in)
+		assert.Equal(t, test.want, got, test.in)
+		gotFlag := driveScopesContainsAppFolder(got)
+		assert.Equal(t, test.wantFlag, gotFlag, test.in)
+	}
+}
+
 /*
 var additionalMimeTypes = map[string]string{
 	"application/vnd.ms-excel.sheet.macroenabled.12":                          ".xlsm",
@@ -243,10 +270,19 @@ func (f *Fs) InternalTestDocumentLink(t *testing.T) {
 }
 
 func (f *Fs) InternalTest(t *testing.T) {
-	t.Run("DocumentImport", f.InternalTestDocumentImport)
-	t.Run("DocumentUpdate", f.InternalTestDocumentUpdate)
-	t.Run("DocumentExport", f.InternalTestDocumentExport)
-	t.Run("DocumentLink", f.InternalTestDocumentLink)
+	// These tests all depend on each other so run them as nested tests
+	t.Run("DocumentImport", func(t *testing.T) {
+		f.InternalTestDocumentImport(t)
+		t.Run("DocumentUpdate", func(t *testing.T) {
+			f.InternalTestDocumentUpdate(t)
+			t.Run("DocumentExport", func(t *testing.T) {
+				f.InternalTestDocumentExport(t)
+				t.Run("DocumentLink", func(t *testing.T) {
+					f.InternalTestDocumentLink(t)
+				})
+			})
+		})
+	})
 }
 
 var _ fstests.InternalTester = (*Fs)(nil)

backend/drive/drive_test.go

@@ -1,4 +1,7 @@
 // Test Drive filesystem interface
+
+// +build go1.9
+
 package drive
 
 import (

backend/drive/drive_unsupported.go

@@ -0,0 +1,6 @@
+// Build for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build !go1.9
+
+package drive

backend/drive/upload.go

@@ -8,6 +8,8 @@
 //
 // This contains code adapted from google.golang.org/api (C) the GO AUTHORS
 
+// +build go1.9
+
 package drive
 
 import (
@@ -183,7 +185,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 	// been 200 OK.
 	//
 	// So parse the response out of the body.  We aren't expecting
-	// any other 2xx codes, so we parse it unconditionaly on
+	// any other 2xx codes, so we parse it unconditionally on
 	// StatusCode
 	if err = json.NewDecoder(res.Body).Decode(&rx.ret); err != nil {
 		return 598, err

backend/dropbox/dropbox.go

@@ -31,9 +31,11 @@ import (
 	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/auth"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/common"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/sharing"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/team"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
@@ -128,8 +130,13 @@ Any files larger than this will be uploaded in chunks of this size.
 Note that chunks are buffered in memory (one at a time) so rclone can
 deal with retries.  Setting this larger will increase the speed
 slightly (at most 10%% for 128MB in tests) at the cost of using more
-memory.  It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxChunkSize)),
-			Default:  fs.SizeSuffix(defaultChunkSize),
+memory.  It can be set smaller if you are tight on memory.`, maxChunkSize),
+			Default:  defaultChunkSize,
+			Advanced: true,
+		}, {
+			Name:     "impersonate",
+			Help:     "Impersonate this user when using a business account.",
+			Default:  "",
 			Advanced: true,
 		}},
 	})
@@ -137,7 +144,8 @@ memory.  It can be set smaller if you are tight on memory.`, fs.SizeSuffix(maxCh
 
 // Options defines the configuration for this backend
 type Options struct {
-	ChunkSize fs.SizeSuffix `config:"chunk_size"`
+	ChunkSize   fs.SizeSuffix `config:"chunk_size"`
+	Impersonate string        `config:"impersonate"`
 }
 
 // Fs represents a remote dropbox server
@@ -149,6 +157,7 @@ type Fs struct {
 	srv            files.Client   // the connection to the dropbox server
 	sharing        sharing.Client // as above, but for generating sharing links
 	users          users.Client   // as above, but for accessing user information
+	team           team.Client    // for the Teams API
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
 	pacer          *pacer.Pacer   // To pace the API calls
@@ -195,7 +204,16 @@ func shouldRetry(err error) (bool, error) {
 		return false, err
 	}
 	baseErrString := errors.Cause(err).Error()
-	// FIXME there is probably a better way of doing this!
+	// handle any official Retry-After header from Dropbox's SDK first
+	switch e := err.(type) {
+	case auth.RateLimitAPIError:
+		if e.RateLimitError.RetryAfter > 0 {
+			fs.Debugf(baseErrString, "Too many requests or write operations. Trying again in %d seconds.", e.RateLimitError.RetryAfter)
+			time.Sleep(time.Duration(e.RateLimitError.RetryAfter) * time.Second)
+		}
+		return true, err
+	}
+	// Keep old behavior for backward compatibility
 	if strings.Contains(baseErrString, "too_many_write_operations") || strings.Contains(baseErrString, "too_many_requests") {
 		return true, err
 	}
@@ -221,7 +239,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -262,6 +280,29 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		Client:          oAuthClient,    // maybe???
 		HeaderGenerator: f.headerGenerator,
 	}
+
+	// NOTE: needs to be created pre-impersonation so we can look up the impersonated user
+	f.team = team.New(config)
+
+	if opt.Impersonate != "" {
+
+		user := team.UserSelectorArg{
+			Email: opt.Impersonate,
+		}
+		user.Tag = "email"
+
+		members := []*team.UserSelectorArg{&user}
+		args := team.NewMembersGetInfoArgs(members)
+
+		memberIds, err := f.team.MembersGetInfo(args)
+
+		if err != nil {
+			return nil, errors.Wrapf(err, "invalid dropbox team member: %q", opt.Impersonate)
+		}
+
+		config.AsMemberID = memberIds[0].MemberInfo.Profile.MemberProfile.TeamMemberId
+	}
+
 	f.srv = files.New(config)
 	f.sharing = sharing.New(config)
 	f.users = users.New(config)

backend/ftp/ftp.go

@@ -166,7 +166,7 @@ func (f *Fs) putFtpConnection(pc **ftp.ServerConn, err error) {
 	f.poolMu.Unlock()
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (ff fs.Fs, err error) {
 	// defer fs.Trace(nil, "name=%q, root=%q", name, root)("fs=%v, err=%v", &ff, &err)
 	// Parse config into Options struct
@@ -646,7 +646,21 @@ func (f *ftpReadCloser) Read(p []byte) (n int, err error) {
 
 // Close the FTP reader and return the connection to the pool
 func (f *ftpReadCloser) Close() error {
-	err := f.rc.Close()
+	var err error
+	errchan := make(chan error, 1)
+	go func() {
+		errchan <- f.rc.Close()
+	}()
+	// Wait for Close for up to 60 seconds
+	timer := time.NewTimer(60 * time.Second)
+	select {
+	case err = <-errchan:
+		timer.Stop()
+	case <-timer.C:
+		// if timer fired assume no error but connection dead
+		fs.Errorf(f.f, "Timeout when waiting for connection Close")
+		return nil
+	}
 	// if errors while reading or closing, dump the connection
 	if err != nil || f.err != nil {
 		_ = f.c.Quit()

backend/googlecloudstorage/googlecloudstorage.go

@@ -1,4 +1,7 @@
 // Package googlecloudstorage provides an interface to Google Cloud Storage
+
+// +build go1.9
+
 package googlecloudstorage
 
 /*
@@ -159,21 +162,36 @@ func init() {
 			}, {
 				Value: "asia-east1",
 				Help:  "Taiwan.",
+			}, {
+				Value: "asia-east2",
+				Help:  "Hong Kong.",
 			}, {
 				Value: "asia-northeast1",
 				Help:  "Tokyo.",
+			}, {
+				Value: "asia-south1",
+				Help:  "Mumbai.",
 			}, {
 				Value: "asia-southeast1",
 				Help:  "Singapore.",
 			}, {
 				Value: "australia-southeast1",
 				Help:  "Sydney.",
+			}, {
+				Value: "europe-north1",
+				Help:  "Finland.",
 			}, {
 				Value: "europe-west1",
 				Help:  "Belgium.",
 			}, {
 				Value: "europe-west2",
 				Help:  "London.",
+			}, {
+				Value: "europe-west3",
+				Help:  "Frankfurt.",
+			}, {
+				Value: "europe-west4",
+				Help:  "Netherlands.",
 			}, {
 				Value: "us-central1",
 				Help:  "Iowa.",
@@ -186,6 +204,9 @@ func init() {
 			}, {
 				Value: "us-west1",
 				Help:  "Oregon.",
+			}, {
+				Value: "us-west2",
+				Help:  "California.",
 			}},
 		}, {
 			Name: "storage_class",
@@ -279,7 +300,7 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// shouldRetry determines whehter a given err rates being retried
+// shouldRetry determines whether a given err rates being retried
 func shouldRetry(err error) (again bool, errOut error) {
 	again = false
 	if err != nil {
@@ -327,7 +348,7 @@ func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
 	return oauth2.NewClient(ctxWithSpecialClient, conf.TokenSource(ctxWithSpecialClient)), nil
 }
 
-// NewFs contstructs an Fs from the path, bucket:path
+// NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	var oAuthClient *http.Client
 

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,4 +1,7 @@
 // Test GoogleCloudStorage filesystem interface
+
+// +build go1.9
+
 package googlecloudstorage_test
 
 import (

backend/googlecloudstorage/googlecloudstorage_unsupported.go

@@ -0,0 +1,6 @@
+// Build for unsupported platforms to stop go complaining
+// about "no buildable Go source files "
+
+// +build !go1.9
+
+package googlecloudstorage

backend/http/http.go

@@ -40,6 +40,9 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "https://example.com",
 				Help:  "Connect to example.com",
+			}, {
+				Value: "https://user:pass@example.com",
+				Help:  "Connect to example.com using a username and password",
 			}},
 		}},
 	}
@@ -193,7 +196,7 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	}
 	err := o.stat()
 	if err != nil {
-		return nil, errors.Wrap(err, "Stat failed")
+		return nil, err
 	}
 	return o, nil
 }
@@ -248,7 +251,7 @@ func parseName(base *url.URL, name string) (string, error) {
 	}
 	// calculate the name relative to the base
 	name = u.Path[len(base.Path):]
-	// musn't be empty
+	// mustn't be empty
 	if name == "" {
 		return "", errNameIsEmpty
 	}
@@ -416,6 +419,9 @@ func (o *Object) url() string {
 func (o *Object) stat() error {
 	url := o.url()
 	res, err := o.fs.httpClient.Head(url)
+	if err == nil && res.StatusCode == http.StatusNotFound {
+		return fs.ErrorObjectNotFound
+	}
 	err = statusError(res, err)
 	if err != nil {
 		return errors.Wrap(err, "failed to stat")

backend/http/http_internal_test.go

@@ -144,6 +144,11 @@ func TestNewObject(t *testing.T) {
 
 	dt, ok := fstest.CheckTimeEqualWithPrecision(tObj, tFile, time.Second)
 	assert.True(t, ok, fmt.Sprintf("%s: Modification time difference too big |%s| > %s (%s vs %s) (precision %s)", o.Remote(), dt, time.Second, tObj, tFile, time.Second))
+
+	// check object not found
+	o, err = f.NewObject("not found.txt")
+	assert.Nil(t, o)
+	assert.Equal(t, fs.ErrorObjectNotFound, err)
 }
 
 func TestOpen(t *testing.T) {

backend/hubic/hubic.go

@@ -9,8 +9,10 @@ package hubic
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/ncw/rclone/backend/swift"
@@ -124,7 +126,9 @@ func (f *Fs) getCredentials() (err error) {
 	}
 	defer fs.CheckClose(resp.Body, &err)
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		return errors.Errorf("failed to get credentials: %s", resp.Status)
+		body, _ := ioutil.ReadAll(resp.Body)
+		bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
+		return errors.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
 	}
 	decoder := json.NewDecoder(resp.Body)
 	var result credentials

backend/jottacloud/api/types.go

@@ -9,7 +9,10 @@ import (
 )
 
 const (
+	// default time format for almost all request and responses
 	timeFormat = "2006-01-02-T15:04:05Z0700"
+	// the API server seems to use a different format
+	apiTimeFormat = "2006-01-02T15:04:05Z07:00"
 )
 
 // Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
@@ -40,6 +43,9 @@ func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 // Return Time string in Jottacloud format
 func (t Time) String() string { return time.Time(t).Format(timeFormat) }
 
+// APIString returns Time string in Jottacloud API format
+func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
+
 // Flag is a hacky type for checking if an attribute is present
 type Flag bool
 
@@ -58,6 +64,15 @@ func (f *Flag) MarshalXMLAttr(name xml.Name) (xml.Attr, error) {
 	return attr, errors.New("unimplemented")
 }
 
+// TokenJSON is the struct representing the HTTP response from OAuth2
+// providers returning a token in JSON form.
+type TokenJSON struct {
+	AccessToken  string `json:"access_token"`
+	TokenType    string `json:"token_type"`
+	RefreshToken string `json:"refresh_token"`
+	ExpiresIn    int32  `json:"expires_in"` // at least PayPal returns string, while most return number
+}
+
 /*
 GET http://www.jottacloud.com/JFS/<account>
 
@@ -265,3 +280,37 @@ func (e *Error) Error() string {
 	}
 	return out
 }
+
+// AllocateFileRequest to prepare an upload to Jottacloud
+type AllocateFileRequest struct {
+	Bytes    int64  `json:"bytes"`
+	Created  string `json:"created"`
+	Md5      string `json:"md5"`
+	Modified string `json:"modified"`
+	Path     string `json:"path"`
+}
+
+// AllocateFileResponse for upload requests
+type AllocateFileResponse struct {
+	Name      string `json:"name"`
+	Path      string `json:"path"`
+	State     string `json:"state"`
+	UploadID  string `json:"upload_id"`
+	UploadURL string `json:"upload_url"`
+	Bytes     int64  `json:"bytes"`
+	ResumePos int64  `json:"resume_pos"`
+}
+
+// UploadResponse after an upload
+type UploadResponse struct {
+	Name      string      `json:"name"`
+	Path      string      `json:"path"`
+	Kind      string      `json:"kind"`
+	ContentID string      `json:"content_id"`
+	Bytes     int64       `json:"bytes"`
+	Md5       string      `json:"md5"`
+	Created   int64       `json:"created"`
+	Modified  int64       `json:"modified"`
+	Deleted   interface{} `json:"deleted"`
+	Mime      string      `json:"mime"`
+}

backend/jottacloud/jottacloud.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net/http"
 	"net/url"
 	"os"
@@ -26,22 +27,41 @@ import (
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
+	"github.com/ncw/rclone/lib/oauthutil"
 	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/rclone/lib/rest"
 	"github.com/pkg/errors"
+	"golang.org/x/oauth2"
 )
 
 // Globals
 const (
-	minSleep          = 10 * time.Millisecond
-	maxSleep          = 2 * time.Second
-	decayConstant     = 2 // bigger for slower decay, exponential
-	defaultDevice     = "Jotta"
-	defaultMountpoint = "Sync"
-	rootURL           = "https://www.jottacloud.com/jfs/"
-	apiURL            = "https://api.jottacloud.com"
-	shareURL          = "https://www.jottacloud.com/"
-	cachePrefix       = "rclone-jcmd5-"
+	minSleep                    = 10 * time.Millisecond
+	maxSleep                    = 2 * time.Second
+	decayConstant               = 2 // bigger for slower decay, exponential
+	defaultDevice               = "Jotta"
+	defaultMountpoint           = "Sync" // nolint
+	rootURL                     = "https://www.jottacloud.com/jfs/"
+	apiURL                      = "https://api.jottacloud.com/files/v1/"
+	baseURL                     = "https://www.jottacloud.com/"
+	tokenURL                    = "https://api.jottacloud.com/auth/v1/token"
+	cachePrefix                 = "rclone-jcmd5-"
+	rcloneClientID              = "nibfk8biu12ju7hpqomr8b1e40"
+	rcloneEncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
+	configUsername              = "user"
+)
+
+var (
+	// Description of how to auth for this app for a personal account
+	oauthConfig = &oauth2.Config{
+		Endpoint: oauth2.Endpoint{
+			AuthURL:  tokenURL,
+			TokenURL: tokenURL,
+		},
+		ClientID:     rcloneClientID,
+		ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
+		RedirectURL:  oauthutil.RedirectLocalhostURL,
+	}
 )
 
 // Register with Fs
@@ -50,13 +70,71 @@ func init() {
 		Name:        "jottacloud",
 		Description: "JottaCloud",
 		NewFs:       NewFs,
+		Config: func(name string, m configmap.Mapper) {
+			tokenString, ok := m.Get("token")
+			if ok && tokenString != "" {
+				fmt.Printf("Already have a token - refresh?\n")
+				if !config.Confirm() {
+					return
+				}
+			}
+
+			username, ok := m.Get(configUsername)
+			if !ok {
+				log.Fatalf("No username defined")
+			}
+			password := config.GetPassword("Your Jottacloud password is only required during config and will not be stored.")
+
+			// prepare out token request with username and password
+			srv := rest.NewClient(fshttp.NewClient(fs.Config))
+			values := url.Values{}
+			values.Set("grant_type", "PASSWORD")
+			values.Set("password", password)
+			values.Set("username", username)
+			values.Set("client_id", oauthConfig.ClientID)
+			values.Set("client_secret", oauthConfig.ClientSecret)
+			opts := rest.Opts{
+				Method:      "POST",
+				RootURL:     oauthConfig.Endpoint.AuthURL,
+				ContentType: "application/x-www-form-urlencoded",
+				Parameters:  values,
+			}
+
+			var jsonToken api.TokenJSON
+			resp, err := srv.CallJSON(&opts, nil, &jsonToken)
+			if err != nil {
+				// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
+				if resp != nil {
+					if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
+						fmt.Printf("This account has 2 factor authentication enabled you will receive a verification code via SMS.\n")
+						fmt.Printf("Enter verification code> ")
+						authCode := config.ReadLine()
+						authCode = strings.Replace(authCode, "-", "", -1) // the sms received contains a pair of 3 digit numbers seperated by '-' but wants a single 6 digit number
+						opts.ExtraHeaders = make(map[string]string)
+						opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
+						resp, err = srv.CallJSON(&opts, nil, &jsonToken)
+					}
+				}
+				if err != nil {
+					log.Fatalf("Failed to get resource token: %v", err)
+				}
+			}
+
+			var token oauth2.Token
+			token.AccessToken = jsonToken.AccessToken
+			token.RefreshToken = jsonToken.RefreshToken
+			token.TokenType = jsonToken.TokenType
+			token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
+
+			// finally save them in the config
+			err = oauthutil.PutToken(name, m, &token, true)
+			if err != nil {
+				log.Fatalf("Error while setting token: %s", err)
+			}
+		},
 		Options: []fs.Option{{
-			Name: "user",
-			Help: "User Name",
-		}, {
-			Name:       "pass",
-			Help:       "Password.",
-			IsPassword: true,
+			Name: configUsername,
+			Help: "User Name:",
 		}, {
 			Name:     "mountpoint",
 			Help:     "The mountpoint to use.",
@@ -83,6 +161,11 @@ func init() {
 			Help:     "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
 			Default:  false,
 			Advanced: true,
+		}, {
+			Name:     "upload_resume_limit",
+			Help:     "Files bigger than this can be resumed if the upload fail's.",
+			Default:  fs.SizeSuffix(10 * 1024 * 1024),
+			Advanced: true,
 		}},
 	})
 }
@@ -90,23 +173,25 @@ func init() {
 // Options defines the configuration for this backend
 type Options struct {
 	User               string        `config:"user"`
-	Pass               string        `config:"pass"`
 	Mountpoint         string        `config:"mountpoint"`
 	MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
 	HardDelete         bool          `config:"hard_delete"`
 	Unlink             bool          `config:"unlink"`
+	UploadThreshold    fs.SizeSuffix `config:"upload_resume_limit"`
 }
 
 // Fs represents a remote jottacloud
 type Fs struct {
-	name        string
-	root        string
-	user        string
-	opt         Options
-	features    *fs.Features
-	endpointURL string
-	srv         *rest.Client
-	pacer       *pacer.Pacer
+	name         string
+	root         string
+	user         string
+	opt          Options
+	features     *fs.Features
+	endpointURL  string
+	srv          *rest.Client
+	apiSrv       *rest.Client
+	pacer        *pacer.Pacer
+	tokenRenewer *oauthutil.Renew // renew the token on expiry
 }
 
 // Object describes a jottacloud object
@@ -261,6 +346,29 @@ func (o *Object) filePath() string {
 	return o.fs.filePath(o.remote)
 }
 
+// Jottacloud requires the grant_type 'refresh_token' string
+// to be uppercase and throws a 400 Bad Request if we use the
+// lower case used by the oauth2 module
+//
+// This filter catches all refresh requests, reads the body,
+// changes the case and then sends it on
+func grantTypeFilter(req *http.Request) {
+	if tokenURL == req.URL.String() {
+		// read the entire body
+		refreshBody, err := ioutil.ReadAll(req.Body)
+		if err != nil {
+			return
+		}
+		_ = req.Body.Close()
+
+		// make the refresh token upper case
+		refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
+
+		// set the new ReadCloser (with a dummy Close())
+		req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
+	}
+}
+
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -273,25 +381,29 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = parsePath(root)
 
-	user := config.FileGet(name, "user")
-	pass := config.FileGet(name, "pass")
-
-	if opt.Pass != "" {
-		var err error
-		opt.Pass, err = obscure.Reveal(opt.Pass)
-		if err != nil {
-			return nil, errors.Wrap(err, "couldn't decrypt password")
-		}
+	// the oauth client for the api servers needs
+	// a filter to fix the grant_type issues (see above)
+	baseClient := fshttp.NewClient(fs.Config)
+	if do, ok := baseClient.Transport.(interface {
+		SetRequestFilter(f func(req *http.Request))
+	}); ok {
+		do.SetRequestFilter(grantTypeFilter)
+	} else {
+		fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
+	}
+	oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
+	if err != nil {
+		return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
 	}
 
 	f := &Fs{
-		name: name,
-		root: root,
-		user: opt.User,
-		opt:  *opt,
-		//endpointURL: rest.URLPathEscape(path.Join(user, defaultDevice, opt.Mountpoint)),
-		srv:   rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(rootURL),
-		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		name:   name,
+		root:   root,
+		user:   opt.User,
+		opt:    *opt,
+		srv:    rest.NewClient(oAuthClient).SetRoot(rootURL),
+		apiSrv: rest.NewClient(oAuthClient).SetRoot(apiURL),
+		pacer:  pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
@@ -299,14 +411,14 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		ReadMimeType:            true,
 		WriteMimeType:           true,
 	}).Fill(f)
-
-	if user == "" || pass == "" {
-		return nil, errors.New("jottacloud needs user and password")
-	}
-
-	f.srv.SetUserPass(opt.User, opt.Pass)
 	f.srv.SetErrorHandler(errorHandler)
 
+	// Renew the token in the background
+	f.tokenRenewer = oauthutil.NewRenew(f.String(), ts, func() error {
+		_, err := f.readMetaDataForPath("")
+		return err
+	})
+
 	err = f.setEndpointURL(opt.Mountpoint)
 	if err != nil {
 		return nil, errors.Wrap(err, "couldn't get account info")
@@ -331,7 +443,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		// return an error with an fs which points to the parent
 		return f, fs.ErrorIsFile
 	}
-
 	return f, nil
 }
 
@@ -348,7 +459,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *api.JottaFile) (fs.Object, e
 		// Set info
 		err = o.setMetaData(info)
 	} else {
-		err = o.readMetaData() // reads info and meta, returning an error
+		err = o.readMetaData(false) // reads info and meta, returning an error
 	}
 	if err != nil {
 		return nil, err
@@ -396,7 +507,7 @@ func (f *Fs) CreateDir(path string) (jf *api.JottaFolder, err error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
-	//fmt.Printf("List: %s\n", dir)
+	//fmt.Printf("List: %s\n", f.filePath(dir))
 	opts := rest.Opts{
 		Method: "GET",
 		Path:   f.filePath(dir),
@@ -658,7 +769,7 @@ func (f *Fs) Purge() error {
 	return f.purgeCheck("", false)
 }
 
-// copyOrMoves copys or moves directories or files depending on the mthod parameter
+// copyOrMoves copies or moves directories or files depending on the method parameter
 func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err error) {
 	opts := rest.Opts{
 		Method:     "POST",
@@ -676,7 +787,6 @@ func (f *Fs) copyOrMove(method, src, dest string) (info *api.JottaFile, err erro
 	if err != nil {
 		return nil, err
 	}
-
 	return info, nil
 }
 
@@ -824,7 +934,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	if result.PublicSharePath == "" {
 		return "", errors.New("couldn't create public link - no link path received")
 	}
-	link = path.Join(shareURL, result.PublicSharePath)
+	link = path.Join(baseURL, result.PublicSharePath)
 	return link, nil
 }
 
@@ -880,7 +990,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	err := o.readMetaData()
+	err := o.readMetaData(false)
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return 0
@@ -896,21 +1006,24 @@ func (o *Object) MimeType() string {
 // setMetaData sets the metadata from info
 func (o *Object) setMetaData(info *api.JottaFile) (err error) {
 	o.hasMetaData = true
-	o.size = int64(info.Size)
+	o.size = info.Size
 	o.md5 = info.MD5
 	o.mimeType = info.MimeType
 	o.modTime = time.Time(info.ModifiedAt)
 	return nil
 }
 
-func (o *Object) readMetaData() (err error) {
-	if o.hasMetaData {
+func (o *Object) readMetaData(force bool) (err error) {
+	if o.hasMetaData && !force {
 		return nil
 	}
 	info, err := o.fs.readMetaDataForPath(o.remote)
 	if err != nil {
 		return err
 	}
+	if info.Deleted {
+		return fs.ErrorObjectNotFound
+	}
 	return o.setMetaData(info)
 }
 
@@ -919,7 +1032,7 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
-	err := o.readMetaData()
+	err := o.readMetaData(false)
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
 		return time.Now()
@@ -967,7 +1080,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 func readMD5(in io.Reader, size, threshold int64) (md5sum string, out io.Reader, cleanup func(), err error) {
 	// we need a MD5
 	md5Hasher := md5.New()
-	// use the teeReader to write to the local file AND caclulate the MD5 while doing so
+	// use the teeReader to write to the local file AND calculate the MD5 while doing so
 	teeReader := io.TeeReader(in, md5Hasher)
 
 	// nothing to clean up by default
@@ -1040,43 +1153,74 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		in = wrap(in)
 	}
 
+	// use the api to allocate the file first and get resume / deduplication info
 	var resp *http.Response
-	var result api.JottaFile
 	opts := rest.Opts{
-		Method:        "POST",
-		Path:          o.filePath(),
-		Body:          in,
-		ContentType:   fs.MimeType(src),
-		ContentLength: &size,
-		ExtraHeaders:  make(map[string]string),
-		Parameters:    url.Values{},
+		Method:       "POST",
+		Path:         "allocate",
+		ExtraHeaders: make(map[string]string),
+	}
+	fileDate := api.Time(src.ModTime()).APIString()
+
+	// the allocate request
+	var request = api.AllocateFileRequest{
+		Bytes:    size,
+		Created:  fileDate,
+		Modified: fileDate,
+		Md5:      md5String,
+		Path:     path.Join(o.fs.opt.Mountpoint, replaceReservedChars(path.Join(o.fs.root, o.remote))),
 	}
 
-	opts.ExtraHeaders["JMd5"] = md5String
-	opts.Parameters.Set("cphash", md5String)
-	opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
-	// opts.ExtraHeaders["JCreated"] = api.Time(src.ModTime()).String()
-	opts.ExtraHeaders["JModified"] = api.Time(src.ModTime()).String()
-
-	// Parameters observed in other implementations
-	//opts.ExtraHeaders["X-Jfs-DeviceName"] = "Jotta"
-	//opts.ExtraHeaders["X-Jfs-Devicename-Base64"] = ""
-	//opts.ExtraHeaders["X-Jftp-Version"] = "2.4" this appears to be the current version
-	//opts.ExtraHeaders["jx_csid"] = ""
-	//opts.ExtraHeaders["jx_lisence"] = ""
-
-	opts.Parameters.Set("umode", "nomultipart")
-
+	// send it
+	var response api.AllocateFileResponse
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		resp, err = o.fs.srv.CallXML(&opts, nil, &result)
+		resp, err = o.fs.apiSrv.CallJSON(&opts, &request, &response)
 		return shouldRetry(resp, err)
 	})
 	if err != nil {
 		return err
 	}
 
-	// TODO: Check returned Metadata? Timeout on big uploads?
-	return o.setMetaData(&result)
+	// If the file state is INCOMPLETE and CORRPUT, try to upload a then
+	if response.State != "COMPLETED" {
+		// how much do we still have to upload?
+		remainingBytes := size - response.ResumePos
+		opts = rest.Opts{
+			Method:        "POST",
+			RootURL:       response.UploadURL,
+			ContentLength: &remainingBytes,
+			ContentType:   "application/octet-stream",
+			Body:          in,
+			ExtraHeaders:  make(map[string]string),
+		}
+		if response.ResumePos != 0 {
+			opts.ExtraHeaders["Range"] = "bytes=" + strconv.FormatInt(response.ResumePos, 10) + "-" + strconv.FormatInt(size-1, 10)
+		}
+
+		// copy the already uploaded bytes into the trash :)
+		var result api.UploadResponse
+		_, err = io.CopyN(ioutil.Discard, in, response.ResumePos)
+		if err != nil {
+			return err
+		}
+
+		// send the remaining bytes
+		resp, err = o.fs.apiSrv.CallJSON(&opts, nil, &result)
+		if err != nil {
+			return err
+		}
+
+		// finally update the meta data
+		o.hasMetaData = true
+		o.size = result.Bytes
+		o.md5 = result.Md5
+		o.modTime = time.Unix(result.Modified/1000, 0)
+	} else {
+		// If the file state is COMPLETE we don't need to upload it because the file was allready found but we still ned to update our metadata
+		return o.readMetaData(true)
+	}
+
+	return nil
 }
 
 // Remove an object

backend/jottacloud/replace.go

@@ -2,7 +2,7 @@
 Translate file names for JottaCloud adapted from OneDrive
 
 
-The following characters are JottaClous reserved characters, and can't
+The following characters are JottaCloud reserved characters, and can't
 be used in JottaCloud folder and file names.
 
   jottacloud  = "/" / "\" / "*" / "<" / ">" / "?" / "!" / "&" / ":" / ";" / "|" / "#" / "%" / """ / "'" / "." / "~"

backend/local/about_unix.go

@@ -16,7 +16,7 @@ func (f *Fs) About() (*fs.Usage, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to read disk usage")
 	}
-	bs := int64(s.Bsize)
+	bs := int64(s.Bsize) // nolint: unconvert
 	usage := &fs.Usage{
 		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
 		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use

backend/local/lchtimes.go

@@ -0,0 +1,20 @@
+// +build windows plan9
+
+package local
+
+import (
+	"time"
+)
+
+const haveLChtimes = false
+
+// lChtimes changes the access and modification times of the named
+// link, similar to the Unix utime() or utimes() functions.
+//
+// The underlying filesystem may truncate or round the values to a
+// less precise time unit.
+// If there is an error, it will be of type *PathError.
+func lChtimes(name string, atime time.Time, mtime time.Time) error {
+	// Does nothing
+	return nil
+}

backend/local/lchtimes_unix.go

@@ -0,0 +1,28 @@
+// +build !windows,!plan9
+
+package local
+
+import (
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+const haveLChtimes = true
+
+// lChtimes changes the access and modification times of the named
+// link, similar to the Unix utime() or utimes() functions.
+//
+// The underlying filesystem may truncate or round the values to a
+// less precise time unit.
+// If there is an error, it will be of type *PathError.
+func lChtimes(name string, atime time.Time, mtime time.Time) error {
+	var utimes [2]unix.Timespec
+	utimes[0] = unix.NsecToTimespec(atime.UnixNano())
+	utimes[1] = unix.NsecToTimespec(mtime.UnixNano())
+	if e := unix.UtimesNanoAt(unix.AT_FDCWD, name, utimes[0:], unix.AT_SYMLINK_NOFOLLOW); e != nil {
+		return &os.PathError{Op: "lchtimes", Path: name, Err: e}
+	}
+	return nil
+}

backend/local/local.go

@@ -2,6 +2,7 @@
 package local
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -21,12 +22,14 @@ import (
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/hash"
+	"github.com/ncw/rclone/lib/file"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 )
 
 // Constants
 const devUnset = 0xdeadbeefcafebabe // a device id meaning it is unset
+const linkSuffix = ".rclonelink"    // The suffix added to a translated symbolic link
 
 // Register with Fs
 func init() {
@@ -48,6 +51,13 @@ func init() {
 			NoPrefix: true,
 			ShortOpt: "L",
 			Advanced: true,
+		}, {
+			Name:     "links",
+			Help:     "Translate symlinks to/from regular files with a '" + linkSuffix + "' extension",
+			Default:  false,
+			NoPrefix: true,
+			ShortOpt: "l",
+			Advanced: true,
 		}, {
 			Name: "skip_links",
 			Help: `Don't warn about skipped symlinks.
@@ -92,12 +102,13 @@ check can be disabled with this flag.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	FollowSymlinks bool `config:"copy_links"`
-	SkipSymlinks   bool `config:"skip_links"`
-	NoUTFNorm      bool `config:"no_unicode_normalization"`
-	NoCheckUpdated bool `config:"no_check_updated"`
-	NoUNC          bool `config:"nounc"`
-	OneFileSystem  bool `config:"one_file_system"`
+	FollowSymlinks    bool `config:"copy_links"`
+	TranslateSymlinks bool `config:"links"`
+	SkipSymlinks      bool `config:"skip_links"`
+	NoUTFNorm         bool `config:"no_unicode_normalization"`
+	NoCheckUpdated    bool `config:"no_check_updated"`
+	NoUNC             bool `config:"nounc"`
+	OneFileSystem     bool `config:"one_file_system"`
 }
 
 // Fs represents a local filesystem rooted at root
@@ -119,17 +130,20 @@ type Fs struct {
 
 // Object represents a local filesystem object
 type Object struct {
-	fs      *Fs    // The Fs this object is part of
-	remote  string // The remote path - properly UTF-8 encoded - for rclone
-	path    string // The local path - may not be properly UTF-8 encoded - for OS
-	size    int64  // file metadata - always present
-	mode    os.FileMode
-	modTime time.Time
-	hashes  map[hash.Type]string // Hashes
+	fs             *Fs    // The Fs this object is part of
+	remote         string // The remote path - properly UTF-8 encoded - for rclone
+	path           string // The local path - may not be properly UTF-8 encoded - for OS
+	size           int64  // file metadata - always present
+	mode           os.FileMode
+	modTime        time.Time
+	hashes         map[hash.Type]string // Hashes
+	translatedLink bool                 // Is this object a translated link
 }
 
 // ------------------------------------------------------------
 
+var errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
+
 // NewFs constructs an Fs from the path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -138,6 +152,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	if opt.TranslateSymlinks && opt.FollowSymlinks {
+		return nil, errLinksAndCopyLinks
+	}
 
 	if opt.NoUTFNorm {
 		fs.Errorf(nil, "The --local-no-unicode-normalization flag is deprecated and will be removed")
@@ -165,7 +182,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err == nil {
 		f.dev = readDevice(fi, f.opt.OneFileSystem)
 	}
-	if err == nil && fi.Mode().IsRegular() {
+	if err == nil && f.isRegular(fi.Mode()) {
 		// It is a file, so use the parent as the root
 		f.root = filepath.Dir(f.root)
 		// return an error with an fs which points to the parent
@@ -174,6 +191,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	return f, nil
 }
 
+// Determine whether a file is a 'regular' file,
+// Symlinks are regular files, only if the TranslateSymlink
+// option is in-effect
+func (f *Fs) isRegular(mode os.FileMode) bool {
+	if !f.opt.TranslateSymlinks {
+		return mode.IsRegular()
+	}
+
+	// fi.Mode().IsRegular() tests that all mode bits are zero
+	// Since symlinks are accepted, test that all other bits are zero,
+	// except the symlink bit
+	return mode&os.ModeType&^os.ModeSymlink == 0
+}
+
 // Name of the remote (as passed into NewFs)
 func (f *Fs) Name() string {
 	return f.name
@@ -194,28 +225,48 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// caseInsenstive returns whether the remote is case insensitive or not
+// caseInsensitive returns whether the remote is case insensitive or not
 func (f *Fs) caseInsensitive() bool {
 	// FIXME not entirely accurate since you can have case
-	// sensitive Fses on darwin and case insenstive Fses on linux.
+	// sensitive Fses on darwin and case insensitive Fses on linux.
 	// Should probably check but that would involve creating a
 	// file in the remote to be most accurate which probably isn't
 	// desirable.
 	return runtime.GOOS == "windows" || runtime.GOOS == "darwin"
 }
 
+// translateLink checks whether the remote is a translated link
+// and returns a new path, removing the suffix as needed,
+// It also returns whether this is a translated link at all
+//
+// for regular files, dstPath is returned unchanged
+func translateLink(remote, dstPath string) (newDstPath string, isTranslatedLink bool) {
+	isTranslatedLink = strings.HasSuffix(remote, linkSuffix)
+	newDstPath = strings.TrimSuffix(dstPath, linkSuffix)
+	return newDstPath, isTranslatedLink
+}
+
 // newObject makes a half completed Object
 //
 // if dstPath is empty then it is made from remote
 func (f *Fs) newObject(remote, dstPath string) *Object {
+	translatedLink := false
+
 	if dstPath == "" {
 		dstPath = f.cleanPath(filepath.Join(f.root, remote))
 	}
 	remote = f.cleanRemote(remote)
+
+	if f.opt.TranslateSymlinks {
+		// Possibly receive a new name for dstPath
+		dstPath, translatedLink = translateLink(remote, dstPath)
+	}
+
 	return &Object{
-		fs:     f,
-		remote: remote,
-		path:   dstPath,
+		fs:             f,
+		remote:         remote,
+		path:           dstPath,
+		translatedLink: translatedLink,
 	}
 }
 
@@ -237,6 +288,11 @@ func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Obj
 			}
 			return nil, err
 		}
+		// Handle the odd case, that a symlink was specified by name without the link suffix
+		if o.fs.opt.TranslateSymlinks && o.mode&os.ModeSymlink != 0 && !o.translatedLink {
+			return nil, fs.ErrorObjectNotFound
+		}
+
 	}
 	if o.mode.IsDir() {
 		return nil, errors.Wrapf(fs.ErrorNotAFile, "%q", remote)
@@ -260,6 +316,7 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // This should return ErrDirNotFound if the directory isn't
 // found.
 func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
+
 	dir = f.dirNames.Load(dir)
 	fsDirPath := f.cleanPath(filepath.Join(f.root, dir))
 	remote := f.cleanRemote(dir)
@@ -316,6 +373,10 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 					entries = append(entries, d)
 				}
 			} else {
+				// Check whether this link should be translated
+				if f.opt.TranslateSymlinks && fi.Mode()&os.ModeSymlink != 0 {
+					newRemote += linkSuffix
+				}
 				fso, err := f.newObjectWithInfo(newRemote, newPath, fi)
 				if err != nil {
 					return nil, err
@@ -529,7 +590,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		// OK
 	} else if err != nil {
 		return nil, err
-	} else if !dstObj.mode.IsRegular() {
+	} else if !dstObj.fs.isRegular(dstObj.mode) {
 		// It isn't a file
 		return nil, errors.New("can't move file onto non-file")
 	}
@@ -651,7 +712,13 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		in, err := os.Open(o.path)
+		var in io.ReadCloser
+
+		if !o.translatedLink {
+			in, err = file.Open(o.path)
+		} else {
+			in, err = o.openTranslatedLink(0, -1)
+		}
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
 		}
@@ -682,7 +749,12 @@ func (o *Object) ModTime() time.Time {
 
 // SetModTime sets the modification time of the local fs object
 func (o *Object) SetModTime(modTime time.Time) error {
-	err := os.Chtimes(o.path, modTime, modTime)
+	var err error
+	if o.translatedLink {
+		err = lChtimes(o.path, modTime, modTime)
+	} else {
+		err = os.Chtimes(o.path, modTime, modTime)
+	}
 	if err != nil {
 		return err
 	}
@@ -700,7 +772,7 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	if mode&os.ModeSymlink != 0 {
+	if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
 		if !o.fs.opt.SkipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
 		}
@@ -761,6 +833,16 @@ func (file *localOpenFile) Close() (err error) {
 	return err
 }
 
+// Returns a ReadCloser() object that contains the contents of a symbolic link
+func (o *Object) openTranslatedLink(offset, limit int64) (lrc io.ReadCloser, err error) {
+	// Read the link and return the destination  it as the contents of the object
+	linkdst, err := os.Readlink(o.path)
+	if err != nil {
+		return nil, err
+	}
+	return readers.NewLimitedReadCloser(ioutil.NopCloser(strings.NewReader(linkdst[offset:])), limit), nil
+}
+
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	var offset, limit int64 = 0, -1
@@ -780,7 +862,12 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		}
 	}
 
-	fd, err := os.Open(o.path)
+	// Handle a translated link
+	if o.translatedLink {
+		return o.openTranslatedLink(offset, limit)
+	}
+
+	fd, err := file.Open(o.path)
 	if err != nil {
 		return
 	}
@@ -811,8 +898,19 @@ func (o *Object) mkdirAll() error {
 	return os.MkdirAll(dir, 0777)
 }
 
+type nopWriterCloser struct {
+	*bytes.Buffer
+}
+
+func (nwc nopWriterCloser) Close() error {
+	// noop
+	return nil
+}
+
 // Update the object from in with modTime and size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
+	var out io.WriteCloser
+
 	hashes := hash.Supported
 	for _, option := range options {
 		switch x := option.(type) {
@@ -826,15 +924,23 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return err
 	}
 
-	out, err := os.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
-	if err != nil {
-		return err
-	}
-
-	// Pre-allocate the file for performance reasons
-	err = preAllocate(src.Size(), out)
-	if err != nil {
-		fs.Debugf(o, "Failed to pre-allocate: %v", err)
+	var symlinkData bytes.Buffer
+	// If the object is a regular file, create it.
+	// If it is a translated link, just read in the contents, and
+	// then create a symlink
+	if !o.translatedLink {
+		f, err := file.OpenFile(o.path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+		if err != nil {
+			return err
+		}
+		// Pre-allocate the file for performance reasons
+		err = preAllocate(src.Size(), f)
+		if err != nil {
+			fs.Debugf(o, "Failed to pre-allocate: %v", err)
+		}
+		out = f
+	} else {
+		out = nopWriterCloser{&symlinkData}
 	}
 
 	// Calculate the hash of the object we are reading as we go along
@@ -849,6 +955,26 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if err == nil {
 		err = closeErr
 	}
+
+	if o.translatedLink {
+		if err == nil {
+			// Remove any current symlink or file, if one exists
+			if _, err := os.Lstat(o.path); err == nil {
+				if removeErr := os.Remove(o.path); removeErr != nil {
+					fs.Errorf(o, "Failed to remove previous file: %v", removeErr)
+					return removeErr
+				}
+			}
+			// Use the contents for the copied object to create a symlink
+			err = os.Symlink(symlinkData.String(), o.path)
+		}
+
+		// only continue if symlink creation succeeded
+		if err != nil {
+			return err
+		}
+	}
+
 	if err != nil {
 		fs.Logf(o, "Removing partially written file on error: %v", err)
 		if removeErr := os.Remove(o.path); removeErr != nil {

backend/local/local_internal_test.go

@@ -1,13 +1,19 @@
 package local
 
 import (
+	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fstest"
+	"github.com/ncw/rclone/lib/file"
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -38,10 +44,13 @@ func TestUpdatingCheck(t *testing.T) {
 	filePath := "sub dir/local test"
 	r.WriteFile(filePath, "content", time.Now())
 
-	fd, err := os.Open(path.Join(r.LocalName, filePath))
+	fd, err := file.Open(path.Join(r.LocalName, filePath))
 	if err != nil {
 		t.Fatalf("failed opening file %q: %v", filePath, err)
 	}
+	defer func() {
+		require.NoError(t, fd.Close())
+	}()
 
 	fi, err := fd.Stat()
 	require.NoError(t, err)
@@ -72,3 +81,108 @@ func TestUpdatingCheck(t *testing.T) {
 	require.NoError(t, err)
 
 }
+
+func TestSymlink(t *testing.T) {
+	r := fstest.NewRun(t)
+	defer r.Finalise()
+	f := r.Flocal.(*Fs)
+	dir := f.root
+
+	// Write a file
+	modTime1 := fstest.Time("2001-02-03T04:05:10.123123123Z")
+	file1 := r.WriteFile("file.txt", "hello", modTime1)
+
+	// Write a symlink
+	modTime2 := fstest.Time("2002-02-03T04:05:10.123123123Z")
+	symlinkPath := filepath.Join(dir, "symlink.txt")
+	require.NoError(t, os.Symlink("file.txt", symlinkPath))
+	require.NoError(t, lChtimes(symlinkPath, modTime2, modTime2))
+
+	// Object viewed as symlink
+	file2 := fstest.NewItem("symlink.txt"+linkSuffix, "file.txt", modTime2)
+	if runtime.GOOS == "windows" {
+		file2.Size = 0 // symlinks are 0 length under Windows
+	}
+
+	// Object viewed as destination
+	file2d := fstest.NewItem("symlink.txt", "hello", modTime1)
+
+	// Check with no symlink flags
+	fstest.CheckItems(t, r.Flocal, file1)
+	fstest.CheckItems(t, r.Fremote)
+
+	// Set fs into "-L" mode
+	f.opt.FollowSymlinks = true
+	f.opt.TranslateSymlinks = false
+	f.lstat = os.Stat
+
+	fstest.CheckItems(t, r.Flocal, file1, file2d)
+	fstest.CheckItems(t, r.Fremote)
+
+	// Set fs into "-l" mode
+	f.opt.FollowSymlinks = false
+	f.opt.TranslateSymlinks = true
+	f.lstat = os.Lstat
+
+	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
+	if haveLChtimes {
+		fstest.CheckItems(t, r.Flocal, file1, file2)
+	}
+
+	// Create a symlink
+	modTime3 := fstest.Time("2002-03-03T04:05:10.123123123Z")
+	file3 := r.WriteObjectTo(r.Flocal, "symlink2.txt"+linkSuffix, "file.txt", modTime3, false)
+	if runtime.GOOS == "windows" {
+		file3.Size = 0 // symlinks are 0 length under Windows
+	}
+	fstest.CheckListingWithPrecision(t, r.Flocal, []fstest.Item{file1, file2, file3}, nil, fs.ModTimeNotSupported)
+	if haveLChtimes {
+		fstest.CheckItems(t, r.Flocal, file1, file2, file3)
+	}
+
+	// Check it got the correct contents
+	symlinkPath = filepath.Join(dir, "symlink2.txt")
+	fi, err := os.Lstat(symlinkPath)
+	require.NoError(t, err)
+	assert.False(t, fi.Mode().IsRegular())
+	linkText, err := os.Readlink(symlinkPath)
+	require.NoError(t, err)
+	assert.Equal(t, "file.txt", linkText)
+
+	// Check that NewObject gets the correct object
+	o, err := r.Flocal.NewObject("symlink2.txt" + linkSuffix)
+	require.NoError(t, err)
+	assert.Equal(t, "symlink2.txt"+linkSuffix, o.Remote())
+	if runtime.GOOS != "windows" {
+		assert.Equal(t, int64(8), o.Size())
+	}
+
+	// Check that NewObject doesn't see the non suffixed version
+	_, err = r.Flocal.NewObject("symlink2.txt")
+	require.Equal(t, fs.ErrorObjectNotFound, err)
+
+	// Check reading the object
+	in, err := o.Open()
+	require.NoError(t, err)
+	contents, err := ioutil.ReadAll(in)
+	require.NoError(t, err)
+	require.Equal(t, "file.txt", string(contents))
+	require.NoError(t, in.Close())
+
+	// Check reading the object with range
+	in, err = o.Open(&fs.RangeOption{Start: 2, End: 5})
+	require.NoError(t, err)
+	contents, err = ioutil.ReadAll(in)
+	require.NoError(t, err)
+	require.Equal(t, "file.txt"[2:5+1], string(contents))
+	require.NoError(t, in.Close())
+}
+
+func TestSymlinkError(t *testing.T) {
+	m := configmap.Simple{
+		"links":      "true",
+		"copy_links": "true",
+	}
+	_, err := NewFs("local", "/", m)
+	assert.Equal(t, errLinksAndCopyLinks, err)
+}

backend/local/read_device_unix.go

@@ -22,5 +22,5 @@ func readDevice(fi os.FileInfo, oneFileSystem bool) uint64 {
 		fs.Debugf(fi.Name(), "Type assertion fi.Sys().(*syscall.Stat_t) failed from: %#v", fi.Sys())
 		return devUnset
 	}
-	return uint64(statT.Dev)
+	return uint64(statT.Dev) // nolint: unconvert
 }

backend/mega/mega.go

@@ -497,7 +497,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 // Creates from the parameters passed in a half finished Object which
 // must have setMetaData called on it
 //
-// Returns the dirNode, obect, leaf and error
+// Returns the dirNode, object, leaf and error
 //
 // Used to create new objects
 func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Object, dirNode *mega.Node, leaf string, err error) {
@@ -523,10 +523,10 @@ func (f *Fs) createObject(remote string, modTime time.Time, size int64) (o *Obje
 // This will create a duplicate if we upload a new file without
 // checking to see if there is one already - use Put() for that.
 func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
-	exisitingObj, err := f.newObjectWithInfo(src.Remote(), nil)
+	existingObj, err := f.newObjectWithInfo(src.Remote(), nil)
 	switch err {
 	case nil:
-		return exisitingObj, exisitingObj.Update(in, src, options...)
+		return existingObj, existingObj.Update(in, src, options...)
 	case fs.ErrorObjectNotFound:
 		// Not found so create it
 		return f.PutUnchecked(in, src)
@@ -847,14 +847,14 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 				return shouldRetry(err)
 			})
 			if err != nil {
-				return errors.Wrapf(err, "MergDirs move failed on %q in %v", info.GetName(), srcDir)
+				return errors.Wrapf(err, "MergeDirs move failed on %q in %v", info.GetName(), srcDir)
 			}
 		}
 		// rmdir (into trash) the now empty source directory
 		fs.Infof(srcDir, "removing empty directory")
 		err = f.deleteNode(srcDirNode)
 		if err != nil {
-			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
+			return errors.Wrapf(err, "MergeDirs move failed to rmdir %q", srcDir)
 		}
 	}
 	return nil
@@ -1076,6 +1076,9 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // The new object may have been created if an error is returned
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (err error) {
 	size := src.Size()
+	if size < 0 {
+		return errors.New("mega backend can't upload a file of unknown length")
+	}
 	//modTime := src.ModTime()
 	remote := o.Remote()
 
@@ -1126,7 +1129,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		return errors.Wrap(err, "failed to finish upload")
 	}
 
-	// If the upload succeded and the original object existed, then delete it
+	// If the upload succeeded and the original object existed, then delete it
 	if o.info != nil {
 		err = o.fs.deleteNode(o.info)
 		if err != nil {

backend/onedrive/api/types.go

@@ -25,7 +25,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	out := e.ErrorInfo.Code
 	if e.ErrorInfo.InnerError.Code != "" {
@@ -35,7 +35,7 @@ func (e *Error) Error() string {
 	return out
 }
 
-// Check Error statisfies the error interface
+// Check Error satisfies the error interface
 var _ error = (*Error)(nil)
 
 // Identity represents an identity of an actor. For example, and actor
@@ -285,6 +285,7 @@ type AsyncOperationStatus struct {
 
 // GetID returns a normalized ID of the item
 // If DriveID is known it will be prefixed to the ID with # seperator
+// Can be parsed using onedrive.parseNormalizedID(normalizedID)
 func (i *Item) GetID() string {
 	if i.IsRemote() && i.RemoteItem.ID != "" {
 		return i.RemoteItem.ParentReference.DriveID + "#" + i.RemoteItem.ID
@@ -294,9 +295,9 @@ func (i *Item) GetID() string {
 	return i.ID
 }
 
-// GetDriveID returns a normalized ParentReferance of the item
+// GetDriveID returns a normalized ParentReference of the item
 func (i *Item) GetDriveID() string {
-	return i.GetParentReferance().DriveID
+	return i.GetParentReference().DriveID
 }
 
 // GetName returns a normalized Name of the item
@@ -397,8 +398,8 @@ func (i *Item) GetLastModifiedDateTime() Timestamp {
 	return i.LastModifiedDateTime
 }
 
-// GetParentReferance returns a normalized ParentReferance of the item
-func (i *Item) GetParentReferance() *ItemReference {
+// GetParentReference returns a normalized ParentReference of the item
+func (i *Item) GetParentReference() *ItemReference {
 	if i.IsRemote() && i.ParentReference == nil {
 		return i.RemoteItem.ParentReference
 	}

backend/onedrive/onedrive.go

@@ -75,9 +75,8 @@ func init() {
 				return
 			}
 
-			// Are we running headless?
-			if automatic, _ := m.Get(config.ConfigAutomatic); automatic != "" {
-				// Yes, okay we are done
+			// Stop if we are running non-interactive config
+			if fs.Config.AutoConfirm {
 				return
 			}
 
@@ -199,7 +198,7 @@ func init() {
 
 			fmt.Printf("Found drive '%s' of type '%s', URL: %s\nIs that okay?\n", rootItem.Name, rootItem.ParentReference.DriveType, rootItem.WebURL)
 			// This does not work, YET :)
-			if !config.Confirm() {
+			if !config.ConfirmWithConfig(m, "config_drive_ok", true) {
 				log.Fatalf("Cancelled by user")
 			}
 
@@ -228,7 +227,7 @@ that the chunks will be buffered into memory.`,
 			Advanced: true,
 		}, {
 			Name:     "drive_type",
-			Help:     "The type of the drive ( personal | business | documentLibrary )",
+			Help:     "The type of the drive ( " + driveTypePersonal + " | " + driveTypeBusiness + " | " + driveTypeSharepoint + " )",
 			Default:  "",
 			Advanced: true,
 		}, {
@@ -325,29 +324,19 @@ var retryErrorCodes = []int{
 // shouldRetry returns a boolean as to whether this resp and err
 // deserve to be retried.  It returns the err as a convenience
 func shouldRetry(resp *http.Response, err error) (bool, error) {
-	authRety := false
+	authRetry := false
 
 	if resp != nil && resp.StatusCode == 401 && len(resp.Header["Www-Authenticate"]) == 1 && strings.Index(resp.Header["Www-Authenticate"][0], "expired_token") >= 0 {
-		authRety = true
+		authRetry = true
 		fs.Debugf(nil, "Should retry: %v", err)
 	}
-	return authRety || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+	return authRetry || fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
 }
 
-// readMetaDataForPath reads the metadata from the path
-func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Response, err error) {
-	var opts rest.Opts
-	if len(path) == 0 {
-		opts = rest.Opts{
-			Method: "GET",
-			Path:   "/root",
-		}
-	} else {
-		opts = rest.Opts{
-			Method: "GET",
-			Path:   "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
-		}
-	}
+// readMetaDataForPathRelativeToID reads the metadata for a path relative to an item that is addressed by its normalized ID.
+// if `relPath` == "", it reads the metadata for the item with that ID.
+func (f *Fs) readMetaDataForPathRelativeToID(normalizedID string, relPath string) (info *api.Item, resp *http.Response, err error) {
+	opts := newOptsCall(normalizedID, "GET", ":/"+rest.URLPathEscape(replaceReservedChars(relPath)))
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.srv.CallJSON(&opts, nil, &info)
 		return shouldRetry(resp, err)
@@ -356,6 +345,72 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Respon
 	return info, resp, err
 }
 
+// readMetaDataForPath reads the metadata from the path (relative to the absolute root)
+func (f *Fs) readMetaDataForPath(path string) (info *api.Item, resp *http.Response, err error) {
+	firstSlashIndex := strings.IndexRune(path, '/')
+
+	if f.driveType != driveTypePersonal || firstSlashIndex == -1 {
+		var opts rest.Opts
+		if len(path) == 0 {
+			opts = rest.Opts{
+				Method: "GET",
+				Path:   "/root",
+			}
+		} else {
+			opts = rest.Opts{
+				Method: "GET",
+				Path:   "/root:/" + rest.URLPathEscape(replaceReservedChars(path)),
+			}
+		}
+		err = f.pacer.Call(func() (bool, error) {
+			resp, err = f.srv.CallJSON(&opts, nil, &info)
+			return shouldRetry(resp, err)
+		})
+		return info, resp, err
+	}
+
+	// The following branch handles the case when we're using OneDrive Personal and the path is in a folder.
+	// For OneDrive Personal, we need to consider the "shared with me" folders.
+	// An item in such a folder can only be addressed by its ID relative to the sharer's driveID or
+	// by its path relative to the folder's ID relative to the sharer's driveID.
+	// Note: A "shared with me" folder can only be placed in the sharee's absolute root.
+	// So we read metadata relative to a suitable folder's normalized ID.
+	var dirCacheFoundRoot bool
+	var rootNormalizedID string
+	if f.dirCache != nil {
+		var ok bool
+		if rootNormalizedID, ok = f.dirCache.Get(""); ok {
+			dirCacheFoundRoot = true
+		}
+	}
+
+	relPath, insideRoot := getRelativePathInsideBase(f.root, path)
+	var firstDir, baseNormalizedID string
+	if !insideRoot || !dirCacheFoundRoot {
+		// We do not have the normalized ID in dirCache for our query to base on. Query it manually.
+		firstDir, relPath = path[:firstSlashIndex], path[firstSlashIndex+1:]
+		info, resp, err := f.readMetaDataForPath(firstDir)
+		if err != nil {
+			return info, resp, err
+		}
+		baseNormalizedID = info.GetID()
+	} else {
+		if f.root != "" {
+			// Read metadata based on root
+			baseNormalizedID = rootNormalizedID
+		} else {
+			// Read metadata based on firstDir
+			firstDir, relPath = path[:firstSlashIndex], path[firstSlashIndex+1:]
+			baseNormalizedID, err = f.dirCache.FindDir(firstDir, false)
+			if err != nil {
+				return nil, nil, err
+			}
+		}
+	}
+
+	return f.readMetaDataForPathRelativeToID(baseNormalizedID, relPath)
+}
+
 // errorHandler parses a non 2xx error response into an error
 func errorHandler(resp *http.Response) error {
 	// Decode error response
@@ -404,13 +459,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}
 
 	if opt.DriveID == "" || opt.DriveType == "" {
-		log.Fatalf("Unable to get drive_id and drive_type. If you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend.")
+		return nil, errors.New("unable to get drive_id and drive_type - if you are upgrading from older versions of rclone, please run `rclone config` and re-configure this backend")
 	}
 
 	root = parsePath(root)
 	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure OneDrive: %v", err)
+		return nil, errors.Wrap(err, "failed to configure OneDrive")
 	}
 
 	f := &Fs{
@@ -437,11 +492,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 
 	// Get rootID
 	rootInfo, _, err := f.readMetaDataForPath("")
-	if err != nil || rootInfo.ID == "" {
+	if err != nil || rootInfo.GetID() == "" {
 		return nil, errors.Wrap(err, "failed to get root")
 	}
 
-	f.dirCache = dircache.New(root, rootInfo.ID, f)
+	f.dirCache = dircache.New(root, rootInfo.GetID(), f)
 
 	// Find the current root
 	err = f.dirCache.FindRoot(false)
@@ -514,18 +569,11 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 // FindLeaf finds a directory of name leaf in the folder with ID pathID
 func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
 	// fs.Debugf(f, "FindLeaf(%q, %q)", pathID, leaf)
-	parent, ok := f.dirCache.GetInv(pathID)
+	_, ok := f.dirCache.GetInv(pathID)
 	if !ok {
 		return "", false, errors.New("couldn't find parent ID")
 	}
-	path := leaf
-	if parent != "" {
-		path = parent + "/" + path
-	}
-	if f.dirCache.FoundRoot() {
-		path = f.rootSlash() + path
-	}
-	info, resp, err := f.readMetaDataForPath(path)
+	info, resp, err := f.readMetaDataForPathRelativeToID(pathID, leaf)
 	if err != nil {
 		if resp != nil && resp.StatusCode == http.StatusNotFound {
 			return "", false, nil
@@ -867,13 +915,13 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	opts.ExtraHeaders = map[string]string{"Prefer": "respond-async"}
 	opts.NoResponse = true
 
-	id, _, _ := parseDirID(directoryID)
+	id, dstDriveID, _ := parseNormalizedID(directoryID)
 
 	replacedLeaf := replaceReservedChars(leaf)
 	copyReq := api.CopyItemRequest{
 		Name: &replacedLeaf,
 		ParentReference: api.ItemReference{
-			DriveID: f.driveID,
+			DriveID: dstDriveID,
 			ID:      id,
 		},
 	}
@@ -940,15 +988,23 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 		return nil, err
 	}
 
+	id, dstDriveID, _ := parseNormalizedID(directoryID)
+	_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
+
+	if dstDriveID != srcObjDriveID {
+		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
+		// "Items cannot be moved between Drives using this request."
+		return nil, fs.ErrorCantMove
+	}
+
 	// Move the object
 	opts := newOptsCall(srcObj.id, "PATCH", "")
 
-	id, _, _ := parseDirID(directoryID)
-
 	move := api.MoveItemRequest{
 		Name: replaceReservedChars(leaf),
 		ParentReference: &api.ItemReference{
-			ID: id,
+			DriveID: dstDriveID,
+			ID:      id,
 		},
 		// We set the mod time too as it gets reset otherwise
 		FileSystemInfo: &api.FileSystemInfoFacet{
@@ -1024,7 +1080,20 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	if err != nil {
 		return err
 	}
-	parsedDstDirID, _, _ := parseDirID(dstDirectoryID)
+	parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
+
+	// Find ID of src
+	srcID, err := srcFs.dirCache.FindDir(srcRemote, false)
+	if err != nil {
+		return err
+	}
+	_, srcDriveID, _ := parseNormalizedID(srcID)
+
+	if dstDriveID != srcDriveID {
+		// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
+		// "Items cannot be moved between Drives using this request."
+		return fs.ErrorCantDirMove
+	}
 
 	// Check destination does not exist
 	if dstRemote != "" {
@@ -1038,14 +1107,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 		}
 	}
 
-	// Find ID of src
-	srcID, err := srcFs.dirCache.FindDir(srcRemote, false)
-	if err != nil {
-		return err
-	}
-
 	// Get timestamps of src so they can be preserved
-	srcInfo, _, err := srcFs.readMetaDataForPath(srcPath)
+	srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(srcID, "")
 	if err != nil {
 		return err
 	}
@@ -1055,7 +1118,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	move := api.MoveItemRequest{
 		Name: replaceReservedChars(leaf),
 		ParentReference: &api.ItemReference{
-			ID: parsedDstDirID,
+			DriveID: dstDriveID,
+			ID:      parsedDstDirID,
 		},
 		// We set the mod time too as it gets reset otherwise
 		FileSystemInfo: &api.FileSystemInfoFacet{
@@ -1122,7 +1186,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	if err != nil {
 		return "", err
 	}
-	opts := newOptsCall(info.ID, "POST", "/createLink")
+	opts := newOptsCall(info.GetID(), "POST", "/createLink")
 
 	share := api.CreateShareLinkRequest{
 		Type:  "view",
@@ -1270,13 +1334,13 @@ func (o *Object) ModTime() time.Time {
 // setModTime sets the modification time of the local fs object
 func (o *Object) setModTime(modTime time.Time) (*api.Item, error) {
 	var opts rest.Opts
-	_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
-	_, drive, rootURL := parseDirID(directoryID)
+	leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
+	trueDirID, drive, rootURL := parseNormalizedID(directoryID)
 	if drive != "" {
 		opts = rest.Opts{
 			Method:  "PATCH",
 			RootURL: rootURL,
-			Path:    "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()),
+			Path:    "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf),
 		}
 	} else {
 		opts = rest.Opts{
@@ -1344,7 +1408,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 // createUploadSession creates an upload session for the object
 func (o *Object) createUploadSession(modTime time.Time) (response *api.CreateUploadResponse, err error) {
 	leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
-	id, drive, rootURL := parseDirID(directoryID)
+	id, drive, rootURL := parseNormalizedID(directoryID)
 	var opts rest.Opts
 	if drive != "" {
 		opts = rest.Opts{
@@ -1477,13 +1541,13 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
 	fs.Debugf(o, "Starting singlepart upload")
 	var resp *http.Response
 	var opts rest.Opts
-	_, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
-	_, drive, rootURL := parseDirID(directoryID)
+	leaf, directoryID, _ := o.fs.dirCache.FindPath(o.remote, false)
+	trueDirID, drive, rootURL := parseNormalizedID(directoryID)
 	if drive != "" {
 		opts = rest.Opts{
 			Method:        "PUT",
 			RootURL:       rootURL,
-			Path:          "/" + drive + "/root:/" + rest.URLPathEscape(o.srvPath()) + ":/content",
+			Path:          "/" + drive + "/items/" + trueDirID + ":/" + rest.URLPathEscape(leaf) + ":/content",
 			ContentLength: &size,
 			Body:          in,
 		}
@@ -1496,10 +1560,6 @@ func (o *Object) uploadSinglepart(in io.Reader, size int64, modTime time.Time) (
 		}
 	}
 
-	if size == 0 {
-		opts.Body = nil
-	}
-
 	err = o.fs.pacer.Call(func() (bool, error) {
 		resp, err = o.fs.srv.CallJSON(&opts, nil, &info)
 		if apiErr, ok := err.(*api.Error); ok {
@@ -1542,7 +1602,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	} else if size == 0 {
 		info, err = o.uploadSinglepart(in, size, modTime)
 	} else {
-		panic("src file size must be >= 0")
+		return errors.New("unknown-sized upload not supported")
 	}
 	if err != nil {
 		return err
@@ -1566,8 +1626,8 @@ func (o *Object) ID() string {
 	return o.id
 }
 
-func newOptsCall(id string, method string, route string) (opts rest.Opts) {
-	id, drive, rootURL := parseDirID(id)
+func newOptsCall(normalizedID string, method string, route string) (opts rest.Opts) {
+	id, drive, rootURL := parseNormalizedID(normalizedID)
 
 	if drive != "" {
 		return rest.Opts{
@@ -1582,7 +1642,10 @@ func newOptsCall(id string, method string, route string) (opts rest.Opts) {
 	}
 }
 
-func parseDirID(ID string) (string, string, string) {
+// parseNormalizedID parses a normalized ID (may be in the form `driveID#itemID` or just `itemID`)
+// and returns itemID, driveID, rootURL.
+// Such a normalized ID can come from (*Item).GetID()
+func parseNormalizedID(ID string) (string, string, string) {
 	if strings.Index(ID, "#") >= 0 {
 		s := strings.Split(ID, "#")
 		return s[1], s[0], graphURL + "/drives"
@@ -1590,6 +1653,21 @@ func parseDirID(ID string) (string, string, string) {
 	return ID, "", ""
 }
 
+// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
+// returns a relative path for `target` based on `base` and a boolean `true`.
+// Otherwise returns "", false.
+func getRelativePathInsideBase(base, target string) (string, bool) {
+	if base == "" {
+		return target, true
+	}
+
+	baseSlash := base + "/"
+	if strings.HasPrefix(target+"/", baseSlash) {
+		return target[len(baseSlash):], true
+	}
+	return "", false
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)

backend/opendrive/opendrive.go

@@ -119,7 +119,7 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
-// NewFs contstructs an Fs from the path, bucket:path
+// NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -785,7 +785,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		remote := path.Join(dir, folder.Name)
 		// cache the directory ID for later lookups
 		f.dirCache.Put(remote, folder.FolderID)
-		d := fs.NewDir(remote, time.Unix(int64(folder.DateModified), 0)).SetID(folder.FolderID)
+		d := fs.NewDir(remote, time.Unix(folder.DateModified, 0)).SetID(folder.FolderID)
 		d.SetItems(int64(folder.ChildFolders))
 		entries = append(entries, d)
 	}

backend/opendrive/types.go

@@ -13,7 +13,7 @@ type Error struct {
 	} `json:"error"`
 }
 
-// Error statisfies the error interface
+// Error satisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
 }

backend/pcloud/api/types.go

@@ -41,7 +41,7 @@ type Error struct {
 	ErrorString string `json:"error"`
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	return fmt.Sprintf("pcloud error: %s (%d)", e.ErrorString, e.Result)
 }
@@ -58,7 +58,7 @@ func (e *Error) Update(err error) error {
 	return e
 }
 
-// Check Error statisfies the error interface
+// Check Error satisfies the error interface
 var _ error = (*Error)(nil)
 
 // Item describes a folder or a file as returned by Get Folder Items and others

backend/pcloud/pcloud.go

@@ -246,7 +246,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	root = parsePath(root)
 	oAuthClient, ts, err := oauthutil.NewClient(name, m, oauthConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure Pcloud: %v", err)
+		return nil, errors.Wrap(err, "failed to configure Pcloud")
 	}
 
 	f := &Fs{
@@ -385,7 +385,7 @@ func fileIDtoNumber(fileID string) string {
 	if len(fileID) > 0 && fileID[0] == 'f' {
 		return fileID[1:]
 	}
-	fs.Debugf(nil, "Invalid filee id %q", fileID)
+	fs.Debugf(nil, "Invalid file id %q", fileID)
 	return fileID
 }
 

backend/qingstor/qingstor.go

@@ -69,17 +69,57 @@ func init() {
 			}},
 		}, {
 			Name:     "connection_retries",
-			Help:     "Number of connnection retries.",
+			Help:     "Number of connection retries.",
 			Default:  3,
 			Advanced: true,
+		}, {
+			Name: "upload_cutoff",
+			Help: `Cutoff for switching to chunked upload
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5GB.`,
+			Default:  defaultUploadCutoff,
+			Advanced: true,
+		}, {
+			Name: "chunk_size",
+			Help: `Chunk size to use for uploading.
+
+When uploading files larger than upload_cutoff they will be uploaded
+as multipart uploads using this chunk size.
+
+Note that "--qingstor-upload-concurrency" chunks of this size are buffered
+in memory per transfer.
+
+If you are transferring large files over high speed links and you have
+enough memory, then increasing this will speed up the transfers.`,
+			Default:  minChunkSize,
+			Advanced: true,
+		}, {
+			Name: "upload_concurrency",
+			Help: `Concurrency for multipart uploads.
+
+This is the number of chunks of the same file that are uploaded
+concurrently.
+
+NB if you set this to > 1 then the checksums of multpart uploads
+become corrupted (the uploads themselves are not corrupted though).
+
+If you are uploading small numbers of large file over high speed link
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.`,
+			Default:  1,
+			Advanced: true,
 		}},
 	})
 }
 
 // Constants
 const (
-	listLimitSize  = 1000                   // Number of items to read at once
-	maxSizeForCopy = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
+	listLimitSize       = 1000                   // Number of items to read at once
+	maxSizeForCopy      = 1024 * 1024 * 1024 * 5 // The maximum size of object we can COPY
+	minChunkSize        = fs.SizeSuffix(minMultiPartSize)
+	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
+	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
 )
 
 // Globals
@@ -92,12 +132,15 @@ func timestampToTime(tp int64) time.Time {
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth           bool   `config:"env_auth"`
-	AccessKeyID       string `config:"access_key_id"`
-	SecretAccessKey   string `config:"secret_access_key"`
-	Endpoint          string `config:"endpoint"`
-	Zone              string `config:"zone"`
-	ConnectionRetries int    `config:"connection_retries"`
+	EnvAuth           bool          `config:"env_auth"`
+	AccessKeyID       string        `config:"access_key_id"`
+	SecretAccessKey   string        `config:"secret_access_key"`
+	Endpoint          string        `config:"endpoint"`
+	Zone              string        `config:"zone"`
+	ConnectionRetries int           `config:"connection_retries"`
+	UploadCutoff      fs.SizeSuffix `config:"upload_cutoff"`
+	ChunkSize         fs.SizeSuffix `config:"chunk_size"`
+	UploadConcurrency int           `config:"upload_concurrency"`
 }
 
 // Fs represents a remote qingstor server
@@ -227,6 +270,36 @@ func qsServiceConnection(opt *Options) (*qs.Service, error) {
 	return qs.Init(cf)
 }
 
+func checkUploadChunkSize(cs fs.SizeSuffix) error {
+	if cs < minChunkSize {
+		return errors.Errorf("%s is less than %s", cs, minChunkSize)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadChunkSize(cs)
+	if err == nil {
+		old, f.opt.ChunkSize = f.opt.ChunkSize, cs
+	}
+	return
+}
+
+func checkUploadCutoff(cs fs.SizeSuffix) error {
+	if cs > maxUploadCutoff {
+		return errors.Errorf("%s is greater than %s", cs, maxUploadCutoff)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadCutoff(cs)
+	if err == nil {
+		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
+	}
+	return
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -235,6 +308,14 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, err
 	}
+	err = checkUploadChunkSize(opt.ChunkSize)
+	if err != nil {
+		return nil, errors.Wrap(err, "qingstor: chunk size")
+	}
+	err = checkUploadCutoff(opt.UploadCutoff)
+	if err != nil {
+		return nil, errors.Wrap(err, "qingstor: upload cutoff")
+	}
 	bucket, key, err := qsParsePath(root)
 	if err != nil {
 		return nil, err
@@ -368,7 +449,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	_, err = bucketInit.PutObject(key, &req)
 	if err != nil {
-		fs.Debugf(f, "Copied Faild, API Error: %v", err)
+		fs.Debugf(f, "Copy Failed, API Error: %v", err)
 		return nil, err
 	}
 	return f.NewObject(remote)
@@ -675,7 +756,7 @@ func (f *Fs) Mkdir(dir string) error {
 		}
 		switch *statistics.Status {
 		case "deleted":
-			fs.Debugf(f, "Wiat for qingstor sync bucket status, retries: %d", retries)
+			fs.Debugf(f, "Wait for qingstor sync bucket status, retries: %d", retries)
 			time.Sleep(time.Second * 1)
 			retries++
 			continue
@@ -794,7 +875,7 @@ func (o *Object) readMetaData() (err error) {
 	fs.Debugf(o, "Read metadata of key: %s", key)
 	resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{})
 	if err != nil {
-		fs.Debugf(o, "Read metadata faild, API Error: %v", err)
+		fs.Debugf(o, "Read metadata failed, API Error: %v", err)
 		if e, ok := err.(*qsErr.QingStorError); ok {
 			if e.StatusCode == http.StatusNotFound {
 				return fs.ErrorObjectNotFound
@@ -913,16 +994,24 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	mimeType := fs.MimeType(src)
 
 	req := uploadInput{
-		body:     in,
-		qsSvc:    o.fs.svc,
-		bucket:   o.fs.bucket,
-		zone:     o.fs.zone,
-		key:      key,
-		mimeType: mimeType,
+		body:        in,
+		qsSvc:       o.fs.svc,
+		bucket:      o.fs.bucket,
+		zone:        o.fs.zone,
+		key:         key,
+		mimeType:    mimeType,
+		partSize:    int64(o.fs.opt.ChunkSize),
+		concurrency: o.fs.opt.UploadConcurrency,
 	}
 	uploader := newUploader(&req)
 
-	err = uploader.upload()
+	size := src.Size()
+	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
+	if multipart {
+		err = uploader.upload()
+	} else {
+		err = uploader.singlePartUpload(in, size)
+	}
 	if err != nil {
 		return err
 	}

backend/qingstor/qingstor_test.go

@@ -2,12 +2,12 @@
 
 // +build !plan9
 
-package qingstor_test
+package qingstor
 
 import (
 	"testing"
 
-	"github.com/ncw/rclone/backend/qingstor"
+	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
@@ -15,6 +15,19 @@ import (
 func TestIntegration(t *testing.T) {
 	fstests.Run(t, &fstests.Opt{
 		RemoteName: "TestQingStor:",
-		NilObject:  (*qingstor.Object)(nil),
+		NilObject:  (*Object)(nil),
+		ChunkedUpload: fstests.ChunkedUploadConfig{
+			MinChunkSize: minChunkSize,
+		},
 	})
 }
+
+func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadChunkSize(cs)
+}
+
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadCutoff(cs)
+}
+
+var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/qingstor/upload.go

@@ -143,7 +143,7 @@ func (u *uploader) init() {
 
 		// Try to adjust partSize if it is too small and account for
 		// integer division truncation.
-		if u.totalSize/u.cfg.partSize >= int64(u.cfg.partSize) {
+		if u.totalSize/u.cfg.partSize >= u.cfg.partSize {
 			// Add one to the part size to account for remainders
 			// during the size calculation. e.g odd number of bytes.
 			u.cfg.partSize = (u.totalSize / int64(u.cfg.maxUploadParts)) + 1
@@ -152,18 +152,18 @@ func (u *uploader) init() {
 }
 
 // singlePartUpload upload a single object that contentLength less than "defaultUploadPartSize"
-func (u *uploader) singlePartUpload(buf io.ReadSeeker) error {
+func (u *uploader) singlePartUpload(buf io.Reader, size int64) error {
 	bucketInit, _ := u.bucketInit()
 
 	req := qs.PutObjectInput{
-		ContentLength: &u.readerPos,
+		ContentLength: &size,
 		ContentType:   &u.cfg.mimeType,
 		Body:          buf,
 	}
 
 	_, err := bucketInit.PutObject(u.cfg.key, &req)
 	if err == nil {
-		fs.Debugf(u, "Upload single objcet finished")
+		fs.Debugf(u, "Upload single object finished")
 	}
 	return err
 }
@@ -179,13 +179,13 @@ func (u *uploader) upload() error {
 	// Do one read to determine if we have more than one part
 	reader, _, err := u.nextReader()
 	if err == io.EOF { // single part
-		fs.Debugf(u, "Tried to upload a singile object to QingStor")
-		return u.singlePartUpload(reader)
+		fs.Debugf(u, "Uploading as single part object to QingStor")
+		return u.singlePartUpload(reader, u.readerPos)
 	} else if err != nil {
 		return errors.Errorf("read upload data failed: %s", err)
 	}
 
-	fs.Debugf(u, "Treied to upload a multi-part object to QingStor")
+	fs.Debugf(u, "Uploading as multi-part object to QingStor")
 	mu := multiUploader{uploader: u}
 	return mu.multiPartUpload(reader)
 }
@@ -261,7 +261,7 @@ func (mu *multiUploader) initiate() error {
 	req := qs.InitiateMultipartUploadInput{
 		ContentType: &mu.cfg.mimeType,
 	}
-	fs.Debugf(mu, "Tried to initiate a multi-part upload")
+	fs.Debugf(mu, "Initiating a multi-part upload")
 	rsp, err := bucketInit.InitiateMultipartUpload(mu.cfg.key, &req)
 	if err == nil {
 		mu.uploadID = rsp.UploadID
@@ -279,12 +279,12 @@ func (mu *multiUploader) send(c chunk) error {
 		ContentLength: &c.size,
 		Body:          c.buffer,
 	}
-	fs.Debugf(mu, "Tried to upload a part to QingStor that partNumber %d and partSize %d", c.partNumber, c.size)
+	fs.Debugf(mu, "Uploading a part to QingStor with partNumber %d and partSize %d", c.partNumber, c.size)
 	_, err := bucketInit.UploadMultipart(mu.cfg.key, &req)
 	if err != nil {
 		return err
 	}
-	fs.Debugf(mu, "Upload part finished that partNumber %d and partSize %d", c.partNumber, c.size)
+	fs.Debugf(mu, "Done uploading part partNumber %d and partSize %d", c.partNumber, c.size)
 
 	mu.mtx.Lock()
 	defer mu.mtx.Unlock()
@@ -304,7 +304,7 @@ func (mu *multiUploader) list() error {
 	req := qs.ListMultipartInput{
 		UploadID: mu.uploadID,
 	}
-	fs.Debugf(mu, "Tried to list a multi-part")
+	fs.Debugf(mu, "Reading multi-part details")
 	rsp, err := bucketInit.ListMultipart(mu.cfg.key, &req)
 	if err == nil {
 		mu.objectParts = rsp.ObjectParts
@@ -331,7 +331,7 @@ func (mu *multiUploader) complete() error {
 		ObjectParts: mu.objectParts,
 		ETag:        &md5String,
 	}
-	fs.Debugf(mu, "Tried to complete a multi-part")
+	fs.Debugf(mu, "Completing multi-part object")
 	_, err = bucketInit.CompleteMultipartUpload(mu.cfg.key, &req)
 	if err == nil {
 		fs.Debugf(mu, "Complete multi-part finished")
@@ -348,7 +348,7 @@ func (mu *multiUploader) abort() error {
 		req := qs.AbortMultipartUploadInput{
 			UploadID: uploadID,
 		}
-		fs.Debugf(mu, "Tried to abort a multi-part")
+		fs.Debugf(mu, "Aborting multi-part object %q", *uploadID)
 		_, err = bucketInit.AbortMultipartUpload(mu.cfg.key, &req)
 	}
 
@@ -392,6 +392,14 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) error {
 		var nextChunkLen int
 		reader, nextChunkLen, err = mu.nextReader()
 		if err != nil && err != io.EOF {
+			// empty ch
+			go func() {
+				for range ch {
+				}
+			}()
+			// Wait for all goroutines finish
+			close(ch)
+			mu.wg.Wait()
 			return err
 		}
 		if nextChunkLen == 0 && partNumber > 0 {

backend/s3/s3.go

@@ -53,7 +53,7 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
+		Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: fs.ConfigProvider,
@@ -61,6 +61,9 @@ func init() {
 			Examples: []fs.OptionExample{{
 				Value: "AWS",
 				Help:  "Amazon Web Services (AWS) S3",
+			}, {
+				Value: "Alibaba",
+				Help:  "Alibaba Cloud Object Storage System (OSS) formerly Aliyun",
 			}, {
 				Value: "Ceph",
 				Help:  "Ceph Object Storage",
@@ -76,6 +79,9 @@ func init() {
 			}, {
 				Value: "Minio",
 				Help:  "Minio Object Storage",
+			}, {
+				Value: "Netease",
+				Help:  "Netease Object Storage (NOS)",
 			}, {
 				Value: "Wasabi",
 				Help:  "Wasabi Object Storage",
@@ -125,6 +131,9 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region\nNeeds location constraint eu-west-2.",
+			}, {
+				Value: "eu-north-1",
+				Help:  "EU (Stockholm) Region\nNeeds location constraint eu-north-1.",
 			}, {
 				Value: "eu-central-1",
 				Help:  "EU (Frankfurt) Region\nNeeds location constraint eu-central-1.",
@@ -150,7 +159,7 @@ func init() {
 		}, {
 			Name:     "region",
 			Help:     "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
-			Provider: "!AWS",
+			Provider: "!AWS,Alibaba",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
@@ -228,10 +237,10 @@ func init() {
 				Help:  "EU Cross Region Amsterdam Private Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.softlayer.net",
-				Help:  "Great Britan Endpoint",
+				Help:  "Great Britain Endpoint",
 			}, {
 				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
-				Help:  "Great Britan Private Endpoint",
+				Help:  "Great Britain Private Endpoint",
 			}, {
 				Value: "s3.ap-geo.objectstorage.softlayer.net",
 				Help:  "APAC Cross Regional Endpoint",
@@ -269,10 +278,73 @@ func init() {
 				Value: "s3.tor01.objectstorage.service.networklayer.com",
 				Help:  "Toronto Single Site Private Endpoint",
 			}},
+		}, {
+			// oss endpoints: https://help.aliyun.com/document_detail/31837.html
+			Name:     "endpoint",
+			Help:     "Endpoint for OSS API.",
+			Provider: "Alibaba",
+			Examples: []fs.OptionExample{{
+				Value: "oss-cn-hangzhou.aliyuncs.com",
+				Help:  "East China 1 (Hangzhou)",
+			}, {
+				Value: "oss-cn-shanghai.aliyuncs.com",
+				Help:  "East China 2 (Shanghai)",
+			}, {
+				Value: "oss-cn-qingdao.aliyuncs.com",
+				Help:  "North China 1 (Qingdao)",
+			}, {
+				Value: "oss-cn-beijing.aliyuncs.com",
+				Help:  "North China 2 (Beijing)",
+			}, {
+				Value: "oss-cn-zhangjiakou.aliyuncs.com",
+				Help:  "North China 3 (Zhangjiakou)",
+			}, {
+				Value: "oss-cn-huhehaote.aliyuncs.com",
+				Help:  "North China 5 (Huhehaote)",
+			}, {
+				Value: "oss-cn-shenzhen.aliyuncs.com",
+				Help:  "South China 1 (Shenzhen)",
+			}, {
+				Value: "oss-cn-hongkong.aliyuncs.com",
+				Help:  "Hong Kong (Hong Kong)",
+			}, {
+				Value: "oss-us-west-1.aliyuncs.com",
+				Help:  "US West 1 (Silicon Valley)",
+			}, {
+				Value: "oss-us-east-1.aliyuncs.com",
+				Help:  "US East 1 (Virginia)",
+			}, {
+				Value: "oss-ap-southeast-1.aliyuncs.com",
+				Help:  "Southeast Asia Southeast 1 (Singapore)",
+			}, {
+				Value: "oss-ap-southeast-2.aliyuncs.com",
+				Help:  "Asia Pacific Southeast 2 (Sydney)",
+			}, {
+				Value: "oss-ap-southeast-3.aliyuncs.com",
+				Help:  "Southeast Asia Southeast 3 (Kuala Lumpur)",
+			}, {
+				Value: "oss-ap-southeast-5.aliyuncs.com",
+				Help:  "Asia Pacific Southeast 5 (Jakarta)",
+			}, {
+				Value: "oss-ap-northeast-1.aliyuncs.com",
+				Help:  "Asia Pacific Northeast 1 (Japan)",
+			}, {
+				Value: "oss-ap-south-1.aliyuncs.com",
+				Help:  "Asia Pacific South 1 (Mumbai)",
+			}, {
+				Value: "oss-eu-central-1.aliyuncs.com",
+				Help:  "Central Europe 1 (Frankfurt)",
+			}, {
+				Value: "oss-eu-west-1.aliyuncs.com",
+				Help:  "West Europe (London)",
+			}, {
+				Value: "oss-me-east-1.aliyuncs.com",
+				Help:  "Middle East 1 (Dubai)",
+			}},
 		}, {
 			Name:     "endpoint",
 			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
-			Provider: "!AWS,IBMCOS",
+			Provider: "!AWS,IBMCOS,Alibaba",
 			Examples: []fs.OptionExample{{
 				Value:    "objects-us-west-1.dream.io",
 				Help:     "Dream Objects endpoint",
@@ -291,7 +363,11 @@ func init() {
 				Provider: "DigitalOcean",
 			}, {
 				Value:    "s3.wasabisys.com",
-				Help:     "Wasabi Object Storage",
+				Help:     "Wasabi US East endpoint",
+				Provider: "Wasabi",
+			}, {
+				Value:    "s3.us-west-1.wasabisys.com",
+				Help:     "Wasabi US West endpoint",
 				Provider: "Wasabi",
 			}},
 		}, {
@@ -319,6 +395,9 @@ func init() {
 			}, {
 				Value: "eu-west-2",
 				Help:  "EU (London) Region.",
+			}, {
+				Value: "eu-north-1",
+				Help:  "EU (Stockholm) Region.",
 			}, {
 				Value: "EU",
 				Help:  "EU Region.",
@@ -371,7 +450,7 @@ func init() {
 				Help:  "US East Region Flex",
 			}, {
 				Value: "us-south-standard",
-				Help:  "US Sout hRegion Standard",
+				Help:  "US South Region Standard",
 			}, {
 				Value: "us-south-vault",
 				Help:  "US South Region Vault",
@@ -395,16 +474,16 @@ func init() {
 				Help:  "EU Cross Region Flex",
 			}, {
 				Value: "eu-gb-standard",
-				Help:  "Great Britan Standard",
+				Help:  "Great Britain Standard",
 			}, {
 				Value: "eu-gb-vault",
-				Help:  "Great Britan Vault",
+				Help:  "Great Britain Vault",
 			}, {
 				Value: "eu-gb-cold",
-				Help:  "Great Britan Cold",
+				Help:  "Great Britain Cold",
 			}, {
 				Value: "eu-gb-flex",
-				Help:  "Great Britan Flex",
+				Help:  "Great Britain Flex",
 			}, {
 				Value: "ap-standard",
 				Help:  "APAC Standard",
@@ -445,11 +524,13 @@ func init() {
 		}, {
 			Name:     "location_constraint",
 			Help:     "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
-			Provider: "!AWS,IBMCOS",
+			Provider: "!AWS,IBMCOS,Alibaba",
 		}, {
 			Name: "acl",
 			Help: `Canned ACL used when creating buckets and storing or copying objects.
 
+This ACL is used for creating objects and if bucket_acl isn't set, for creating buckets too.
+
 For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
 
 Note that this ACL is applied when server side copying objects as S3
@@ -495,6 +576,28 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 				Help:     "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS",
 				Provider: "IBMCOS",
 			}},
+		}, {
+			Name: "bucket_acl",
+			Help: `Canned ACL used when creating buckets.
+
+For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+
+Note that this ACL is applied when only when creating buckets.  If it
+isn't set then "acl" is used instead.`,
+			Advanced: true,
+			Examples: []fs.OptionExample{{
+				Value: "private",
+				Help:  "Owner gets FULL_CONTROL. No one else has access rights (default).",
+			}, {
+				Value: "public-read",
+				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
+			}, {
+				Value: "public-read-write",
+				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.",
+			}, {
+				Value: "authenticated-read",
+				Help:  "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.",
+			}},
 		}, {
 			Name:     "server_side_encryption",
 			Help:     "The server-side encryption algorithm used when storing this object in S3.",
@@ -539,13 +642,42 @@ doesn't copy the ACL from the source but rather writes a fresh one.`,
 			}, {
 				Value: "ONEZONE_IA",
 				Help:  "One Zone Infrequent Access storage class",
+			}, {
+				Value: "GLACIER",
+				Help:  "Glacier storage class",
 			}},
+		}, {
+			// Mapping from here: https://www.alibabacloud.com/help/doc-detail/64919.htm
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing new objects in OSS.",
+			Provider: "Alibaba",
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "Default",
+			}, {
+				Value: "STANDARD",
+				Help:  "Standard storage class",
+			}, {
+				Value: "GLACIER",
+				Help:  "Archive storage mode.",
+			}, {
+				Value: "STANDARD_IA",
+				Help:  "Infrequent access storage mode.",
+			}},
+		}, {
+			Name: "upload_cutoff",
+			Help: `Cutoff for switching to chunked upload
+
+Any files larger than this will be uploaded in chunks of chunk_size.
+The minimum is 0 and the maximum is 5GB.`,
+			Default:  defaultUploadCutoff,
+			Advanced: true,
 		}, {
 			Name: "chunk_size",
 			Help: `Chunk size to use for uploading.
 
-Any files larger than this will be uploaded in chunks of this
-size. The default is 5MB. The minimum is 5MB.
+When uploading files larger than upload_cutoff they will be uploaded
+as multipart uploads using this chunk size.
 
 Note that "--s3-upload-concurrency" chunks of this size are buffered
 in memory per transfer.
@@ -573,7 +705,7 @@ concurrently.
 If you are uploading small numbers of large file over high speed link
 and these uploads do not fully utilize your bandwidth, then increasing
 this may help to speed up the transfers.`,
-			Default:  2,
+			Default:  4,
 			Advanced: true,
 		}, {
 			Name: "force_path_style",
@@ -603,14 +735,16 @@ Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.`,
 
 // Constants
 const (
-	metaMtime      = "Mtime"                       // the meta key to store mtime in - eg X-Amz-Meta-Mtime
-	metaMD5Hash    = "Md5chksum"                   // the meta key to store md5hash in
-	listChunkSize  = 1000                          // number of items to read at once
-	maxRetries     = 10                            // number of retries to make of operations
-	maxSizeForCopy = 5 * 1024 * 1024 * 1024        // The maximum size of object we can COPY
-	maxFileSize    = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
-	minChunkSize   = fs.SizeSuffix(s3manager.MinUploadPartSize)
-	minSleep       = 10 * time.Millisecond // In case of error, start at 10ms sleep.
+	metaMtime           = "Mtime"                       // the meta key to store mtime in - eg X-Amz-Meta-Mtime
+	metaMD5Hash         = "Md5chksum"                   // the meta key to store md5hash in
+	listChunkSize       = 1000                          // number of items to read at once
+	maxRetries          = 10                            // number of retries to make of operations
+	maxSizeForCopy      = 5 * 1024 * 1024 * 1024        // The maximum size of object we can COPY
+	maxFileSize         = 5 * 1024 * 1024 * 1024 * 1024 // largest possible upload file size
+	minChunkSize        = fs.SizeSuffix(s3manager.MinUploadPartSize)
+	defaultUploadCutoff = fs.SizeSuffix(200 * 1024 * 1024)
+	maxUploadCutoff     = fs.SizeSuffix(5 * 1024 * 1024 * 1024)
+	minSleep            = 10 * time.Millisecond // In case of error, start at 10ms sleep.
 )
 
 // Options defines the configuration for this backend
@@ -623,9 +757,11 @@ type Options struct {
 	Endpoint             string        `config:"endpoint"`
 	LocationConstraint   string        `config:"location_constraint"`
 	ACL                  string        `config:"acl"`
+	BucketACL            string        `config:"bucket_acl"`
 	ServerSideEncryption string        `config:"server_side_encryption"`
 	SSEKMSKeyID          string        `config:"sse_kms_key_id"`
 	StorageClass         string        `config:"storage_class"`
+	UploadCutoff         fs.SizeSuffix `config:"upload_cutoff"`
 	ChunkSize            fs.SizeSuffix `config:"chunk_size"`
 	DisableChecksum      bool          `config:"disable_checksum"`
 	SessionToken         string        `config:"session_token"`
@@ -647,6 +783,7 @@ type Fs struct {
 	bucketOK      bool             // true if we have created the bucket
 	bucketDeleted bool             // true if we have deleted the bucket
 	pacer         *pacer.Pacer     // To pace the API calls
+	srv           *http.Client     // a plain http client
 }
 
 // Object describes a s3 object
@@ -695,23 +832,31 @@ func (f *Fs) Features() *fs.Features {
 // retryErrorCodes is a slice of error codes that we will retry
 // See: https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 var retryErrorCodes = []int{
-	409, // Conflict - various states that could be resolved on a retry
+	// 409, // Conflict - various states that could be resolved on a retry
 	503, // Service Unavailable/Slow Down - "Reduce your request rate"
 }
 
 //S3 is pretty resilient, and the built in retry handling is probably sufficient
 // as it should notice closed connections and timeouts which are the most likely
 // sort of failure modes
-func shouldRetry(err error) (bool, error) {
-
+func (f *Fs) shouldRetry(err error) (bool, error) {
 	// If this is an awserr object, try and extract more useful information to determine if we should retry
 	if awsError, ok := err.(awserr.Error); ok {
-		// Simple case, check the original embedded error in case it's generically retriable
+		// Simple case, check the original embedded error in case it's generically retryable
 		if fserrors.ShouldRetry(awsError.OrigErr()) {
 			return true, err
 		}
-		//Failing that, if it's a RequestFailure it's probably got an http status code we can check
+		// Failing that, if it's a RequestFailure it's probably got an http status code we can check
 		if reqErr, ok := err.(awserr.RequestFailure); ok {
+			// 301 if wrong region for bucket
+			if reqErr.StatusCode() == http.StatusMovedPermanently {
+				urfbErr := f.updateRegionForBucket()
+				if urfbErr != nil {
+					fs.Errorf(f, "Failed to update region for bucket: %v", urfbErr)
+					return false, err
+				}
+				return true, err
+			}
 			for _, e := range retryErrorCodes {
 				if reqErr.StatusCode() == e {
 					return true, err
@@ -719,7 +864,7 @@ func shouldRetry(err error) (bool, error) {
 			}
 		}
 	}
-	//Ok, not an awserr, check for generic failure conditions
+	// Ok, not an awserr, check for generic failure conditions
 	return fserrors.ShouldRetry(err), err
 }
 
@@ -796,16 +941,37 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
 	if opt.Region == "" {
 		opt.Region = "us-east-1"
 	}
+	if opt.Provider == "Alibaba" || opt.Provider == "Netease" {
+		opt.ForcePathStyle = false
+	}
 	awsConfig := aws.NewConfig().
-		WithRegion(opt.Region).
 		WithMaxRetries(maxRetries).
 		WithCredentials(cred).
-		WithEndpoint(opt.Endpoint).
 		WithHTTPClient(fshttp.NewClient(fs.Config)).
 		WithS3ForcePathStyle(opt.ForcePathStyle)
+	if opt.Region != "" {
+		awsConfig.WithRegion(opt.Region)
+	}
+	if opt.Endpoint != "" {
+		awsConfig.WithEndpoint(opt.Endpoint)
+	}
+
 	// awsConfig.WithLogLevel(aws.LogDebugWithSigning)
-	ses := session.New()
-	c := s3.New(ses, awsConfig)
+	awsSessionOpts := session.Options{
+		Config: *awsConfig,
+	}
+	if opt.EnvAuth && opt.AccessKeyID == "" && opt.SecretAccessKey == "" {
+		// Enable loading config options from ~/.aws/config (selected by AWS_PROFILE env)
+		awsSessionOpts.SharedConfigState = session.SharedConfigEnable
+		// The session constructor (aws/session/mergeConfigSrcs) will only use the user's preferred credential source
+		// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
+		awsSessionOpts.Config.Credentials = nil
+	}
+	ses, err := session.NewSessionWithOptions(awsSessionOpts)
+	if err != nil {
+		return nil, nil, err
+	}
+	c := s3.New(ses)
 	if opt.V2Auth || opt.Region == "other-v2-signature" {
 		fs.Debugf(nil, "Using v2 auth")
 		signer := func(req *request.Request) {
@@ -837,6 +1003,21 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
 	return
 }
 
+func checkUploadCutoff(cs fs.SizeSuffix) error {
+	if cs > maxUploadCutoff {
+		return errors.Errorf("%s is greater than %s", cs, maxUploadCutoff)
+	}
+	return nil
+}
+
+func (f *Fs) setUploadCutoff(cs fs.SizeSuffix) (old fs.SizeSuffix, err error) {
+	err = checkUploadCutoff(cs)
+	if err == nil {
+		old, f.opt.UploadCutoff = f.opt.UploadCutoff, cs
+	}
+	return
+}
+
 // NewFs constructs an Fs from the path, bucket:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
@@ -849,10 +1030,20 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "s3: chunk size")
 	}
+	err = checkUploadCutoff(opt.UploadCutoff)
+	if err != nil {
+		return nil, errors.Wrap(err, "s3: upload cutoff")
+	}
 	bucket, directory, err := s3ParsePath(root)
 	if err != nil {
 		return nil, err
 	}
+	if opt.ACL == "" {
+		opt.ACL = "private"
+	}
+	if opt.BucketACL == "" {
+		opt.BucketACL = opt.ACL
+	}
 	c, ses, err := s3Connection(opt)
 	if err != nil {
 		return nil, err
@@ -865,6 +1056,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		bucket: bucket,
 		ses:    ses,
 		pacer:  pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
+		srv:    fshttp.NewClient(fs.Config),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -880,7 +1072,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		}
 		err = f.pacer.Call(func() (bool, error) {
 			_, err = f.c.HeadObject(&req)
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err == nil {
 			f.root = path.Dir(directory)
@@ -930,6 +1122,51 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
+// Gets the bucket location
+func (f *Fs) getBucketLocation() (string, error) {
+	req := s3.GetBucketLocationInput{
+		Bucket: &f.bucket,
+	}
+	var resp *s3.GetBucketLocationOutput
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.c.GetBucketLocation(&req)
+		return f.shouldRetry(err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return s3.NormalizeBucketLocation(aws.StringValue(resp.LocationConstraint)), nil
+}
+
+// Updates the region for the bucket by reading the region from the
+// bucket then updating the session.
+func (f *Fs) updateRegionForBucket() error {
+	region, err := f.getBucketLocation()
+	if err != nil {
+		return errors.Wrap(err, "reading bucket location failed")
+	}
+	if aws.StringValue(f.c.Config.Endpoint) != "" {
+		return errors.Errorf("can't set region to %q as endpoint is set", region)
+	}
+	if aws.StringValue(f.c.Config.Region) == region {
+		return errors.Errorf("region is already %q - not updating", region)
+	}
+
+	// Make a new session with the new region
+	oldRegion := f.opt.Region
+	f.opt.Region = region
+	c, ses, err := s3Connection(&f.opt)
+	if err != nil {
+		return errors.Wrap(err, "creating new session failed")
+	}
+	f.c = c
+	f.ses = ses
+
+	fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
+	return nil
+}
+
 // listFn is called from list to handle an object.
 type listFn func(remote string, object *s3.Object, isDirectory bool) error
 
@@ -962,7 +1199,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 		var err error
 		err = f.pacer.Call(func() (bool, error) {
 			resp, err = f.c.ListObjects(&req)
-			return shouldRetry(err)
+			return f.shouldRetry(err)
 		})
 		if err != nil {
 			if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1091,7 +1328,7 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	var resp *s3.ListBucketsOutput
 	err = f.pacer.Call(func() (bool, error) {
 		resp, err = f.c.ListBuckets(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1179,7 +1416,7 @@ func (f *Fs) dirExists() (bool, error) {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.HeadBucket(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err == nil {
 		return true, nil
@@ -1210,7 +1447,7 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	req := s3.CreateBucketInput{
 		Bucket: &f.bucket,
-		ACL:    &f.opt.ACL,
+		ACL:    &f.opt.BucketACL,
 	}
 	if f.opt.LocationConstraint != "" {
 		req.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
@@ -1219,7 +1456,7 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.CreateBucket(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err, ok := err.(awserr.Error); ok {
 		if err.Code() == "BucketAlreadyOwnedByYou" {
@@ -1229,6 +1466,7 @@ func (f *Fs) Mkdir(dir string) error {
 	if err == nil {
 		f.bucketOK = true
 		f.bucketDeleted = false
+		fs.Infof(f, "Bucket created with ACL %q", *req.ACL)
 	}
 	return err
 }
@@ -1247,11 +1485,12 @@ func (f *Fs) Rmdir(dir string) error {
 	}
 	err := f.pacer.Call(func() (bool, error) {
 		_, err := f.c.DeleteBucket(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err == nil {
 		f.bucketOK = false
 		f.bucketDeleted = true
+		fs.Infof(f, "Bucket deleted")
 	}
 	return err
 }
@@ -1307,7 +1546,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	}
 	err = f.pacer.Call(func() (bool, error) {
 		_, err = f.c.CopyObject(&req)
-		return shouldRetry(err)
+		return f.shouldRetry(err)
 	})
 	if err != nil {
 		return nil, err
@@ -1389,7 +1628,7 @@ func (o *Object) readMetaData() (err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.HeadObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	if err != nil {
 		if awsErr, ok := err.(awserr.RequestFailure); ok {
@@ -1485,7 +1724,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	}
 	err = o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.CopyObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	return err
 }
@@ -1517,7 +1756,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	err = o.fs.pacer.Call(func() (bool, error) {
 		var err error
 		resp, err = o.fs.c.GetObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	if err, ok := err.(awserr.RequestFailure); ok {
 		if err.Code() == "InvalidObjectState" {
@@ -1539,38 +1778,46 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	modTime := src.ModTime()
 	size := src.Size()
 
-	uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
-		u.Concurrency = o.fs.opt.UploadConcurrency
-		u.LeavePartsOnError = false
-		u.S3 = o.fs.c
-		u.PartSize = int64(o.fs.opt.ChunkSize)
+	multipart := size < 0 || size >= int64(o.fs.opt.UploadCutoff)
+	var uploader *s3manager.Uploader
+	if multipart {
+		uploader = s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
+			u.Concurrency = o.fs.opt.UploadConcurrency
+			u.LeavePartsOnError = false
+			u.S3 = o.fs.c
+			u.PartSize = int64(o.fs.opt.ChunkSize)
 
-		if size == -1 {
-			// Make parts as small as possible while still being able to upload to the
-			// S3 file size limit. Rounded up to nearest MB.
-			u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
-			return
-		}
-		// Adjust PartSize until the number of parts is small enough.
-		if size/u.PartSize >= s3manager.MaxUploadParts {
-			// Calculate partition size rounded up to the nearest MB
-			u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20
-		}
-	})
+			if size == -1 {
+				// Make parts as small as possible while still being able to upload to the
+				// S3 file size limit. Rounded up to nearest MB.
+				u.PartSize = (((maxFileSize / s3manager.MaxUploadParts) >> 20) + 1) << 20
+				return
+			}
+			// Adjust PartSize until the number of parts is small enough.
+			if size/u.PartSize >= s3manager.MaxUploadParts {
+				// Calculate partition size rounded up to the nearest MB
+				u.PartSize = (((size / s3manager.MaxUploadParts) >> 20) + 1) << 20
+			}
+		})
+	}
 
 	// Set the mtime in the meta data
 	metadata := map[string]*string{
 		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
 	}
 
-	if !o.fs.opt.DisableChecksum && size > uploader.PartSize {
+	// read the md5sum if available for non multpart and if
+	// disable checksum isn't present.
+	var md5sum string
+	if !multipart || !o.fs.opt.DisableChecksum {
 		hash, err := src.Hash(hash.MD5)
-
 		if err == nil && matchMd5.MatchString(hash) {
 			hashBytes, err := hex.DecodeString(hash)
-
 			if err == nil {
-				metadata[metaMD5Hash] = aws.String(base64.StdEncoding.EncodeToString(hashBytes))
+				md5sum = base64.StdEncoding.EncodeToString(hashBytes)
+				if multipart {
+					metadata[metaMD5Hash] = &md5sum
+				}
 			}
 		}
 	}
@@ -1579,30 +1826,98 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	mimeType := fs.MimeType(src)
 
 	key := o.fs.root + o.remote
-	req := s3manager.UploadInput{
-		Bucket:      &o.fs.bucket,
-		ACL:         &o.fs.opt.ACL,
-		Key:         &key,
-		Body:        in,
-		ContentType: &mimeType,
-		Metadata:    metadata,
-		//ContentLength: &size,
-	}
-	if o.fs.opt.ServerSideEncryption != "" {
-		req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
-	}
-	if o.fs.opt.SSEKMSKeyID != "" {
-		req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
-	}
-	if o.fs.opt.StorageClass != "" {
-		req.StorageClass = &o.fs.opt.StorageClass
-	}
-	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-		_, err = uploader.Upload(&req)
-		return shouldRetry(err)
-	})
-	if err != nil {
-		return err
+	if multipart {
+		req := s3manager.UploadInput{
+			Bucket:      &o.fs.bucket,
+			ACL:         &o.fs.opt.ACL,
+			Key:         &key,
+			Body:        in,
+			ContentType: &mimeType,
+			Metadata:    metadata,
+			//ContentLength: &size,
+		}
+		if o.fs.opt.ServerSideEncryption != "" {
+			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
+		}
+		if o.fs.opt.SSEKMSKeyID != "" {
+			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
+		}
+		if o.fs.opt.StorageClass != "" {
+			req.StorageClass = &o.fs.opt.StorageClass
+		}
+		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+			_, err = uploader.Upload(&req)
+			return o.fs.shouldRetry(err)
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		req := s3.PutObjectInput{
+			Bucket:      &o.fs.bucket,
+			ACL:         &o.fs.opt.ACL,
+			Key:         &key,
+			ContentType: &mimeType,
+			Metadata:    metadata,
+		}
+		if md5sum != "" {
+			req.ContentMD5 = &md5sum
+		}
+		if o.fs.opt.ServerSideEncryption != "" {
+			req.ServerSideEncryption = &o.fs.opt.ServerSideEncryption
+		}
+		if o.fs.opt.SSEKMSKeyID != "" {
+			req.SSEKMSKeyId = &o.fs.opt.SSEKMSKeyID
+		}
+		if o.fs.opt.StorageClass != "" {
+			req.StorageClass = &o.fs.opt.StorageClass
+		}
+
+		// Create the request
+		putObj, _ := o.fs.c.PutObjectRequest(&req)
+
+		// Sign it so we can upload using a presigned request.
+		//
+		// Note the SDK doesn't currently support streaming to
+		// PutObject so we'll use this work-around.
+		url, headers, err := putObj.PresignRequest(15 * time.Minute)
+		if err != nil {
+			return errors.Wrap(err, "s3 upload: sign request")
+		}
+
+		// Set request to nil if empty so as not to make chunked encoding
+		if size == 0 {
+			in = nil
+		}
+
+		// create the vanilla http request
+		httpReq, err := http.NewRequest("PUT", url, in)
+		if err != nil {
+			return errors.Wrap(err, "s3 upload: new request")
+		}
+
+		// set the headers we signed and the length
+		httpReq.Header = headers
+		httpReq.ContentLength = size
+
+		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+			resp, err := o.fs.srv.Do(httpReq)
+			if err != nil {
+				return o.fs.shouldRetry(err)
+			}
+			body, err := rest.ReadBody(resp)
+			if err != nil {
+				return o.fs.shouldRetry(err)
+			}
+			if resp.StatusCode >= 200 && resp.StatusCode < 299 {
+				return false, nil
+			}
+			err = errors.Errorf("s3 upload: %s: %s", resp.Status, body)
+			return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
+		})
+		if err != nil {
+			return err
+		}
 	}
 
 	// Read the metadata from the newly created object
@@ -1620,7 +1935,7 @@ func (o *Object) Remove() error {
 	}
 	err := o.fs.pacer.Call(func() (bool, error) {
 		_, err := o.fs.c.DeleteObject(&req)
-		return shouldRetry(err)
+		return o.fs.shouldRetry(err)
 	})
 	return err
 }

backend/s3/s3_test.go

@@ -23,4 +23,8 @@ func (f *Fs) SetUploadChunkSize(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
 	return f.setUploadChunkSize(cs)
 }
 
+func (f *Fs) SetUploadCutoff(cs fs.SizeSuffix) (fs.SizeSuffix, error) {
+	return f.setUploadCutoff(cs)
+}
+
 var _ fstests.SetUploadChunkSizer = (*Fs)(nil)

backend/sftp/sftp.go

@@ -28,7 +28,7 @@ import (
 	"github.com/ncw/rclone/lib/readers"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
-	"github.com/xanzy/ssh-agent"
+	sshagent "github.com/xanzy/ssh-agent"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/time/rate"
 )
@@ -66,7 +66,22 @@ func init() {
 			IsPassword: true,
 		}, {
 			Name: "key_file",
-			Help: "Path to unencrypted PEM-encoded private key file, leave blank to use ssh-agent.",
+			Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
+		}, {
+			Name: "key_file_pass",
+			Help: `The passphrase to decrypt the PEM-encoded private key file.
+
+Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
+in the new OpenSSH format can't be used.`,
+			IsPassword: true,
+		}, {
+			Name: "key_use_agent",
+			Help: `When set forces the usage of the ssh-agent.
+
+When key-file is also set, the ".pub" file of the specified key-file is read and only the associated key is
+requested from the ssh-agent. This allows to avoid ` + "`Too many authentication failures for *username*`" + ` errors
+when the ssh-agent contains many keys.`,
+			Default: false,
 		}, {
 			Name:    "use_insecure_cipher",
 			Help:    "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
@@ -122,6 +137,8 @@ type Options struct {
 	Port              string `config:"port"`
 	Pass              string `config:"pass"`
 	KeyFile           string `config:"key_file"`
+	KeyFilePass       string `config:"key_file_pass"`
+	KeyUseAgent       bool   `config:"key_use_agent"`
 	UseInsecureCipher bool   `config:"use_insecure_cipher"`
 	DisableHashCheck  bool   `config:"disable_hashcheck"`
 	AskPassword       bool   `config:"ask_password"`
@@ -298,6 +315,18 @@ func (f *Fs) putSftpConnection(pc **conn, err error) {
 	f.poolMu.Unlock()
 }
 
+// shellExpand replaces a leading "~" with "${HOME}" and expands all environment
+// variables afterwards.
+func shellExpand(s string) string {
+	if s != "" {
+		if s[0] == '~' {
+			s = "${HOME}" + s[1:]
+		}
+		s = os.ExpandEnv(s)
+	}
+	return s
+}
+
 // NewFs creates a new Fs object from the name and root. It connects to
 // the host specified in the config file.
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
@@ -325,8 +354,9 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		sshConfig.Config.Ciphers = append(sshConfig.Config.Ciphers, "aes128-cbc")
 	}
 
+	keyFile := shellExpand(opt.KeyFile)
 	// Add ssh agent-auth if no password or file specified
-	if opt.Pass == "" && opt.KeyFile == "" {
+	if (opt.Pass == "" && keyFile == "") || opt.KeyUseAgent {
 		sshAgentClient, _, err := sshagent.New()
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't connect to ssh-agent")
@@ -335,16 +365,46 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't read ssh agent signers")
 		}
-		sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
+		if keyFile != "" {
+			pubBytes, err := ioutil.ReadFile(keyFile + ".pub")
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to read public key file")
+			}
+			pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes)
+			if err != nil {
+				return nil, errors.Wrap(err, "failed to parse public key file")
+			}
+			pubM := pub.Marshal()
+			found := false
+			for _, s := range signers {
+				if bytes.Equal(pubM, s.PublicKey().Marshal()) {
+					sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(s))
+					found = true
+					break
+				}
+			}
+			if !found {
+				return nil, errors.New("private key not found in the ssh-agent")
+			}
+		} else {
+			sshConfig.Auth = append(sshConfig.Auth, ssh.PublicKeys(signers...))
+		}
 	}
 
 	// Load key file if specified
-	if opt.KeyFile != "" {
-		key, err := ioutil.ReadFile(opt.KeyFile)
+	if keyFile != "" {
+		key, err := ioutil.ReadFile(keyFile)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read private key file")
 		}
-		signer, err := ssh.ParsePrivateKey(key)
+		clearpass := ""
+		if opt.KeyFilePass != "" {
+			clearpass, err = obscure.Reveal(opt.KeyFilePass)
+			if err != nil {
+				return nil, err
+			}
+		}
+		signer, err := ssh.ParsePrivateKeyWithPassphrase(key, []byte(clearpass))
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to parse private key file")
 		}
@@ -505,9 +565,13 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 		// If file is a symlink (not a regular file is the best cross platform test we can do), do a stat to
 		// pick up the size and type of the destination, instead of the size and type of the symlink.
 		if !info.Mode().IsRegular() {
+			oldInfo := info
 			info, err = f.stat(remote)
 			if err != nil {
-				return nil, errors.Wrap(err, "stat of non-regular file/dir failed")
+				if !os.IsNotExist(err) {
+					fs.Errorf(remote, "stat of non-regular file/dir failed: %v", err)
+				}
+				info = oldInfo
 			}
 		}
 		if info.IsDir() {
@@ -594,12 +658,22 @@ func (f *Fs) Mkdir(dir string) error {
 
 // Rmdir removes the root directory of the Fs object
 func (f *Fs) Rmdir(dir string) error {
+	// Check to see if directory is empty as some servers will
+	// delete recursively with RemoveDirectory
+	entries, err := f.List(dir)
+	if err != nil {
+		return errors.Wrap(err, "Rmdir")
+	}
+	if len(entries) != 0 {
+		return fs.ErrorDirectoryNotEmpty
+	}
+	// Remove the directory
 	root := path.Join(f.root, dir)
 	c, err := f.getSftpConnection()
 	if err != nil {
 		return errors.Wrap(err, "Rmdir")
 	}
-	err = c.sftpClient.Remove(root)
+	err = c.sftpClient.RemoveDirectory(root)
 	f.putSftpConnection(&c, err)
 	return err
 }

backend/swift/swift.go

@@ -21,6 +21,7 @@ import (
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/operations"
 	"github.com/ncw/rclone/fs/walk"
+	"github.com/ncw/rclone/lib/pacer"
 	"github.com/ncw/swift"
 	"github.com/pkg/errors"
 )
@@ -30,6 +31,7 @@ const (
 	directoryMarkerContentType = "application/directory" // content type of directory marker objects
 	listChunks                 = 1000                    // chunk size to read directory listings
 	defaultChunkSize           = 5 * fs.GibiByte
+	minSleep                   = 10 * time.Millisecond // In case of error, start at 10ms sleep.
 )
 
 // SharedOptions are shared between swift and hubic
@@ -41,6 +43,20 @@ Above this size files will be chunked into a _segments container.  The
 default for this is 5GB which is its maximum value.`,
 	Default:  defaultChunkSize,
 	Advanced: true,
+}, {
+	Name: "no_chunk",
+	Help: `Don't chunk files during streaming upload.
+
+When doing streaming uploads (eg using rcat or mount) setting this
+flag will cause the swift backend to not upload chunked files.
+
+This will limit the maximum upload size to 5GB. However non chunked
+files are easier to deal with and have an MD5SUM.
+
+Rclone will still chunk files bigger than chunk_size when doing normal
+copy operations.`,
+	Default:  false,
+	Advanced: true,
 }}
 
 // Register with Fs
@@ -114,6 +130,15 @@ func init() {
 		}, {
 			Name: "auth_token",
 			Help: "Auth Token from alternate authentication - optional (OS_AUTH_TOKEN)",
+		}, {
+			Name: "application_credential_id",
+			Help: "Application Credential ID (OS_APPLICATION_CREDENTIAL_ID)",
+		}, {
+			Name: "application_credential_name",
+			Help: "Application Credential Name (OS_APPLICATION_CREDENTIAL_NAME)",
+		}, {
+			Name: "application_credential_secret",
+			Help: "Application Credential Secret (OS_APPLICATION_CREDENTIAL_SECRET)",
 		}, {
 			Name:    "auth_version",
 			Help:    "AuthVersion - optional - set to (1,2,3) if your auth URL has no version (ST_AUTH_VERSION)",
@@ -157,22 +182,26 @@ provider.`,
 
 // Options defines the configuration for this backend
 type Options struct {
-	EnvAuth       bool          `config:"env_auth"`
-	User          string        `config:"user"`
-	Key           string        `config:"key"`
-	Auth          string        `config:"auth"`
-	UserID        string        `config:"user_id"`
-	Domain        string        `config:"domain"`
-	Tenant        string        `config:"tenant"`
-	TenantID      string        `config:"tenant_id"`
-	TenantDomain  string        `config:"tenant_domain"`
-	Region        string        `config:"region"`
-	StorageURL    string        `config:"storage_url"`
-	AuthToken     string        `config:"auth_token"`
-	AuthVersion   int           `config:"auth_version"`
-	StoragePolicy string        `config:"storage_policy"`
-	EndpointType  string        `config:"endpoint_type"`
-	ChunkSize     fs.SizeSuffix `config:"chunk_size"`
+	EnvAuth                     bool          `config:"env_auth"`
+	User                        string        `config:"user"`
+	Key                         string        `config:"key"`
+	Auth                        string        `config:"auth"`
+	UserID                      string        `config:"user_id"`
+	Domain                      string        `config:"domain"`
+	Tenant                      string        `config:"tenant"`
+	TenantID                    string        `config:"tenant_id"`
+	TenantDomain                string        `config:"tenant_domain"`
+	Region                      string        `config:"region"`
+	StorageURL                  string        `config:"storage_url"`
+	AuthToken                   string        `config:"auth_token"`
+	AuthVersion                 int           `config:"auth_version"`
+	ApplicationCredentialID     string        `config:"application_credential_id"`
+	ApplicationCredentialName   string        `config:"application_credential_name"`
+	ApplicationCredentialSecret string        `config:"application_credential_secret"`
+	StoragePolicy               string        `config:"storage_policy"`
+	EndpointType                string        `config:"endpoint_type"`
+	ChunkSize                   fs.SizeSuffix `config:"chunk_size"`
+	NoChunk                     bool          `config:"no_chunk"`
 }
 
 // Fs represents a remote swift server
@@ -187,16 +216,20 @@ type Fs struct {
 	containerOK       bool              // true if we have created the container
 	segmentsContainer string            // container to store the segments (if any) in
 	noCheckContainer  bool              // don't check the container before creating it
+	pacer             *pacer.Pacer      // To pace the API calls
 }
 
 // Object describes a swift object
 //
 // Will definitely have info but maybe not meta
 type Object struct {
-	fs      *Fs           // what this object is part of
-	remote  string        // The remote path
-	info    swift.Object  // Info from the swift object if known
-	headers swift.Headers // The object headers if known
+	fs           *Fs    // what this object is part of
+	remote       string // The remote path
+	size         int64
+	lastModified time.Time
+	contentType  string
+	md5          string
+	headers      swift.Headers // The object headers if known
 }
 
 // ------------------------------------------------------------
@@ -227,6 +260,32 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
+// retryErrorCodes is a slice of error codes that we will retry
+var retryErrorCodes = []int{
+	401, // Unauthorized (eg "Token has expired")
+	408, // Request Timeout
+	409, // Conflict - various states that could be resolved on a retry
+	429, // Rate exceeded.
+	500, // Get occasional 500 Internal Server Error
+	503, // Service Unavailable/Slow Down - "Reduce your request rate"
+	504, // Gateway Time-out
+}
+
+// shouldRetry returns a boolean as to whether this err deserves to be
+// retried.  It returns the err as a convenience
+func shouldRetry(err error) (bool, error) {
+	// If this is an swift.Error object extract the HTTP error code
+	if swiftError, ok := err.(*swift.Error); ok {
+		for _, e := range retryErrorCodes {
+			if swiftError.StatusCode == e {
+				return true, err
+			}
+		}
+	}
+	// Check for generic failure conditions
+	return fserrors.ShouldRetry(err), err
+}
+
 // Pattern to match a swift path
 var matcher = regexp.MustCompile(`^/*([^/]*)(.*)$`)
 
@@ -246,22 +305,25 @@ func parsePath(path string) (container, directory string, err error) {
 func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	c := &swift.Connection{
 		// Keep these in the same order as the Config for ease of checking
-		UserName:       opt.User,
-		ApiKey:         opt.Key,
-		AuthUrl:        opt.Auth,
-		UserId:         opt.UserID,
-		Domain:         opt.Domain,
-		Tenant:         opt.Tenant,
-		TenantId:       opt.TenantID,
-		TenantDomain:   opt.TenantDomain,
-		Region:         opt.Region,
-		StorageUrl:     opt.StorageURL,
-		AuthToken:      opt.AuthToken,
-		AuthVersion:    opt.AuthVersion,
-		EndpointType:   swift.EndpointType(opt.EndpointType),
-		ConnectTimeout: 10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
-		Timeout:        10 * fs.Config.Timeout,        // Use the timeouts in the transport
-		Transport:      fshttp.NewTransport(fs.Config),
+		UserName:                    opt.User,
+		ApiKey:                      opt.Key,
+		AuthUrl:                     opt.Auth,
+		UserId:                      opt.UserID,
+		Domain:                      opt.Domain,
+		Tenant:                      opt.Tenant,
+		TenantId:                    opt.TenantID,
+		TenantDomain:                opt.TenantDomain,
+		Region:                      opt.Region,
+		StorageUrl:                  opt.StorageURL,
+		AuthToken:                   opt.AuthToken,
+		AuthVersion:                 opt.AuthVersion,
+		ApplicationCredentialId:     opt.ApplicationCredentialID,
+		ApplicationCredentialName:   opt.ApplicationCredentialName,
+		ApplicationCredentialSecret: opt.ApplicationCredentialSecret,
+		EndpointType:                swift.EndpointType(opt.EndpointType),
+		ConnectTimeout:              10 * fs.Config.ConnectTimeout, // Use the timeouts in the transport
+		Timeout:                     10 * fs.Config.Timeout,        // Use the timeouts in the transport
+		Transport:                   fshttp.NewTransport(fs.Config),
 	}
 	if opt.EnvAuth {
 		err := c.ApplyEnvironment()
@@ -271,11 +333,13 @@ func swiftConnection(opt *Options, name string) (*swift.Connection, error) {
 	}
 	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
 	if !c.Authenticated() {
-		if c.UserName == "" && c.UserId == "" {
-			return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
-		}
-		if c.ApiKey == "" {
-			return nil, errors.New("key not found")
+		if (c.ApplicationCredentialId != "" || c.ApplicationCredentialName != "") && c.ApplicationCredentialSecret == "" {
+			if c.UserName == "" && c.UserId == "" {
+				return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
+			}
+			if c.ApiKey == "" {
+				return nil, errors.New("key not found")
+			}
 		}
 		if c.AuthUrl == "" {
 			return nil, errors.New("auth not found")
@@ -337,6 +401,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 		segmentsContainer: container + "_segments",
 		root:              directory,
 		noCheckContainer:  noCheckContainer,
+		pacer:             pacer.New().SetMinSleep(minSleep).SetPacer(pacer.S3Pacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -346,7 +411,11 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the object exists - ignoring directory markers
-		info, _, err := f.c.Object(container, directory)
+		var info swift.Object
+		err = f.pacer.Call(func() (bool, error) {
+			info, _, err = f.c.Object(container, directory)
+			return shouldRetry(err)
+		})
 		if err == nil && info.ContentType != directoryMarkerContentType {
 			f.root = path.Dir(directory)
 			if f.root == "." {
@@ -361,7 +430,7 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
 	return f, nil
 }
 
-// NewFs contstructs an Fs from the path, container:path
+// NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	// Parse config into Options struct
 	opt := new(Options)
@@ -398,7 +467,10 @@ func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, er
 	}
 	if info != nil {
 		// Set info but not headers
-		o.info = *info
+		err := o.decodeMetaData(info)
+		if err != nil {
+			return nil, err
+		}
 	} else {
 		err := o.readMetaData() // reads info and headers, returning an error
 		if err != nil {
@@ -436,7 +508,12 @@ func (f *Fs) listContainerRoot(container, root string, dir string, recurse bool,
 	}
 	rootLength := len(root)
 	return f.c.ObjectsWalk(container, &opts, func(opts *swift.ObjectsOpts) (interface{}, error) {
-		objects, err := f.c.Objects(container, opts)
+		var objects []swift.Object
+		var err error
+		err = f.pacer.Call(func() (bool, error) {
+			objects, err = f.c.Objects(container, opts)
+			return shouldRetry(err)
+		})
 		if err == nil {
 			for i := range objects {
 				object := &objects[i]
@@ -525,7 +602,11 @@ func (f *Fs) listContainers(dir string) (entries fs.DirEntries, err error) {
 	if dir != "" {
 		return nil, fs.ErrorListBucketRequired
 	}
-	containers, err := f.c.ContainersAll(nil)
+	var containers []swift.Container
+	err = f.pacer.Call(func() (bool, error) {
+		containers, err = f.c.ContainersAll(nil)
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, errors.Wrap(err, "container listing failed")
 	}
@@ -586,7 +667,12 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
-	containers, err := f.c.ContainersAll(nil)
+	var containers []swift.Container
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		containers, err = f.c.ContainersAll(nil)
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, errors.Wrap(err, "container listing failed")
 	}
@@ -636,14 +722,20 @@ func (f *Fs) Mkdir(dir string) error {
 	// Check to see if container exists first
 	var err error = swift.ContainerNotFound
 	if !f.noCheckContainer {
-		_, _, err = f.c.Container(f.container)
+		err = f.pacer.Call(func() (bool, error) {
+			_, _, err = f.c.Container(f.container)
+			return shouldRetry(err)
+		})
 	}
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
 		if f.opt.StoragePolicy != "" {
 			headers["X-Storage-Policy"] = f.opt.StoragePolicy
 		}
-		err = f.c.ContainerCreate(f.container, headers)
+		err = f.pacer.Call(func() (bool, error) {
+			err = f.c.ContainerCreate(f.container, headers)
+			return shouldRetry(err)
+		})
 	}
 	if err == nil {
 		f.containerOK = true
@@ -660,7 +752,11 @@ func (f *Fs) Rmdir(dir string) error {
 	if f.root != "" || dir != "" {
 		return nil
 	}
-	err := f.c.ContainerDelete(f.container)
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		err = f.c.ContainerDelete(f.container)
+		return shouldRetry(err)
+	})
 	if err == nil {
 		f.containerOK = false
 	}
@@ -719,7 +815,10 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 		return nil, fs.ErrorCantCopy
 	}
 	srcFs := srcObj.fs
-	_, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.c.ObjectCopy(srcFs.container, srcFs.root+srcObj.remote, f.container, f.root+remote, nil)
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -768,7 +867,7 @@ func (o *Object) Hash(t hash.Type) (string, error) {
 		fs.Debugf(o, "Returning empty Md5sum for swift large object")
 		return "", nil
 	}
-	return strings.ToLower(o.info.Hash), nil
+	return strings.ToLower(o.md5), nil
 }
 
 // hasHeader checks for the header passed in returning false if the
@@ -797,7 +896,22 @@ func (o *Object) isStaticLargeObject() (bool, error) {
 
 // Size returns the size of an object in bytes
 func (o *Object) Size() int64 {
-	return o.info.Bytes
+	return o.size
+}
+
+// decodeMetaData sets the metadata in the object from a swift.Object
+//
+// Sets
+//  o.lastModified
+//  o.size
+//  o.md5
+//  o.contentType
+func (o *Object) decodeMetaData(info *swift.Object) (err error) {
+	o.lastModified = info.LastModified
+	o.size = info.Bytes
+	o.md5 = info.Hash
+	o.contentType = info.ContentType
+	return nil
 }
 
 // readMetaData gets the metadata if it hasn't already been fetched
@@ -809,15 +923,23 @@ func (o *Object) readMetaData() (err error) {
 	if o.headers != nil {
 		return nil
 	}
-	info, h, err := o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
+	var info swift.Object
+	var h swift.Headers
+	err = o.fs.pacer.Call(func() (bool, error) {
+		info, h, err = o.fs.c.Object(o.fs.container, o.fs.root+o.remote)
+		return shouldRetry(err)
+	})
 	if err != nil {
 		if err == swift.ObjectNotFound {
 			return fs.ErrorObjectNotFound
 		}
 		return err
 	}
-	o.info = info
 	o.headers = h
+	err = o.decodeMetaData(&info)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -828,17 +950,17 @@ func (o *Object) readMetaData() (err error) {
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
 	if fs.Config.UseServerModTime {
-		return o.info.LastModified
+		return o.lastModified
 	}
 	err := o.readMetaData()
 	if err != nil {
 		fs.Debugf(o, "Failed to read metadata: %s", err)
-		return o.info.LastModified
+		return o.lastModified
 	}
 	modTime, err := o.headers.ObjectMetadata().GetModTime()
 	if err != nil {
 		// fs.Logf(o, "Failed to read mtime from object: %v", err)
-		return o.info.LastModified
+		return o.lastModified
 	}
 	return modTime
 }
@@ -861,7 +983,10 @@ func (o *Object) SetModTime(modTime time.Time) error {
 			newHeaders[k] = v
 		}
 	}
-	return o.fs.c.ObjectUpdate(o.fs.container, o.fs.root+o.remote, newHeaders)
+	return o.fs.pacer.Call(func() (bool, error) {
+		err = o.fs.c.ObjectUpdate(o.fs.container, o.fs.root+o.remote, newHeaders)
+		return shouldRetry(err)
+	})
 }
 
 // Storable returns if this object is storable
@@ -869,14 +994,17 @@ func (o *Object) SetModTime(modTime time.Time) error {
 // It compares the Content-Type to directoryMarkerContentType - that
 // makes it a directory marker which is not storable.
 func (o *Object) Storable() bool {
-	return o.info.ContentType != directoryMarkerContentType
+	return o.contentType != directoryMarkerContentType
 }
 
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	headers := fs.OpenOptionHeaders(options)
 	_, isRanging := headers["Range"]
-	in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		in, _, err = o.fs.c.ObjectOpen(o.fs.container, o.fs.root+o.remote, !isRanging, headers)
+		return shouldRetry(err)
+	})
 	return
 }
 
@@ -903,13 +1031,20 @@ func (o *Object) removeSegments(except string) error {
 		}
 		segmentPath := segmentsRoot + remote
 		fs.Debugf(o, "Removing segment file %q in container %q", segmentPath, o.fs.segmentsContainer)
-		return o.fs.c.ObjectDelete(o.fs.segmentsContainer, segmentPath)
+		var err error
+		return o.fs.pacer.Call(func() (bool, error) {
+			err = o.fs.c.ObjectDelete(o.fs.segmentsContainer, segmentPath)
+			return shouldRetry(err)
+		})
 	})
 	if err != nil {
 		return err
 	}
 	// remove the segments container if empty, ignore errors
-	err = o.fs.c.ContainerDelete(o.fs.segmentsContainer)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		err = o.fs.c.ContainerDelete(o.fs.segmentsContainer)
+		return shouldRetry(err)
+	})
 	if err == nil {
 		fs.Debugf(o, "Removed empty container %q", o.fs.segmentsContainer)
 	}
@@ -938,13 +1073,19 @@ func urlEncode(str string) string {
 func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64, contentType string) (string, error) {
 	// Create the segmentsContainer if it doesn't exist
 	var err error
-	_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
+		return shouldRetry(err)
+	})
 	if err == swift.ContainerNotFound {
 		headers := swift.Headers{}
 		if o.fs.opt.StoragePolicy != "" {
 			headers["X-Storage-Policy"] = o.fs.opt.StoragePolicy
 		}
-		err = o.fs.c.ContainerCreate(o.fs.segmentsContainer, headers)
+		err = o.fs.pacer.Call(func() (bool, error) {
+			err = o.fs.c.ContainerCreate(o.fs.segmentsContainer, headers)
+			return shouldRetry(err)
+		})
 	}
 	if err != nil {
 		return "", err
@@ -973,7 +1114,10 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 		segmentReader := io.LimitReader(in, n)
 		segmentPath := fmt.Sprintf("%s/%08d", segmentsPath, i)
 		fs.Debugf(o, "Uploading segment file %q into %q", segmentPath, o.fs.segmentsContainer)
-		_, err := o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
+		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+			_, err = o.fs.c.ObjectPut(o.fs.segmentsContainer, segmentPath, segmentReader, true, "", "", headers)
+			return shouldRetry(err)
+		})
 		if err != nil {
 			return "", err
 		}
@@ -984,7 +1128,10 @@ func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64,
 	headers["Content-Length"] = "0" // set Content-Length as we know it
 	emptyReader := bytes.NewReader(nil)
 	manifestName := o.fs.root + o.remote
-	_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		_, err = o.fs.c.ObjectPut(o.fs.container, manifestName, emptyReader, true, "", contentType, headers)
+		return shouldRetry(err)
+	})
 	return uniquePrefix + "/", err
 }
 
@@ -1014,17 +1161,31 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	contentType := fs.MimeType(src)
 	headers := m.ObjectHeaders()
 	uniquePrefix := ""
-	if size > int64(o.fs.opt.ChunkSize) || size == -1 {
+	if size > int64(o.fs.opt.ChunkSize) || (size == -1 && !o.fs.opt.NoChunk) {
 		uniquePrefix, err = o.updateChunks(in, headers, size, contentType)
 		if err != nil {
 			return err
 		}
+		o.headers = nil // wipe old metadata
 	} else {
-		headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length as we know it
-		_, err := o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
+		if size >= 0 {
+			headers["Content-Length"] = strconv.FormatInt(size, 10) // set Content-Length if we know it
+		}
+		var rxHeaders swift.Headers
+		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+			rxHeaders, err = o.fs.c.ObjectPut(o.fs.container, o.fs.root+o.remote, in, true, "", contentType, headers)
+			return shouldRetry(err)
+		})
 		if err != nil {
 			return err
 		}
+		// set Metadata since ObjectPut checked the hash and length so we know the
+		// object has been safely uploaded
+		o.lastModified = modTime
+		o.size = size
+		o.md5 = rxHeaders["ETag"]
+		o.contentType = contentType
+		o.headers = headers
 	}
 
 	// If file was a dynamic large object then remove old/all segments
@@ -1035,8 +1196,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		}
 	}
 
-	// Read the metadata from the newly created object
-	o.headers = nil // wipe old metadata
+	// Read the metadata from the newly created object if necessary
 	return o.readMetaData()
 }
 
@@ -1047,7 +1207,10 @@ func (o *Object) Remove() error {
 		return err
 	}
 	// Remove file/manifest first
-	err = o.fs.c.ObjectDelete(o.fs.container, o.fs.root+o.remote)
+	err = o.fs.pacer.Call(func() (bool, error) {
+		err = o.fs.c.ObjectDelete(o.fs.container, o.fs.root+o.remote)
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return err
 	}
@@ -1063,7 +1226,7 @@ func (o *Object) Remove() error {
 
 // MimeType of an Object if known, "" otherwise
 func (o *Object) MimeType() string {
-	return o.info.ContentType
+	return o.contentType
 }
 
 // Check the interfaces are satisfied

backend/union/union.go

@@ -177,8 +177,8 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 // At least one value will be written to the channel,
 // specifying the initial value and updated values might
 // follow. A 0 Duration should pause the polling.
-// The ChangeNotify implemantion must empty the channel
-// regulary. When the channel gets closed, the implemantion
+// The ChangeNotify implementation must empty the channel
+// regularly. When the channel gets closed, the implementation
 // should stop polling and release resources.
 func (f *Fs) ChangeNotify(fn func(string, fs.EntryType), ch <-chan time.Duration) {
 	var remoteChans []chan time.Duration
@@ -376,6 +376,11 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	}).Fill(f)
 	features = features.Mask(f.wr) // mask the features just on the writable fs
 
+	// Really need the union of all remotes for these, so
+	// re-instate and calculate separately.
+	features.ChangeNotify = f.ChangeNotify
+	features.DirCacheFlush = f.DirCacheFlush
+
 	// FIXME maybe should be masking the bools here?
 
 	// Clear ChangeNotify and DirCacheFlush if all are nil

backend/webdav/api/types.go

@@ -6,7 +6,11 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/hash"
 )
 
 const (
@@ -62,11 +66,13 @@ type Response struct {
 // Note that status collects all the status values for which we just
 // check the first is OK.
 type Prop struct {
-	Status   []string  `xml:"DAV: status"`
-	Name     string    `xml:"DAV: prop>displayname,omitempty"`
-	Type     *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
-	Size     int64     `xml:"DAV: prop>getcontentlength,omitempty"`
-	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
+	Status       []string  `xml:"DAV: status"`
+	Name         string    `xml:"DAV: prop>displayname,omitempty"`
+	Type         *xml.Name `xml:"DAV: prop>resourcetype>collection,omitempty"`
+	IsCollection *int      `xml:"DAV: prop>iscollection,omitempty"` // this is a Microsoft extension see #2716
+	Size         int64     `xml:"DAV: prop>getcontentlength,omitempty"`
+	Modified     Time      `xml:"DAV: prop>getlastmodified,omitempty"`
+	Checksums    []string  `xml:"prop>checksums>checksum,omitempty"`
 }
 
 // Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
@@ -92,13 +98,33 @@ func (p *Prop) StatusOK() bool {
 	return false
 }
 
+// Hashes returns a map of all checksums - may be nil
+func (p *Prop) Hashes() (hashes map[hash.Type]string) {
+	if len(p.Checksums) == 0 {
+		return nil
+	}
+	hashes = make(map[hash.Type]string)
+	for _, checksums := range p.Checksums {
+		checksums = strings.ToLower(checksums)
+		for _, checksum := range strings.Split(checksums, " ") {
+			switch {
+			case strings.HasPrefix(checksum, "sha1:"):
+				hashes[hash.SHA1] = checksum[5:]
+			case strings.HasPrefix(checksum, "md5:"):
+				hashes[hash.MD5] = checksum[4:]
+			}
+		}
+	}
+	return hashes
+}
+
 // PropValue is a tagged name and value
 type PropValue struct {
 	XMLName xml.Name `xml:""`
 	Value   string   `xml:",chardata"`
 }
 
-// Error is used to desribe webdav errors
+// Error is used to describe webdav errors
 //
 // <d:error xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns">
 //   <s:exception>Sabre\DAV\Exception\NotFound</s:exception>
@@ -111,7 +137,7 @@ type Error struct {
 	StatusCode int
 }
 
-// Error returns a string for the error and statistifes the error interface
+// Error returns a string for the error and satisfies the error interface
 func (e *Error) Error() string {
 	var out []string
 	if e.Message != "" {
@@ -148,6 +174,8 @@ var timeFormats = []string{
 	time.RFC3339,   // Wed, 31 Oct 2018 13:57:11 CET (as used by komfortcloud.de)
 }
 
+var oneTimeError sync.Once
+
 // UnmarshalXML turns XML into a Time
 func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var v string
@@ -171,5 +199,33 @@ func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 			break
 		}
 	}
+	if err != nil {
+		oneTimeError.Do(func() {
+			fs.Errorf(nil, "Failed to parse time %q - using the epoch", v)
+		})
+		// Return the epoch instead
+		*t = Time(time.Unix(0, 0))
+		// ignore error
+		err = nil
+	}
 	return err
 }
+
+// Quota is used to read the bytes used and available
+//
+// <d:multistatus xmlns:d="DAV:" xmlns:s="http://sabredav.org/ns" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
+//  <d:response>
+//   <d:href>/remote.php/webdav/</d:href>
+//   <d:propstat>
+//    <d:prop>
+//     <d:quota-available-bytes>-3</d:quota-available-bytes>
+//     <d:quota-used-bytes>376461895</d:quota-used-bytes>
+//    </d:prop>
+//    <d:status>HTTP/1.1 200 OK</d:status>
+//   </d:propstat>
+//  </d:response>
+// </d:multistatus>
+type Quota struct {
+	Available int64 `xml:"DAV: response>propstat>prop>quota-available-bytes"`
+	Used      int64 `xml:"DAV: response>propstat>prop>quota-used-bytes"`
+}

backend/webdav/odrvcookie/fetch.go

@@ -102,7 +102,7 @@ func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
 func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
 	spRoot, err := url.Parse(ca.endpoint)
 	if err != nil {
-		return nil, errors.Wrap(err, "Error while contructing endpoint URL")
+		return nil, errors.Wrap(err, "Error while constructing endpoint URL")
 	}
 
 	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
@@ -121,7 +121,7 @@ func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error
 		Jar: jar,
 	}
 
-	// Send the previously aquired Token as a Post parameter
+	// Send the previously acquired Token as a Post parameter
 	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
 		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
 	}

backend/webdav/odrvcookie/renew.go

@@ -2,13 +2,10 @@ package odrvcookie
 
 import (
 	"time"
-
-	"github.com/ncw/rclone/lib/rest"
 )
 
 // CookieRenew holds information for the renew
 type CookieRenew struct {
-	srv     *rest.Client
 	timer   *time.Ticker
 	renewFn func()
 }

backend/webdav/webdav.go

@@ -2,23 +2,13 @@
 // object storage system.
 package webdav
 
-// Owncloud: Getting Oc-Checksum:
-// SHA1:f572d396fae9206628714fb2ce00f72e94f2258f on HEAD but not on
-// nextcloud?
-
-// docs for file webdav
-// https://docs.nextcloud.com/server/12/developer_manual/client_apis/WebDAV/index.html
-
-// indicates checksums can be set as metadata here
-// https://github.com/nextcloud/server/issues/6129
-// owncloud seems to have checksums as metadata though - can read them
-
 // SetModTime might be possible
 // https://stackoverflow.com/questions/3579608/webdav-can-a-client-modify-the-mtime-of-a-file
 // ...support for a PROPSET to lastmodified (mind the missing get) which does the utime() call might be an option.
 // For example the ownCloud WebDAV server does it that way.
 
 import (
+	"bytes"
 	"encoding/xml"
 	"fmt"
 	"io"
@@ -31,7 +21,6 @@ import (
 	"github.com/ncw/rclone/backend/webdav/api"
 	"github.com/ncw/rclone/backend/webdav/odrvcookie"
 	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/configmap"
 	"github.com/ncw/rclone/fs/config/configstruct"
 	"github.com/ncw/rclone/fs/config/obscure"
@@ -96,10 +85,11 @@ func init() {
 
 // Options defines the configuration for this backend
 type Options struct {
-	URL    string `config:"url"`
-	Vendor string `config:"vendor"`
-	User   string `config:"user"`
-	Pass   string `config:"pass"`
+	URL         string `config:"url"`
+	Vendor      string `config:"vendor"`
+	User        string `config:"user"`
+	Pass        string `config:"pass"`
+	BearerToken string `config:"bearer_token"`
 }
 
 // Fs represents a remote webdav
@@ -116,6 +106,7 @@ type Fs struct {
 	canStream          bool          // set if can stream
 	useOCMtime         bool          // set if can use X-OC-Mtime
 	retryWithZeroDepth bool          // some vendors (sharepoint) won't list files when Depth is 1 (our default)
+	hasChecksums       bool          // set if can use owncloud style checksums
 }
 
 // Object describes a webdav object
@@ -127,7 +118,8 @@ type Object struct {
 	hasMetaData bool      // whether info below has been set
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
-	sha1        string    // SHA-1 of the object content
+	sha1        string    // SHA-1 of the object content if known
+	md5         string    // MD5 of the object content if known
 }
 
 // ------------------------------------------------------------
@@ -180,6 +172,11 @@ func itemIsDir(item *api.Response) bool {
 		}
 		fs.Debugf(nil, "Unknown resource type %q/%q on %q", t.Space, t.Local, item.Props.Name)
 	}
+	// the iscollection prop is a Microsoft extension, but if present it is a reliable indicator
+	// if the above check failed - see #2716
+	if t := item.Props.IsCollection; t != nil {
+		return *t != 0
+	}
 	return false
 }
 
@@ -194,6 +191,9 @@ func (f *Fs) readMetaDataForPath(path string, depth string) (info *api.Prop, err
 		},
 		NoRedirect: true,
 	}
+	if f.hasChecksums {
+		opts.Body = bytes.NewBuffer(owncloudProps)
+	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -249,7 +249,7 @@ func errorHandler(resp *http.Response) error {
 	return errResponse
 }
 
-// addShlash makes sure s is terminated with a / if non empty
+// addSlash makes sure s is terminated with a / if non empty
 func addSlash(s string) string {
 	if s != "" && !strings.HasSuffix(s, "/") {
 		s += "/"
@@ -283,9 +283,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	rootIsDir := strings.HasSuffix(root, "/")
 	root = strings.Trim(root, "/")
 
-	user := config.FileGet(name, "user")
-	pass := config.FileGet(name, "pass")
-	bearerToken := config.FileGet(name, "bearer_token")
 	if !strings.HasSuffix(opt.URL, "/") {
 		opt.URL += "/"
 	}
@@ -320,10 +317,10 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(f)
-	if user != "" || pass != "" {
+	if opt.User != "" || opt.Pass != "" {
 		f.srv.SetUserPass(opt.User, opt.Pass)
-	} else if bearerToken != "" {
-		f.srv.SetHeader("Authorization", "BEARER "+bearerToken)
+	} else if opt.BearerToken != "" {
+		f.srv.SetHeader("Authorization", "BEARER "+opt.BearerToken)
 	}
 	f.srv.SetErrorHandler(errorHandler)
 	err = f.setQuirks(opt.Vendor)
@@ -360,9 +357,11 @@ func (f *Fs) setQuirks(vendor string) error {
 		f.canStream = true
 		f.precision = time.Second
 		f.useOCMtime = true
+		f.hasChecksums = true
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
+		f.hasChecksums = true
 	case "sharepoint":
 		// To mount sharepoint, two Cookies are required
 		// They have to be set instead of BasicAuth
@@ -429,6 +428,22 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 	return f.newObjectWithInfo(remote, nil)
 }
 
+// Read the normal props, plus the checksums
+//
+// <oc:checksums><oc:checksum>SHA1:f572d396fae9206628714fb2ce00f72e94f2258f MD5:b1946ac92492d2347c6235b4d2611184 ADLER32:084b021f</oc:checksum></oc:checksums>
+var owncloudProps = []byte(`<?xml version="1.0"?>
+<d:propfind  xmlns:d="DAV:" xmlns:oc="http://owncloud.org/ns" xmlns:nc="http://nextcloud.org/ns">
+ <d:prop>
+  <d:displayname />
+  <d:getlastmodified />
+  <d:getcontentlength />
+  <d:resourcetype />
+  <d:getcontenttype />
+  <oc:checksums />
+ </d:prop>
+</d:propfind>
+`)
+
 // list the objects into the function supplied
 //
 // If directories is set it only sends directories
@@ -448,6 +463,9 @@ func (f *Fs) listAll(dir string, directoriesOnly bool, filesOnly bool, depth str
 			"Depth": depth,
 		},
 	}
+	if f.hasChecksums {
+		opts.Body = bytes.NewBuffer(owncloudProps)
+	}
 	var result api.Multistatus
 	var resp *http.Response
 	err = f.pacer.Call(func() (bool, error) {
@@ -604,10 +622,9 @@ func (f *Fs) mkParentDir(dirPath string) error {
 	return f.mkdir(parent)
 }
 
-// mkdir makes the directory and parents using native paths
-func (f *Fs) mkdir(dirPath string) error {
-	// defer log.Trace(dirPath, "")("")
-	// We assume the root is already ceated
+// low level mkdir, only makes the directory, doesn't attempt to create parents
+func (f *Fs) _mkdir(dirPath string) error {
+	// We assume the root is already created
 	if dirPath == "" {
 		return nil
 	}
@@ -620,20 +637,27 @@ func (f *Fs) mkdir(dirPath string) error {
 		Path:       dirPath,
 		NoResponse: true,
 	}
-	err := f.pacer.Call(func() (bool, error) {
+	return f.pacer.Call(func() (bool, error) {
 		resp, err := f.srv.Call(&opts)
 		return shouldRetry(resp, err)
 	})
+}
+
+// mkdir makes the directory and parents using native paths
+func (f *Fs) mkdir(dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
+	err := f._mkdir(dirPath)
 	if apiErr, ok := err.(*api.Error); ok {
 		// already exists
-		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
+		// owncloud returns 423/StatusLocked if the create is already in progress
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable || apiErr.StatusCode == http.StatusLocked {
 			return nil
 		}
-		// parent does not exists
+		// parent does not exist
 		if apiErr.StatusCode == http.StatusConflict {
 			err = f.mkParentDir(dirPath)
 			if err == nil {
-				err = f.mkdir(dirPath)
+				err = f._mkdir(dirPath)
 			}
 		}
 	}
@@ -845,9 +869,52 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
+	if f.hasChecksums {
+		return hash.NewHashSet(hash.MD5, hash.SHA1)
+	}
 	return hash.Set(hash.None)
 }
 
+// About gets quota information
+func (f *Fs) About() (*fs.Usage, error) {
+	opts := rest.Opts{
+		Method: "PROPFIND",
+		Path:   "",
+		ExtraHeaders: map[string]string{
+			"Depth": "0",
+		},
+	}
+	opts.Body = bytes.NewBuffer([]byte(`<?xml version="1.0" ?>
+<D:propfind xmlns:D="DAV:">
+ <D:prop>
+  <D:quota-available-bytes/>
+  <D:quota-used-bytes/>
+ </D:prop>
+</D:propfind>
+`))
+	var q = api.Quota{
+		Available: -1,
+		Used:      -1,
+	}
+	var resp *http.Response
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallXML(&opts, nil, &q)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "about call failed")
+	}
+	usage := &fs.Usage{}
+	if q.Available >= 0 && q.Used >= 0 {
+		usage.Total = fs.NewUsageValue(q.Available + q.Used)
+	}
+	if q.Used >= 0 {
+		usage.Used = fs.NewUsageValue(q.Used)
+	}
+	return usage, nil
+}
+
 // ------------------------------------------------------------
 
 // Fs returns the parent Fs
@@ -868,12 +935,17 @@ func (o *Object) Remote() string {
 	return o.remote
 }
 
-// Hash returns the SHA-1 of an object returning a lowercase hex string
+// Hash returns the SHA1 or MD5 of an object returning a lowercase hex string
 func (o *Object) Hash(t hash.Type) (string, error) {
-	if t != hash.SHA1 {
-		return "", hash.ErrUnsupported
+	if o.fs.hasChecksums {
+		switch t {
+		case hash.SHA1:
+			return o.sha1, nil
+		case hash.MD5:
+			return o.md5, nil
+		}
 	}
-	return o.sha1, nil
+	return "", hash.ErrUnsupported
 }
 
 // Size returns the size of an object in bytes
@@ -891,6 +963,11 @@ func (o *Object) setMetaData(info *api.Prop) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 	o.modTime = time.Time(info.Modified)
+	if o.fs.hasChecksums {
+		hashes := info.Hashes()
+		o.sha1 = hashes[hash.SHA1]
+		o.md5 = hashes[hash.MD5]
+	}
 	return nil
 }
 
@@ -970,9 +1047,21 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		ContentLength: &size, // FIXME this isn't necessary with owncloud - See https://github.com/nextcloud/nextcloud-snap/issues/365
 		ContentType:   fs.MimeType(src),
 	}
-	if o.fs.useOCMtime {
-		opts.ExtraHeaders = map[string]string{
-			"X-OC-Mtime": fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9),
+	if o.fs.useOCMtime || o.fs.hasChecksums {
+		opts.ExtraHeaders = map[string]string{}
+		if o.fs.useOCMtime {
+			opts.ExtraHeaders["X-OC-Mtime"] = fmt.Sprintf("%f", float64(src.ModTime().UnixNano())/1E9)
+		}
+		if o.fs.hasChecksums {
+			// Set an upload checksum - prefer SHA1
+			//
+			// This is used as an upload integrity test. If we set
+			// only SHA1 here, owncloud will calculate the MD5 too.
+			if sha1, _ := src.Hash(hash.SHA1); sha1 != "" {
+				opts.ExtraHeaders["OC-Checksum"] = "SHA1:" + sha1
+			} else if md5, _ := src.Hash(hash.MD5); md5 != "" {
+				opts.ExtraHeaders["OC-Checksum"] = "MD5:" + md5
+			}
 		}
 	}
 	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
@@ -1016,5 +1105,6 @@ var (
 	_ fs.Copier      = (*Fs)(nil)
 	_ fs.Mover       = (*Fs)(nil)
 	_ fs.DirMover    = (*Fs)(nil)
+	_ fs.Abouter     = (*Fs)(nil)
 	_ fs.Object      = (*Object)(nil)
 )

backend/yandex/api/api_upload.go

@@ -1,34 +0,0 @@
-package src
-
-//from  yadisk
-
-import (
-	"io"
-	"net/http"
-)
-
-//RootAddr is the base URL for Yandex Disk API.
-const RootAddr = "https://cloud-api.yandex.com" //also https://cloud-api.yandex.net and https://cloud-api.yandex.ru
-
-func (c *Client) setRequestScope(req *http.Request) {
-	req.Header.Add("Accept", "application/json")
-	req.Header.Add("Content-Type", "application/json")
-	req.Header.Add("Authorization", "OAuth "+c.token)
-}
-
-func (c *Client) scopedRequest(method, urlPath string, body io.Reader) (*http.Request, error) {
-	fullURL := RootAddr
-	if urlPath[:1] != "/" {
-		fullURL += "/" + urlPath
-	} else {
-		fullURL += urlPath
-	}
-
-	req, err := http.NewRequest(method, fullURL, body)
-	if err != nil {
-		return req, err
-	}
-
-	c.setRequestScope(req)
-	return req, nil
-}

backend/yandex/api/client.go

@@ -1,133 +0,0 @@
-package src
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/pkg/errors"
-)
-
-//Client struct
-type Client struct {
-	token      string
-	basePath   string
-	HTTPClient *http.Client
-}
-
-//NewClient creates new client
-func NewClient(token string, client ...*http.Client) *Client {
-	return newClientInternal(
-		token,
-		"https://cloud-api.yandex.com/v1/disk", //also "https://cloud-api.yandex.net/v1/disk" "https://cloud-api.yandex.ru/v1/disk"
-		client...)
-}
-
-func newClientInternal(token string, basePath string, client ...*http.Client) *Client {
-	c := &Client{
-		token:    token,
-		basePath: basePath,
-	}
-	if len(client) != 0 {
-		c.HTTPClient = client[0]
-	} else {
-		c.HTTPClient = http.DefaultClient
-	}
-	return c
-}
-
-//ErrorHandler type
-type ErrorHandler func(*http.Response) error
-
-var defaultErrorHandler ErrorHandler = func(resp *http.Response) error {
-	if resp.StatusCode/100 == 5 {
-		return errors.New("server error")
-	}
-
-	if resp.StatusCode/100 == 4 {
-		var response DiskClientError
-		contents, _ := ioutil.ReadAll(resp.Body)
-		err := json.Unmarshal(contents, &response)
-		if err != nil {
-			return err
-		}
-		return response
-	}
-
-	if resp.StatusCode/100 == 3 {
-		return errors.New("redirect error")
-	}
-	return nil
-}
-
-func (HTTPRequest *HTTPRequest) run(client *Client) ([]byte, error) {
-	var err error
-	values := make(url.Values)
-	for k, v := range HTTPRequest.Parameters {
-		values.Set(k, fmt.Sprintf("%v", v))
-	}
-
-	var req *http.Request
-	if HTTPRequest.Method == "POST" {
-		// TODO json serialize
-		req, err = http.NewRequest(
-			"POST",
-			client.basePath+HTTPRequest.Path,
-			strings.NewReader(values.Encode()))
-		if err != nil {
-			return nil, err
-		}
-		// TODO
-		// req.Header.Set("Content-Type", "application/json")
-	} else {
-		req, err = http.NewRequest(
-			HTTPRequest.Method,
-			client.basePath+HTTPRequest.Path+"?"+values.Encode(),
-			nil)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	for headerName := range HTTPRequest.Headers {
-		var headerValues = HTTPRequest.Headers[headerName]
-		for _, headerValue := range headerValues {
-			req.Header.Set(headerName, headerValue)
-		}
-	}
-	return runRequest(client, req)
-}
-
-func runRequest(client *Client, req *http.Request) ([]byte, error) {
-	return runRequestWithErrorHandler(client, req, defaultErrorHandler)
-}
-
-func runRequestWithErrorHandler(client *Client, req *http.Request, errorHandler ErrorHandler) (out []byte, err error) {
-	resp, err := client.HTTPClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer CheckClose(resp.Body, &err)
-
-	return checkResponseForErrorsWithErrorHandler(resp, errorHandler)
-}
-
-func checkResponseForErrorsWithErrorHandler(resp *http.Response, errorHandler ErrorHandler) ([]byte, error) {
-	if resp.StatusCode/100 > 2 {
-		return nil, errorHandler(resp)
-	}
-	return ioutil.ReadAll(resp.Body)
-}
-
-// CheckClose is a utility function used to check the return from
-// Close in a defer statement.
-func CheckClose(c io.Closer, err *error) {
-	cerr := c.Close()
-	if *err == nil {
-		*err = cerr
-	}
-}

backend/yandex/api/custom_property.go

@@ -1,51 +0,0 @@
-package src
-
-import (
-	"bytes"
-	"encoding/json"
-	"io"
-	"net/url"
-)
-
-//CustomPropertyResponse struct we send and is returned by the API for CustomProperty request.
-type CustomPropertyResponse struct {
-	CustomProperties map[string]interface{} `json:"custom_properties"`
-}
-
-//SetCustomProperty will set specified data from Yandex Disk
-func (c *Client) SetCustomProperty(remotePath string, property string, value string) error {
-	rcm := map[string]interface{}{
-		property: value,
-	}
-	cpr := CustomPropertyResponse{rcm}
-	data, _ := json.Marshal(cpr)
-	body := bytes.NewReader(data)
-	err := c.SetCustomPropertyRequest(remotePath, body)
-	if err != nil {
-		return err
-	}
-	return err
-}
-
-//SetCustomPropertyRequest will make an CustomProperty request and return a URL to CustomProperty data to.
-func (c *Client) SetCustomPropertyRequest(remotePath string, body io.Reader) (err error) {
-	values := url.Values{}
-	values.Add("path", remotePath)
-	req, err := c.scopedRequest("PATCH", "/v1/disk/resources?"+values.Encode(), body)
-	if err != nil {
-		return err
-	}
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-	if err := CheckAPIError(resp); err != nil {
-		return err
-	}
-	defer CheckClose(resp.Body, &err)
-
-	//If needed we can read response and check if custom_property is set.
-
-	return nil
-}

backend/yandex/api/delete.go

@@ -1,23 +0,0 @@
-package src
-
-import (
-	"net/url"
-	"strconv"
-)
-
-// Delete will remove specified file/folder from Yandex Disk
-func (c *Client) Delete(remotePath string, permanently bool) error {
-
-	values := url.Values{}
-	values.Add("permanently", strconv.FormatBool(permanently))
-	values.Add("path", remotePath)
-	urlPath := "/v1/disk/resources?" + values.Encode()
-	fullURL := RootAddr
-	if urlPath[:1] != "/" {
-		fullURL += "/" + urlPath
-	} else {
-		fullURL += urlPath
-	}
-
-	return c.PerformDelete(fullURL)
-}

backend/yandex/api/disk_info_request.go

@@ -1,48 +0,0 @@
-package src
-
-import "encoding/json"
-
-//DiskInfoRequest type
-type DiskInfoRequest struct {
-	client      *Client
-	HTTPRequest *HTTPRequest
-}
-
-func (req *DiskInfoRequest) request() *HTTPRequest {
-	return req.HTTPRequest
-}
-
-//DiskInfoResponse struct is returned by the API for DiskInfo request.
-type DiskInfoResponse struct {
-	TrashSize     uint64            `json:"TrashSize"`
-	TotalSpace    uint64            `json:"TotalSpace"`
-	UsedSpace     uint64            `json:"UsedSpace"`
-	SystemFolders map[string]string `json:"SystemFolders"`
-}
-
-//NewDiskInfoRequest create new DiskInfo Request
-func (c *Client) NewDiskInfoRequest() *DiskInfoRequest {
-	return &DiskInfoRequest{
-		client:      c,
-		HTTPRequest: createGetRequest(c, "/", nil),
-	}
-}
-
-//Exec run DiskInfo Request
-func (req *DiskInfoRequest) Exec() (*DiskInfoResponse, error) {
-	data, err := req.request().run(req.client)
-	if err != nil {
-		return nil, err
-	}
-
-	var info DiskInfoResponse
-	err = json.Unmarshal(data, &info)
-	if err != nil {
-		return nil, err
-	}
-	if info.SystemFolders == nil {
-		info.SystemFolders = make(map[string]string)
-	}
-
-	return &info, nil
-}

backend/yandex/api/download.go

@@ -1,66 +0,0 @@
-package src
-
-import (
-	"encoding/json"
-	"io"
-	"net/url"
-)
-
-// DownloadResponse struct is returned by the API for Download request.
-type DownloadResponse struct {
-	HRef      string `json:"href"`
-	Method    string `json:"method"`
-	Templated bool   `json:"templated"`
-}
-
-// Download will get specified data from Yandex.Disk supplying the extra headers
-func (c *Client) Download(remotePath string, headers map[string]string) (io.ReadCloser, error) { //io.Writer
-	ur, err := c.DownloadRequest(remotePath)
-	if err != nil {
-		return nil, err
-	}
-	return c.PerformDownload(ur.HRef, headers)
-}
-
-// DownloadRequest will make an download request and return a URL to download data to.
-func (c *Client) DownloadRequest(remotePath string) (ur *DownloadResponse, err error) {
-	values := url.Values{}
-	values.Add("path", remotePath)
-
-	req, err := c.scopedRequest("GET", "/v1/disk/resources/download?"+values.Encode(), nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	if err := CheckAPIError(resp); err != nil {
-		return nil, err
-	}
-	defer CheckClose(resp.Body, &err)
-
-	ur, err = ParseDownloadResponse(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-
-	return ur, nil
-}
-
-// ParseDownloadResponse tries to read and parse DownloadResponse struct.
-func ParseDownloadResponse(data io.Reader) (*DownloadResponse, error) {
-	dec := json.NewDecoder(data)
-	var ur DownloadResponse
-
-	if err := dec.Decode(&ur); err == io.EOF {
-		// ok
-	} else if err != nil {
-		return nil, err
-	}
-
-	// TODO: check if there is any trash data after JSON and crash if there is.
-
-	return &ur, nil
-}

backend/yandex/api/empty_trash.go

@@ -1,9 +0,0 @@
-package src
-
-// EmptyTrash will permanently delete all trashed files/folders from Yandex Disk
-func (c *Client) EmptyTrash() error {
-	fullURL := RootAddr
-	fullURL += "/v1/disk/trash/resources"
-
-	return c.PerformDelete(fullURL)
-}

backend/yandex/api/error.go

@@ -1,84 +0,0 @@
-package src
-
-//from yadisk
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"net/http"
-)
-
-// ErrorResponse represents erroneous API response.
-// Implements go's built in `error`.
-type ErrorResponse struct {
-	ErrorName   string `json:"error"`
-	Description string `json:"description"`
-	Message     string `json:"message"`
-
-	StatusCode int `json:""`
-}
-
-func (e *ErrorResponse) Error() string {
-	return fmt.Sprintf("[%d - %s] %s (%s)", e.StatusCode, e.ErrorName, e.Description, e.Message)
-}
-
-// ProccessErrorResponse tries to represent data passed as
-// an ErrorResponse object.
-func ProccessErrorResponse(data io.Reader) (*ErrorResponse, error) {
-	dec := json.NewDecoder(data)
-	var errorResponse ErrorResponse
-
-	if err := dec.Decode(&errorResponse); err == io.EOF {
-		// ok
-	} else if err != nil {
-		return nil, err
-	}
-
-	// TODO: check if there is any trash data after JSON and crash if there is.
-
-	return &errorResponse, nil
-}
-
-// CheckAPIError is a convenient function to turn erroneous
-// API response into go error.  It closes the Body on error.
-func CheckAPIError(resp *http.Response) (err error) {
-	if resp.StatusCode >= 200 && resp.StatusCode < 400 {
-		return nil
-	}
-
-	defer CheckClose(resp.Body, &err)
-
-	errorResponse, err := ProccessErrorResponse(resp.Body)
-	if err != nil {
-		return err
-	}
-	errorResponse.StatusCode = resp.StatusCode
-
-	return errorResponse
-}
-
-// ProccessErrorString tries to represent data passed as
-// an ErrorResponse object.
-func ProccessErrorString(data string) (*ErrorResponse, error) {
-	var errorResponse ErrorResponse
-	if err := json.Unmarshal([]byte(data), &errorResponse); err == nil {
-		// ok
-	} else if err != nil {
-		return nil, err
-	}
-
-	// TODO: check if there is any trash data after JSON and crash if there is.
-
-	return &errorResponse, nil
-}
-
-// ParseAPIError Parse json error response from API
-func (c *Client) ParseAPIError(jsonErr string) (string, error) { //ErrorName
-	errorResponse, err := ProccessErrorString(jsonErr)
-	if err != nil {
-		return err.Error(), err
-	}
-
-	return errorResponse.ErrorName, nil
-}

backend/yandex/api/errors.go

@@ -1,14 +0,0 @@
-package src
-
-import "encoding/json"
-
-//DiskClientError struct
-type DiskClientError struct {
-	Description string `json:"Description"`
-	Code        string `json:"Error"`
-}
-
-func (e DiskClientError) Error() string {
-	b, _ := json.Marshal(e)
-	return string(b)
-}

backend/yandex/api/files_resource_list.go

@@ -1,8 +0,0 @@
-package src
-
-// FilesResourceListResponse struct is returned by the API for requests.
-type FilesResourceListResponse struct {
-	Items  []ResourceInfoResponse `json:"items"`
-	Limit  *uint64                `json:"limit"`
-	Offset *uint64                `json:"offset"`
-}

backend/yandex/api/flat_file_list_request.go

@@ -1,78 +0,0 @@
-package src
-
-import (
-	"encoding/json"
-	"strings"
-)
-
-// FlatFileListRequest struct client for FlatFileList Request
-type FlatFileListRequest struct {
-	client      *Client
-	HTTPRequest *HTTPRequest
-}
-
-// FlatFileListRequestOptions struct - options for request
-type FlatFileListRequestOptions struct {
-	MediaType   []MediaType
-	Limit       *uint32
-	Offset      *uint32
-	Fields      []string
-	PreviewSize *PreviewSize
-	PreviewCrop *bool
-}
-
-// Request get request
-func (req *FlatFileListRequest) Request() *HTTPRequest {
-	return req.HTTPRequest
-}
-
-// NewFlatFileListRequest create new FlatFileList Request
-func (c *Client) NewFlatFileListRequest(options ...FlatFileListRequestOptions) *FlatFileListRequest {
-	var parameters = make(map[string]interface{})
-	if len(options) > 0 {
-		opt := options[0]
-		if opt.Limit != nil {
-			parameters["limit"] = *opt.Limit
-		}
-		if opt.Offset != nil {
-			parameters["offset"] = *opt.Offset
-		}
-		if opt.Fields != nil {
-			parameters["fields"] = strings.Join(opt.Fields, ",")
-		}
-		if opt.PreviewSize != nil {
-			parameters["preview_size"] = opt.PreviewSize.String()
-		}
-		if opt.PreviewCrop != nil {
-			parameters["preview_crop"] = *opt.PreviewCrop
-		}
-		if opt.MediaType != nil {
-			var strMediaTypes = make([]string, len(opt.MediaType))
-			for i, t := range opt.MediaType {
-				strMediaTypes[i] = t.String()
-			}
-			parameters["media_type"] = strings.Join(strMediaTypes, ",")
-		}
-	}
-	return &FlatFileListRequest{
-		client:      c,
-		HTTPRequest: createGetRequest(c, "/resources/files", parameters),
-	}
-}
-
-// Exec run FlatFileList Request
-func (req *FlatFileListRequest) Exec() (*FilesResourceListResponse, error) {
-	data, err := req.Request().run(req.client)
-	if err != nil {
-		return nil, err
-	}
-	var info FilesResourceListResponse
-	err = json.Unmarshal(data, &info)
-	if err != nil {
-		return nil, err
-	}
-	if cap(info.Items) == 0 {
-		info.Items = []ResourceInfoResponse{}
-	}
-	return &info, nil
-}

backend/yandex/api/http_request.go

@@ -1,24 +0,0 @@
-package src
-
-// HTTPRequest struct
-type HTTPRequest struct {
-	Method     string
-	Path       string
-	Parameters map[string]interface{}
-	Headers    map[string][]string
-}
-
-func createGetRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest {
-	return createRequest(client, "GET", path, params)
-}
-
-func createRequest(client *Client, method string, path string, parameters map[string]interface{}) *HTTPRequest {
-	var headers = make(map[string][]string)
-	headers["Authorization"] = []string{"OAuth " + client.token}
-	return &HTTPRequest{
-		Method:     method,
-		Path:       path,
-		Parameters: parameters,
-		Headers:    headers,
-	}
-}

backend/yandex/api/last_uploaded_resource_list.go

@@ -1,7 +0,0 @@
-package src
-
-// LastUploadedResourceListResponse struct
-type LastUploadedResourceListResponse struct {
-	Items []ResourceInfoResponse `json:"items"`
-	Limit *uint64                `json:"limit"`
-}

backend/yandex/api/last_uploaded_resource_list_request.go

@@ -1,74 +0,0 @@
-package src
-
-import (
-	"encoding/json"
-	"strings"
-)
-
-// LastUploadedResourceListRequest struct
-type LastUploadedResourceListRequest struct {
-	client      *Client
-	HTTPRequest *HTTPRequest
-}
-
-// LastUploadedResourceListRequestOptions struct
-type LastUploadedResourceListRequestOptions struct {
-	MediaType   []MediaType
-	Limit       *uint32
-	Fields      []string
-	PreviewSize *PreviewSize
-	PreviewCrop *bool
-}
-
-// Request return request
-func (req *LastUploadedResourceListRequest) Request() *HTTPRequest {
-	return req.HTTPRequest
-}
-
-// NewLastUploadedResourceListRequest create new LastUploadedResourceList Request
-func (c *Client) NewLastUploadedResourceListRequest(options ...LastUploadedResourceListRequestOptions) *LastUploadedResourceListRequest {
-	var parameters = make(map[string]interface{})
-	if len(options) > 0 {
-		opt := options[0]
-		if opt.Limit != nil {
-			parameters["limit"] = opt.Limit
-		}
-		if opt.Fields != nil {
-			parameters["fields"] = strings.Join(opt.Fields, ",")
-		}
-		if opt.PreviewSize != nil {
-			parameters["preview_size"] = opt.PreviewSize.String()
-		}
-		if opt.PreviewCrop != nil {
-			parameters["preview_crop"] = opt.PreviewCrop
-		}
-		if opt.MediaType != nil {
-			var strMediaTypes = make([]string, len(opt.MediaType))
-			for i, t := range opt.MediaType {
-				strMediaTypes[i] = t.String()
-			}
-			parameters["media_type"] = strings.Join(strMediaTypes, ",")
-		}
-	}
-	return &LastUploadedResourceListRequest{
-		client:      c,
-		HTTPRequest: createGetRequest(c, "/resources/last-uploaded", parameters),
-	}
-}
-
-// Exec run LastUploadedResourceList Request
-func (req *LastUploadedResourceListRequest) Exec() (*LastUploadedResourceListResponse, error) {
-	data, err := req.Request().run(req.client)
-	if err != nil {
-		return nil, err
-	}
-	var info LastUploadedResourceListResponse
-	err = json.Unmarshal(data, &info)
-	if err != nil {
-		return nil, err
-	}
-	if cap(info.Items) == 0 {
-		info.Items = []ResourceInfoResponse{}
-	}
-	return &info, nil
-}

backend/yandex/api/media_type.go

@@ -1,144 +0,0 @@
-package src
-
-// MediaType struct - media types
-type MediaType struct {
-	mediaType string
-}
-
-// Audio - media type
-func (m *MediaType) Audio() *MediaType {
-	return &MediaType{
-		mediaType: "audio",
-	}
-}
-
-// Backup - media type
-func (m *MediaType) Backup() *MediaType {
-	return &MediaType{
-		mediaType: "backup",
-	}
-}
-
-// Book - media type
-func (m *MediaType) Book() *MediaType {
-	return &MediaType{
-		mediaType: "book",
-	}
-}
-
-// Compressed - media type
-func (m *MediaType) Compressed() *MediaType {
-	return &MediaType{
-		mediaType: "compressed",
-	}
-}
-
-// Data - media type
-func (m *MediaType) Data() *MediaType {
-	return &MediaType{
-		mediaType: "data",
-	}
-}
-
-// Development - media type
-func (m *MediaType) Development() *MediaType {
-	return &MediaType{
-		mediaType: "development",
-	}
-}
-
-// Diskimage - media type
-func (m *MediaType) Diskimage() *MediaType {
-	return &MediaType{
-		mediaType: "diskimage",
-	}
-}
-
-// Document - media type
-func (m *MediaType) Document() *MediaType {
-	return &MediaType{
-		mediaType: "document",
-	}
-}
-
-// Encoded - media type
-func (m *MediaType) Encoded() *MediaType {
-	return &MediaType{
-		mediaType: "encoded",
-	}
-}
-
-// Executable - media type
-func (m *MediaType) Executable() *MediaType {
-	return &MediaType{
-		mediaType: "executable",
-	}
-}
-
-// Flash - media type
-func (m *MediaType) Flash() *MediaType {
-	return &MediaType{
-		mediaType: "flash",
-	}
-}
-
-// Font - media type
-func (m *MediaType) Font() *MediaType {
-	return &MediaType{
-		mediaType: "font",
-	}
-}
-
-// Image - media type
-func (m *MediaType) Image() *MediaType {
-	return &MediaType{
-		mediaType: "image",
-	}
-}
-
-// Settings - media type
-func (m *MediaType) Settings() *MediaType {
-	return &MediaType{
-		mediaType: "settings",
-	}
-}
-
-// Spreadsheet - media type
-func (m *MediaType) Spreadsheet() *MediaType {
-	return &MediaType{
-		mediaType: "spreadsheet",
-	}
-}
-
-// Text - media type
-func (m *MediaType) Text() *MediaType {
-	return &MediaType{
-		mediaType: "text",
-	}
-}
-
-// Unknown - media type
-func (m *MediaType) Unknown() *MediaType {
-	return &MediaType{
-		mediaType: "unknown",
-	}
-}
-
-// Video - media type
-func (m *MediaType) Video() *MediaType {
-	return &MediaType{
-		mediaType: "video",
-	}
-}
-
-// Web - media type
-func (m *MediaType) Web() *MediaType {
-	return &MediaType{
-		mediaType: "web",
-	}
-}
-
-// String - media type
-func (m *MediaType) String() string {
-	return m.mediaType
-}

backend/yandex/api/mkdir.go

@@ -1,21 +0,0 @@
-package src
-
-import (
-	"net/url"
-)
-
-// Mkdir will make specified folder on Yandex Disk
-func (c *Client) Mkdir(remotePath string) (int, string, error) {
-
-	values := url.Values{}
-	values.Add("path", remotePath) // only one current folder will be created. Not all the folders in the path.
-	urlPath := "/v1/disk/resources?" + values.Encode()
-	fullURL := RootAddr
-	if urlPath[:1] != "/" {
-		fullURL += "/" + urlPath
-	} else {
-		fullURL += urlPath
-	}
-
-	return c.PerformMkdir(fullURL)
-}

backend/yandex/api/performdelete.go

@@ -1,35 +0,0 @@
-package src
-
-import (
-	"io/ioutil"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// PerformDelete does the actual delete via DELETE request.
-func (c *Client) PerformDelete(url string) error {
-	req, err := http.NewRequest("DELETE", url, nil)
-	if err != nil {
-		return err
-	}
-
-	//set access token and headers
-	c.setRequestScope(req)
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-
-	//204 - resource deleted.
-	//202 - folder not empty, content will be deleted soon (async delete).
-	if resp.StatusCode != 204 && resp.StatusCode != 202 {
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-		return errors.Errorf("delete error [%d]: %s", resp.StatusCode, string(body))
-	}
-	return nil
-}

backend/yandex/api/performdownload.go

@@ -1,40 +0,0 @@
-package src
-
-import (
-	"io"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// PerformDownload does the actual download via unscoped GET request.
-func (c *Client) PerformDownload(url string, headers map[string]string) (out io.ReadCloser, err error) {
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	// Set any extra headers
-	for k, v := range headers {
-		req.Header.Set(k, v)
-	}
-
-	//c.setRequestScope(req)
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-
-	_, isRanging := req.Header["Range"]
-	if !(resp.StatusCode == http.StatusOK || (isRanging && resp.StatusCode == http.StatusPartialContent)) {
-		defer CheckClose(resp.Body, &err)
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return nil, err
-		}
-		return nil, errors.Errorf("download error [%d]: %s", resp.StatusCode, string(body))
-	}
-	return resp.Body, err
-}

backend/yandex/api/performmkdir.go

@@ -1,34 +0,0 @@
-package src
-
-import (
-	"io/ioutil"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// PerformMkdir does the actual mkdir via PUT request.
-func (c *Client) PerformMkdir(url string) (int, string, error) {
-	req, err := http.NewRequest("PUT", url, nil)
-	if err != nil {
-		return 0, "", err
-	}
-
-	//set access token and headers
-	c.setRequestScope(req)
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return 0, "", err
-	}
-
-	if resp.StatusCode != 201 {
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return 0, "", err
-		}
-		//third parameter is the json error response body
-		return resp.StatusCode, string(body), errors.Errorf("create folder error [%d]: %s", resp.StatusCode, string(body))
-	}
-	return resp.StatusCode, "", nil
-}

backend/yandex/api/performupload.go

@@ -1,38 +0,0 @@
-package src
-
-//from yadisk
-
-import (
-	"io"
-	"io/ioutil"
-	"net/http"
-
-	"github.com/pkg/errors"
-)
-
-// PerformUpload does the actual upload via unscoped PUT request.
-func (c *Client) PerformUpload(url string, data io.Reader, contentType string) (err error) {
-	req, err := http.NewRequest("PUT", url, data)
-	if err != nil {
-		return err
-	}
-	req.Header.Set("Content-Type", contentType)
-
-	//c.setRequestScope(req)
-
-	resp, err := c.HTTPClient.Do(req)
-	if err != nil {
-		return err
-	}
-	defer CheckClose(resp.Body, &err)
-
-	if resp.StatusCode != 201 {
-		body, err := ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return err
-		}
-
-		return errors.Errorf("upload error [%d]: %s", resp.StatusCode, string(body))
-	}
-	return nil
-}

backend/yandex/api/preview_size.go

@@ -1,75 +0,0 @@
-package src
-
-import "fmt"
-
-// PreviewSize struct
-type PreviewSize struct {
-	size string
-}
-
-// PredefinedSizeS - set preview size
-func (s *PreviewSize) PredefinedSizeS() *PreviewSize {
-	return &PreviewSize{
-		size: "S",
-	}
-}
-
-// PredefinedSizeM - set preview size
-func (s *PreviewSize) PredefinedSizeM() *PreviewSize {
-	return &PreviewSize{
-		size: "M",
-	}
-}
-
-// PredefinedSizeL - set preview size
-func (s *PreviewSize) PredefinedSizeL() *PreviewSize {
-	return &PreviewSize{
-		size: "L",
-	}
-}
-
-// PredefinedSizeXL - set preview size
-func (s *PreviewSize) PredefinedSizeXL() *PreviewSize {
-	return &PreviewSize{
-		size: "XL",
-	}
-}
-
-// PredefinedSizeXXL - set preview size
-func (s *PreviewSize) PredefinedSizeXXL() *PreviewSize {
-	return &PreviewSize{
-		size: "XXL",
-	}
-}
-
-// PredefinedSizeXXXL - set preview size
-func (s *PreviewSize) PredefinedSizeXXXL() *PreviewSize {
-	return &PreviewSize{
-		size: "XXXL",
-	}
-}
-
-// ExactWidth - set preview size
-func (s *PreviewSize) ExactWidth(width uint32) *PreviewSize {
-	return &PreviewSize{
-		size: fmt.Sprintf("%dx", width),
-	}
-}
-
-// ExactHeight - set preview size
-func (s *PreviewSize) ExactHeight(height uint32) *PreviewSize {
-	return &PreviewSize{
-		size: fmt.Sprintf("x%d", height),
-	}
-}
-
-// ExactSize - set preview size
-func (s *PreviewSize) ExactSize(width uint32, height uint32) *PreviewSize {
-	return &PreviewSize{
-		size: fmt.Sprintf("%dx%d", width, height),
-	}
-}
-
-func (s *PreviewSize) String() string {
-	return s.size
-}

backend/yandex/api/resource.go

@@ -1,19 +0,0 @@
-package src
-
-//ResourceInfoResponse struct is returned by the API for metedata requests.
-type ResourceInfoResponse struct {
-	PublicKey        string                 `json:"public_key"`
-	Name             string                 `json:"name"`
-	Created          string                 `json:"created"`
-	CustomProperties map[string]interface{} `json:"custom_properties"`
-	Preview          string                 `json:"preview"`
-	PublicURL        string                 `json:"public_url"`
-	OriginPath       string                 `json:"origin_path"`
-	Modified         string                 `json:"modified"`
-	Path             string                 `json:"path"`
-	Md5              string                 `json:"md5"`
-	ResourceType     string                 `json:"type"`
-	MimeType         string                 `json:"mime_type"`
-	Size             uint64                 `json:"size"`
-	Embedded         *ResourceListResponse  `json:"_embedded"`
-}

backend/yandex/api/resource_info_request.go

@@ -1,45 +0,0 @@
-package src
-
-import "encoding/json"
-
-// ResourceInfoRequest struct
-type ResourceInfoRequest struct {
-	client      *Client
-	HTTPRequest *HTTPRequest
-}
-
-// Request of ResourceInfoRequest
-func (req *ResourceInfoRequest) Request() *HTTPRequest {
-	return req.HTTPRequest
-}
-
-// NewResourceInfoRequest create new ResourceInfo Request
-func (c *Client) NewResourceInfoRequest(path string, options ...ResourceInfoRequestOptions) *ResourceInfoRequest {
-	return &ResourceInfoRequest{
-		client:      c,
-		HTTPRequest: createResourceInfoRequest(c, "/resources", path, options...),
-	}
-}
-
-// Exec run ResourceInfo Request
-func (req *ResourceInfoRequest) Exec() (*ResourceInfoResponse, error) {
-	data, err := req.Request().run(req.client)
-	if err != nil {
-		return nil, err
-	}
-
-	var info ResourceInfoResponse
-	err = json.Unmarshal(data, &info)
-	if err != nil {
-		return nil, err
-	}
-	if info.CustomProperties == nil {
-		info.CustomProperties = make(map[string]interface{})
-	}
-	if info.Embedded != nil {
-		if cap(info.Embedded.Items) == 0 {
-			info.Embedded.Items = []ResourceInfoResponse{}
-		}
-	}
-	return &info, nil
-}

backend/yandex/api/resource_info_request_helpers.go

@@ -1,33 +0,0 @@
-package src
-
-import "strings"
-
-func createResourceInfoRequest(c *Client,
-	apiPath string,
-	path string,
-	options ...ResourceInfoRequestOptions) *HTTPRequest {
-	var parameters = make(map[string]interface{})
-	parameters["path"] = path
-	if len(options) > 0 {
-		opt := options[0]
-		if opt.SortMode != nil {
-			parameters["sort"] = opt.SortMode.String()
-		}
-		if opt.Limit != nil {
-			parameters["limit"] = *opt.Limit
-		}
-		if opt.Offset != nil {
-			parameters["offset"] = *opt.Offset
-		}
-		if opt.Fields != nil {
-			parameters["fields"] = strings.Join(opt.Fields, ",")
-		}
-		if opt.PreviewSize != nil {
-			parameters["preview_size"] = opt.PreviewSize.String()
-		}
-		if opt.PreviewCrop != nil {
-			parameters["preview_crop"] = *opt.PreviewCrop
-		}
-	}
-	return createGetRequest(c, apiPath, parameters)
-}

backend/yandex/api/resource_info_request_options.go

@@ -1,11 +0,0 @@
-package src
-
-// ResourceInfoRequestOptions struct
-type ResourceInfoRequestOptions struct {
-	SortMode    *SortMode
-	Limit       *uint32
-	Offset      *uint32
-	Fields      []string
-	PreviewSize *PreviewSize
-	PreviewCrop *bool
-}

backend/yandex/api/resource_list.go

@@ -1,12 +0,0 @@
-package src
-
-// ResourceListResponse struct
-type ResourceListResponse struct {
-	Sort      *SortMode              `json:"sort"`
-	PublicKey string                 `json:"public_key"`
-	Items     []ResourceInfoResponse `json:"items"`
-	Path      string                 `json:"path"`
-	Limit     *uint64                `json:"limit"`
-	Offset    *uint64                `json:"offset"`
-	Total     *uint64                `json:"total"`
-}

backend/yandex/api/sort_mode.go

@@ -1,79 +0,0 @@
-package src
-
-import "strings"
-
-// SortMode struct - sort mode
-type SortMode struct {
-	mode string
-}
-
-// Default - sort mode
-func (m *SortMode) Default() *SortMode {
-	return &SortMode{
-		mode: "",
-	}
-}
-
-// ByName - sort mode
-func (m *SortMode) ByName() *SortMode {
-	return &SortMode{
-		mode: "name",
-	}
-}
-
-// ByPath - sort mode
-func (m *SortMode) ByPath() *SortMode {
-	return &SortMode{
-		mode: "path",
-	}
-}
-
-// ByCreated - sort mode
-func (m *SortMode) ByCreated() *SortMode {
-	return &SortMode{
-		mode: "created",
-	}
-}
-
-// ByModified - sort mode
-func (m *SortMode) ByModified() *SortMode {
-	return &SortMode{
-		mode: "modified",
-	}
-}
-
-// BySize - sort mode
-func (m *SortMode) BySize() *SortMode {
-	return &SortMode{
-		mode: "size",
-	}
-}
-
-// Reverse - sort mode
-func (m *SortMode) Reverse() *SortMode {
-	if strings.HasPrefix(m.mode, "-") {
-		return &SortMode{
-			mode: m.mode[1:],
-		}
-	}
-	return &SortMode{
-		mode: "-" + m.mode,
-	}
-}
-
-func (m *SortMode) String() string {
-	return m.mode
-}
-
-// UnmarshalJSON sort mode
-func (m *SortMode) UnmarshalJSON(value []byte) error {
-	if value == nil || len(value) == 0 {
-		m.mode = ""
-		return nil
-	}
-	m.mode = string(value)
-	if strings.HasPrefix(m.mode, "\"") && strings.HasSuffix(m.mode, "\"") {
-		m.mode = m.mode[1 : len(m.mode)-1]
-	}
-	return nil
-}