mount: use the equivalent of kernel_cache by default #FIXME WIP

Before this copyto would parse windows paths incorrectly.

This change moves the parsing code into fspath and makes sure
fspath.Split calls fspath.Parse which does the parsing correctly for

This also renames fspath.RemoteParse to fspath.Parse for consistency

.gometalinter.json

@@ -0,0 +1,14 @@
+{
+    "Enable": [
+        "deadcode",
+        "errcheck",
+        "goimports",
+        "golint",
+        "ineffassign",
+        "structcheck",
+        "varcheck",
+        "vet"
+    ],
+    "EnableGC": true,
+    "Vendor": true
+}

.travis.yml

@@ -4,11 +4,10 @@ dist: trusty
 os:
 - linux
 go:
-- 1.6.4
 - 1.7.6
 - 1.8.7
 - 1.9.3
-- "1.10"
+- "1.10.1"
 - tip
 before_install:
 - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo modprobe fuse ; sudo chmod 666 /dev/fuse ; sudo chown root:$USER /etc/fuse.conf ; fi
@@ -39,7 +38,7 @@ matrix:
   - go: tip
   include:
   - os: osx
-    go: "1.10"
+    go: "1.10.1"
     env: GOTAGS=""
 deploy:
   provider: script
@@ -47,5 +46,5 @@ deploy:
   skip_cleanup: true
   on:
     all_branches: true
-    go: "1.10"
-    condition: $TRAVIS_OS_NAME == linux && $TRAVIS_PULL_REQUEST == false
+    go: "1.10.1"
+    condition: $TRAVIS_PULL_REQUEST == false

CONTRIBUTING.md

@@ -235,15 +235,15 @@ in the `vendor` directory for perfectly reproducable builds.
 
 The `vendor` directory is entirely managed by the `dep` tool.
 
-To add a new dependency
+To add a new dependency, run `dep ensure` and `dep` will pull in the
+new dependency to the `vendor` directory and update the `Gopkg.lock`
+file.
 
-    dep ensure -add github.com/pkg/errors
+You can add constraints on that package in the `Gopkg.toml` file (see
+the `dep` documentation), but don't unless you really need to.
 
-You can add constraints on that package (see the `dep` documentation),
-but don't unless you really need to.
-
-Please check in the changes generated by dep including the `vendor`
-directory and `Godep.toml` and `Godep.locl` in a single commit
+Please check in the changes generated by `dep` including the `vendor`
+directory and `Godep.toml` and `Godep.lock` in a single commit
 separate from any other code changes.  Watch out for new files in
 `vendor`.
 
@@ -303,8 +303,7 @@ Getting going
 Unit tests
 
   * Create a config entry called `TestRemote` for the unit tests to use
-  * Add your fs to the end of `fstest/fstests/gen_tests.go`
-  * generate `backend/remote/remote_test.go` unit tests `cd fstest/fstests; go generate`
+  * Create a `backend/remote/remote_test.go` - copy and adjust your example remote
   * Make sure all tests pass with `go test -v`
 
 Integration tests

Gopkg.lock

@@ -9,19 +9,22 @@
     "fs",
     "fuseutil"
   ]
-  revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748"
+  revision = "65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e"
 
 [[projects]]
   name = "cloud.google.com/go"
   packages = ["compute/metadata"]
-  revision = "050b16d2314d5fc3d4c9a51e4cd5c7468e77f162"
-  version = "v0.17.0"
+  revision = "0fd7230b2a7505833d5f69b75cbd6c9582401479"
+  version = "v0.23.0"
 
 [[projects]]
   name = "github.com/Azure/azure-sdk-for-go"
-  packages = ["storage"]
-  revision = "eae258195456be76b2ec9ad2ee2ab63cdda365d9"
-  version = "v12.2.0-beta"
+  packages = [
+    "storage",
+    "version"
+  ]
+  revision = "fbe7db0e3f9793ba3e5704efbab84f51436c136e"
+  version = "v18.0.0"
 
 [[projects]]
   name = "github.com/Azure/go-autorest"
@@ -31,8 +34,8 @@
     "autorest/azure",
     "autorest/date"
   ]
-  revision = "6311d7a76f54cf2b6dea03d737d9bd9a6022ac5f"
-  version = "v9.7.1"
+  revision = "1f7cd6cfe0adea687ad44a512dfe76140f804318"
+  version = "v10.12.0"
 
 [[projects]]
   branch = "master"
@@ -41,16 +44,16 @@
   revision = "ef1e4c783f8f0478bd8bff0edb3dd0bade552599"
 
 [[projects]]
-  branch = "master"
   name = "github.com/VividCortex/ewma"
   packages = ["."]
-  revision = "43880d236f695d39c62cf7aa4ebd4508c258e6c0"
+  revision = "b24eb346a94c3ba12c1da1e564dbac1b498a77ce"
+  version = "v1.1.1"
 
 [[projects]]
   branch = "master"
   name = "github.com/a8m/tree"
   packages = ["."]
-  revision = "cf42b1e486f0b025942a768a9ad59c9939d6ca40"
+  revision = "3cf936ce15d6100c49d9c75f79c220ae7e579599"
 
 [[projects]]
   name = "github.com/abbot/go-http-auth"
@@ -59,7 +62,6 @@
   version = "v0.4.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/aws/aws-sdk-go"
   packages = [
     "aws",
@@ -72,14 +74,19 @@
     "aws/credentials/ec2rolecreds",
     "aws/credentials/endpointcreds",
     "aws/credentials/stscreds",
+    "aws/csm",
     "aws/defaults",
     "aws/ec2metadata",
     "aws/endpoints",
     "aws/request",
     "aws/session",
     "aws/signer/v4",
+    "internal/sdkio",
+    "internal/sdkrand",
     "internal/shareddefaults",
     "private/protocol",
+    "private/protocol/eventstream",
+    "private/protocol/eventstream/eventstreamapi",
     "private/protocol/query",
     "private/protocol/query/queryutil",
     "private/protocol/rest",
@@ -90,25 +97,26 @@
     "service/s3/s3manager",
     "service/sts"
   ]
-  revision = "2fe57096de348e6cff4031af99254613f8ef73ea"
+  revision = "bfc1a07cf158c30c41a3eefba8aae043d0bb5bff"
+  version = "v1.14.8"
 
 [[projects]]
   name = "github.com/billziss-gh/cgofuse"
   packages = ["fuse"]
-  revision = "487e2baa5611bab252a906d7f9b869f944607305"
-  version = "v1.0.4"
+  revision = "ea66f9809c71af94522d494d3d617545662ea59d"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
   name = "github.com/coreos/bbolt"
   packages = ["."]
-  revision = "ee30b748bcfbd74ec1d8439ae8fd4f9123a5c94e"
+  revision = "af9db2027c98c61ecd8e17caa5bd265792b9b9a2"
 
 [[projects]]
   name = "github.com/cpuguy83/go-md2man"
   packages = ["md2man"]
-  revision = "1d903dcb749992f3741d744c0f8376b4bd7eb3e1"
-  version = "v1.0.7"
+  revision = "20f5889cbdc3c73dbd2862796665e7c465ade7d1"
+  version = "v1.0.8"
 
 [[projects]]
   name = "github.com/davecgh/go-spew"
@@ -119,8 +127,8 @@
 [[projects]]
   name = "github.com/dgrijalva/jwt-go"
   packages = ["."]
-  revision = "dbeaa9332f19a944acb5736b4456cfcc02140e29"
-  version = "v3.1.0"
+  revision = "06ea1031745cb8b3dab3f6a236daf2b0aa468b7e"
+  version = "v3.2.0"
 
 [[projects]]
   name = "github.com/djherbis/times"
@@ -129,27 +137,34 @@
   version = "v1.0.1"
 
 [[projects]]
-  branch = "master"
   name = "github.com/dropbox/dropbox-sdk-go-unofficial"
   packages = [
     "dropbox",
     "dropbox/async",
+    "dropbox/common",
     "dropbox/file_properties",
-    "dropbox/files"
+    "dropbox/files",
+    "dropbox/seen_state",
+    "dropbox/sharing",
+    "dropbox/team_common",
+    "dropbox/team_policies",
+    "dropbox/users",
+    "dropbox/users_common"
   ]
-  revision = "9c27e83ceccc8f8bbc9afdc17c50798529d608b1"
+  revision = "7afa861bfde5a348d765522b303b6fbd9d250155"
+  version = "v4.1.0"
 
 [[projects]]
   name = "github.com/go-ini/ini"
   packages = ["."]
-  revision = "32e4c1e6bc4e7d0d8451aa6b75200d19e37a536a"
-  version = "v1.32.0"
+  revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5"
+  version = "v1.37.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/golang/protobuf"
   packages = ["proto"]
-  revision = "1e59b77b52bf8e4b449a57e6f79f21226d571845"
+  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
@@ -167,7 +182,7 @@
   branch = "master"
   name = "github.com/jlaffaye/ftp"
   packages = ["."]
-  revision = "427467931c6fbc25acba4537c9d3cbc40cfa569b"
+  revision = "2403248fa8cc9f7909862627aa7337f13f8e0bf1"
 
 [[projects]]
   name = "github.com/jmespath/go-jmespath"
@@ -181,15 +196,16 @@
   revision = "ae77be60afb1dcacde03767a8c37337fad28ac14"
 
 [[projects]]
-  branch = "master"
   name = "github.com/kr/fs"
   packages = ["."]
-  revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
+  revision = "1455def202f6e05b95cc7bfc7e8ae67ae5141eba"
+  version = "v0.1.0"
 
 [[projects]]
   name = "github.com/marstr/guid"
   packages = ["."]
-  revision = "8bdf7d1a087ccc975cf37dd6507da50698fd19ca"
+  revision = "8bd9a64bf37eb297b492a4101fb28e80ac0b290f"
+  version = "v1.1.0"
 
 [[projects]]
   name = "github.com/mattn/go-runewidth"
@@ -204,16 +220,16 @@
   revision = "887eb06ab6a255fbf5744b5812788e884078620a"
 
 [[projects]]
-  branch = "master"
   name = "github.com/ncw/swift"
   packages = ["."]
-  revision = "ae9f0ea1605b9aa6434ed5c731ca35d83ba67c55"
+  revision = "b2a7479cf26fa841ff90dd932d0221cb5c50782d"
+  version = "v1.0.39"
 
 [[projects]]
   branch = "master"
   name = "github.com/nsf/termbox-go"
   packages = ["."]
-  revision = "8c5e0793e04afcda7fe23d0751791e7321df4265"
+  revision = "5c94acc5e6eb520f1bcd183974e01171cc4c23b3"
 
 [[projects]]
   branch = "master"
@@ -236,20 +252,20 @@
     "log",
     "reopen"
   ]
-  revision = "b98065a377794d577e2a0e32869378b9ce4b8952"
-  version = "v0.1.1"
+  revision = "807ee759d82c84982a89fb3dc875ef884942f1e5"
+  version = "v0.2.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/pkg/errors"
   packages = ["."]
-  revision = "e881fd58d78e04cf6d0de1217f8707c8cc2249bc"
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/pkg/sftp"
   packages = ["."]
-  revision = "72ec6e85598d2480c30f633c154b07b6c112eade"
+  revision = "57673e38ea946592a59c26592b7e6fbda646975b"
+  version = "1.8.0"
 
 [[projects]]
   name = "github.com/pmezard/go-difflib"
@@ -258,16 +274,16 @@
   version = "v1.0.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/rfjakob/eme"
   packages = ["."]
-  revision = "2222dbd4ba467ab3fc7e8af41562fcfe69c0d770"
+  revision = "01668ae55fe0b79a483095689043cce3e80260db"
+  version = "v1.1"
 
 [[projects]]
   name = "github.com/russross/blackfriday"
   packages = ["."]
-  revision = "4048872b16cc0fc2c5fd9eacf0ed2c2fedaa0c8c"
-  version = "v1.5"
+  revision = "55d61fa8aa702f59229e6cff85793c22e580eaf5"
+  version = "v1.5.1"
 
 [[projects]]
   name = "github.com/satori/go.uuid"
@@ -276,10 +292,10 @@
   version = "v1.2.0"
 
 [[projects]]
+  branch = "master"
   name = "github.com/sevlyar/go-daemon"
   packages = ["."]
-  revision = "e49ef56654f54139c4dc0285f973f74e9649e729"
-  version = "v0.1.2"
+  revision = "f9261e73885de99b1647d68bedadf2b9a99ad11f"
 
 [[projects]]
   branch = "master"
@@ -288,28 +304,34 @@
   revision = "75fb7ed4208cf72d323d7d02fd1a5964a7a9073c"
 
 [[projects]]
-  branch = "master"
   name = "github.com/spf13/cobra"
   packages = [
     ".",
     "doc"
   ]
-  revision = "0c34d16c3123764e413b9ed982ada58b1c3d53ea"
+  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+  version = "v0.0.3"
 
 [[projects]]
-  branch = "master"
   name = "github.com/spf13/pflag"
   packages = ["."]
-  revision = "4c012f6dcd9546820e378d0bdda4d8fc772cdfea"
+  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
+  version = "v1.0.1"
 
 [[projects]]
-  branch = "master"
   name = "github.com/stretchr/testify"
   packages = [
     "assert",
     "require"
   ]
-  revision = "87b1dfb5b2fa649f52695dd9eae19abe404a4308"
+  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+  version = "v1.2.2"
+
+[[projects]]
+  branch = "master"
+  name = "github.com/t3rm1n4l/go-mega"
+  packages = ["."]
+  revision = "57978a63bd3f91fa7e188b751a7e7e6dd4e33813"
 
 [[projects]]
   branch = "master"
@@ -332,8 +354,8 @@
     "service",
     "utils"
   ]
-  revision = "51fa3b6bb3c24f4d646eefff251cd2e6ba716600"
-  version = "v2.2.9"
+  revision = "4f9ac88c5fec7350e960aabd0de1f1ede0ad2895"
+  version = "v2.2.14"
 
 [[projects]]
   branch = "master"
@@ -344,6 +366,8 @@
     "curve25519",
     "ed25519",
     "ed25519/internal/edwards25519",
+    "internal/chacha20",
+    "internal/subtle",
     "nacl/secretbox",
     "pbkdf2",
     "poly1305",
@@ -353,7 +377,7 @@
     "ssh/agent",
     "ssh/terminal"
   ]
-  revision = "13931e22f9e72ea58bb73048bc752b48c6d4d4ac"
+  revision = "027cca12c2d63e3d62b670d901e8a2c95854feec"
 
 [[projects]]
   branch = "master"
@@ -363,10 +387,16 @@
     "context/ctxhttp",
     "html",
     "html/atom",
+    "http/httpguts",
+    "http2",
+    "http2/hpack",
+    "idna",
+    "publicsuffix",
     "webdav",
-    "webdav/internal/xml"
+    "webdav/internal/xml",
+    "websocket"
   ]
-  revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec"
+  revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"
 
 [[projects]]
   branch = "master"
@@ -378,7 +408,7 @@
     "jws",
     "jwt"
   ]
-  revision = "30785a2c434e431ef7c507b54617d6a951d5f2b4"
+  revision = "1e0a3fa8ba9a5c9eb35c271780101fdaf1b205d7"
 
 [[projects]]
   branch = "master"
@@ -387,26 +417,34 @@
     "unix",
     "windows"
   ]
-  revision = "fff93fa7cd278d84afc205751523809c464168ab"
+  revision = "6c888cc515d3ed83fc103cf1d84468aad274b0a7"
 
 [[projects]]
-  branch = "master"
   name = "golang.org/x/text"
   packages = [
+    "collate",
+    "collate/build",
+    "internal/colltab",
     "internal/gen",
+    "internal/tag",
     "internal/triegen",
     "internal/ucd",
+    "language",
+    "secure/bidirule",
     "transform",
+    "unicode/bidi",
     "unicode/cldr",
-    "unicode/norm"
+    "unicode/norm",
+    "unicode/rangetable"
   ]
-  revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/time"
   packages = ["rate"]
-  revision = "6dc17368e09b0e8634d71cac8168d853e869a0c7"
+  revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
 
 [[projects]]
   branch = "master"
@@ -418,7 +456,7 @@
     "googleapi/internal/uritemplates",
     "storage/v1"
   ]
-  revision = "de3aa2cfa7f1c18dcb7f91738099bad280117b8e"
+  revision = "2eea9ba0a3d94f6ab46508083e299a00bbbc65f6"
 
 [[projects]]
   name = "google.golang.org/appengine"
@@ -435,18 +473,18 @@
     "log",
     "urlfetch"
   ]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
+  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
+  version = "v1.1.0"
 
 [[projects]]
-  branch = "v2"
   name = "gopkg.in/yaml.v2"
   packages = ["."]
-  revision = "d670f9405373e636a5a2765eea47fac0c9bc91a4"
+  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+  version = "v2.2.1"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "f9e9adb0675a970e6b6a9f28fa75e5bbee74d001359c688bd37c78f035be565a"
+  inputs-digest = "c1378c5fc821e27711155958ff64b3c74b56818ba4733dbfe0c86d518c32880e"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -1,154 +1,11 @@
-
-## Gopkg.toml example (these lines may be deleted)
-
-## "required" lists a set of packages (not projects) that must be included in
-## Gopkg.lock. This list is merged with the set of packages imported by the current
-## project. Use it when your project needs a package it doesn't explicitly import -
-## including "main" packages.
-# required = ["github.com/user/thing/cmd/thing"]
-
-## "ignored" lists a set of packages (not projects) that are ignored when
-## dep statically analyzes source code. Ignored packages can be in this project,
-## or in a dependency.
-# ignored = ["github.com/user/project/badpkg"]
-
-## Dependencies define constraints on dependent projects. They are respected by
-## dep whether coming from the Gopkg.toml of the current project or a dependency.
-# [[constraint]]
-## Required: the root import path of the project being constrained.
-# name = "github.com/user/project"
-#
-## Recommended: the version constraint to enforce for the project.
-## Only one of "branch", "version" or "revision" can be specified.
-# version = "1.0.0"
-# branch = "master"
-# revision = "abc123"
-#
-## Optional: an alternate location (URL or import path) for the project's source.
-# source = "https://github.com/myfork/package.git"
-
-## Overrides have the same structure as [[constraint]], but supercede all
-## [[constraint]] declarations from all projects. Only the current project's
-## [[override]] are applied.
-##
-## Overrides are a sledgehammer. Use them only as a last resort.
-# [[override]]
-## Required: the root import path of the project being constrained.
-# name = "github.com/user/project"
-#
-## Optional: specifying a version constraint override will cause all other
-## constraints on this project to be ignored; only the overriden constraint
-## need be satisfied.
-## Again, only one of "branch", "version" or "revision" can be specified.
-# version = "1.0.0"
-# branch = "master"
-# revision = "abc123"
-#
-## Optional: specifying an alternate source location as an override will
-## enforce that the alternate location is used for that project, regardless of
-## what source location any dependent projects specify.
-# source = "https://github.com/myfork/package.git"
-
-[[constraint]]
+# pin this to master to pull in the macOS changes
+# can likely remove for 1.43
+[[override]]
   branch = "master"
-  name = "bazil.org/fuse"
+  name = "github.com/sevlyar/go-daemon"
 
-[[constraint]]
-  branch = "master"
-  name = "github.com/Unknwon/goconfig"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/VividCortex/ewma"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/aws/aws-sdk-go"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ncw/go-acd"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ncw/swift"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/pkg/errors"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/pkg/sftp"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/rfjakob/eme"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/skratchdot/open-golang"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/pflag"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/stacktic/dropbox"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/stretchr/testify"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/net"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/sys"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/text"
-
-[[constraint]]
-  branch = "master"
-  name = "google.golang.org/api"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/dropbox/dropbox-sdk-go-unofficial"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/yunify/qingstor-sdk-go"
-
-[[constraint]]
+# pin this to master to pull in the fix for linux/mips
+# can likely remove for 1.43
+[[override]]
   branch = "master"
   name = "github.com/coreos/bbolt"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/patrickmn/go-cache"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/okzk/sdnotify"
-
-[[constraint]]
-  name = "github.com/sevlyar/go-daemon"
-  version = "0.1.2"

Makefile

@@ -1,12 +1,22 @@
 SHELL = bash
-TAG := $(shell echo $$(git describe --abbrev=8 --tags)-$${APPVEYOR_REPO_BRANCH:-$${TRAVIS_BRANCH:-$$(git rev-parse --abbrev-ref HEAD)}} | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/; s/-\(HEAD\|master\)$$//')
+BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
+TAG_BRANCH := -$(BRANCH)
+BRANCH_PATH := branch/
+ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
+	TAG_BRANCH :=
+	BRANCH_PATH :=
+endif
+TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
 LAST_TAG := $(shell git describe --tags --abbrev=0)
 NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
 # Run full tests if go >= go1.9
 FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 9)')
-BETA_URL := https://beta.rclone.org/$(TAG)/
+BETA_PATH := $(BRANCH_PATH)$(TAG)
+BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
+BETA_UPLOAD_ROOT := memstore:beta-rclone-org
+BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
@@ -21,6 +31,7 @@ rclone:
 
 vars:
 	@echo SHELL="'$(SHELL)'"
+	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
@@ -56,16 +67,30 @@ else
 	@echo Skipping source quality tests as version of go too old
 endif
 
+gometalinter_install:
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install --update
+
+# We aren't using gometalinter as the default linter yet because
+# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
+# 2. can't get -printfuncs working with the vet linter
+gometalinter:
+	gometalinter ./...
+
 # Get the build dependencies
 build_dep:
 ifdef FULL_TESTS
 	go get -u github.com/kisielk/errcheck
 	go get -u golang.org/x/tools/cmd/goimports
 	go get -u github.com/golang/lint/golint
-	go get -u github.com/inconshreveable/mousetrap
 	go get -u github.com/tools/godep
 endif
 
+# Get the release dependencies
+release_dep:
+	go get -u github.com/goreleaser/nfpm/...
+	go get -u github.com/aktau/github-release
+
 # Update dependencies
 update:
 	go get -u github.com/golang/dep/cmd/dep
@@ -88,6 +113,9 @@ MANUAL.txt:	MANUAL.md
 commanddocs: rclone
 	rclone gendocs docs/content/commands/
 
+rcdocs: rclone
+	bin/make_rc_docs.sh
+
 install: rclone
 	install -d ${DESTDIR}/usr/bin
 	install -t ${DESTDIR}/usr/bin ${GOPATH}/bin/rclone
@@ -105,12 +133,12 @@ upload_website:	website
 	rclone -v sync docs/public memstore:www-rclone-org
 
 tarball:
-	git archive -9 --format=tar.gz --prefix=rclone-$(TAG) -o build/rclone-$(TAG).tar.gz $(TAG)
+	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
 sign_upload:
-	cd build && md5sum rclone-* | gpg --clearsign > MD5SUMS
-	cd build && sha1sum rclone-* | gpg --clearsign > SHA1SUMS
-	cd build && sha256sum rclone-* | gpg --clearsign > SHA256SUMS
+	cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS
+	cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS
+	cd build && sha256sum rclone-v* | gpg --clearsign > SHA256SUMS
 
 check_sign:
 	cd build && gpg --verify MD5SUMS && gpg --decrypt MD5SUMS | md5sum -c
@@ -118,7 +146,8 @@ check_sign:
 	cd build && gpg --verify SHA256SUMS && gpg --decrypt SHA256SUMS | sha256sum -c
 
 upload:
-	rclone -v copy build/ memstore:downloads-rclone-org
+	rclone -v copy --exclude '*current*' build/ memstore:downloads-rclone-org/$(TAG)
+	rclone -v copy --include '*current*' --include version.txt build/ memstore:downloads-rclone-org
 
 upload_github:
 	./bin/upload-github $(TAG)
@@ -142,25 +171,32 @@ else
 endif
 
 appveyor_upload:
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ memstore:beta-rclone-org/$(TAG)
-ifeq ($(APPVEYOR_REPO_BRANCH),master)
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ memstore:beta-rclone-org
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
+BUILD_FLAGS := -exclude "^(windows|darwin)/"
+ifeq ($(TRAVIS_OS_NAME),osx)
+	BUILD_FLAGS := -include "^darwin/" -cgo
+endif
+
 travis_beta:
+ifeq ($(TRAVIS_OS_NAME),linux)
 	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
-	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt -exclude "^windows/" -parallel 8 $(BUILDTAGS) $(TAG)β
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ memstore:beta-rclone-org/$(TAG)
-ifeq ($(TRAVIS_BRANCH),master)
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ memstore:beta-rclone-org
+	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)β
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
 # Fetch the windows builds from appveyor
 fetch_windows:
-	rclone -v copy --include 'rclone-v*-windows-*.zip' memstore:beta-rclone-org/$(TAG) build/
+	rclone -v copy --include 'rclone-v*-windows-*.zip' $(BETA_UPLOAD) build/
 	-#cp -av build/rclone-v*-windows-386.zip build/rclone-current-windows-386.zip
 	-#cp -av build/rclone-v*-windows-amd64.zip build/rclone-current-windows-amd64.zip
 	md5sum build/rclone-*-windows-*.zip | sort
@@ -188,9 +224,6 @@ startdev:
 	echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(LAST_TAG)-DEV\"\n" | gofmt > fs/version.go
 	git commit -m "Start $(LAST_TAG)-DEV development" fs/version.go
 
-gen_tests:
-	cd fstest/fstests && go generate
-
 winzip:
 	zip -9 rclone-$(TAG).zip rclone.exe
 

README.md

@@ -25,8 +25,10 @@ Rclone is a command line program to sync files and directories to and from
   * Google Drive
   * HTTP
   * Hubic
+  * Mega
   * Microsoft Azure Blob Storage
   * Microsoft OneDrive
+  * OpenDrive
   * Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage
   * pCloud
   * QingStor

RELEASE.md

@@ -13,6 +13,7 @@ Making a release
   * git status - to check for new man pages - git add them
   * git commit -a -v -m "Version v1.XX"
   * make retag
+  * make release_dep
   * # Set the GOPATH for a current stable go compiler
   * make cross
   * git checkout docs/content/commands # to undo date changes in commands
@@ -30,6 +31,7 @@ Making a release
   * # announce with forum post, twitter post, G+ post
 
 Early in the next release cycle update the vendored dependencies
+  * Review any pinned packages in Gopkg.toml and remove if possible
   * make update
   * git status
   * git add new files

backend/alias/alias_internal_test.go

@@ -15,8 +15,6 @@ import (
 
 var (
 	remoteName = "TestAlias"
-	testPath   = "test"
-	filesPath  = filepath.Join(testPath, "files")
 )
 
 func prepare(t *testing.T, root string) {

backend/all/all.go

@@ -16,7 +16,9 @@ import (
 	_ "github.com/ncw/rclone/backend/http"
 	_ "github.com/ncw/rclone/backend/hubic"
 	_ "github.com/ncw/rclone/backend/local"
+	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"
+	_ "github.com/ncw/rclone/backend/opendrive"
 	_ "github.com/ncw/rclone/backend/pcloud"
 	_ "github.com/ncw/rclone/backend/qingstor"
 	_ "github.com/ncw/rclone/backend/s3"

backend/amazonclouddrive/amazonclouddrive.go

@@ -18,7 +18,6 @@ import (
 	"log"
 	"net/http"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -40,7 +39,6 @@ import (
 const (
 	folderKind      = "FOLDER"
 	fileKind        = "FILE"
-	assetKind       = "ASSET"
 	statusAvailable = "AVAILABLE"
 	timeFormat      = time.RFC3339 // 2014-03-07T22:31:12.173Z
 	minSleep        = 20 * time.Millisecond
@@ -138,9 +136,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a acd path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an acd 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -1320,6 +1315,14 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoin
 	return checkpoint
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	if o.info.Id == nil {
+		return ""
+	}
+	return *o.info.Id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs     = (*Fs)(nil)
@@ -1331,4 +1334,5 @@ var (
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = &Object{}
+	_ fs.IDer            = &Object{}
 )

backend/amazonclouddrive/amazonclouddrive_test.go

@@ -1,7 +1,4 @@
 // Test AmazonCloudDrive filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 
 // +build acd
 
@@ -15,64 +12,9 @@ import (
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
 	fstests.NilObject = fs.Object((*amazonclouddrive.Object)(nil))
 	fstests.RemoteName = "TestAmazonCloudDrive:"
+	fstests.Run(t)
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/azureblob/azureblob.go

@@ -1,7 +1,4 @@
 // Package azureblob provides an interface to the Microsoft Azure blob object storage system
-
-// +build go1.7
-
 package azureblob
 
 import (
@@ -11,9 +8,9 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
-	gohash "hash"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"regexp"
 	"strconv"
@@ -35,16 +32,16 @@ import (
 )
 
 const (
-	apiVersion         = "2017-04-17"
-	minSleep           = 10 * time.Millisecond
-	maxSleep           = 10 * time.Second
-	decayConstant      = 1    // bigger for slower decay, exponential
-	listChunkSize      = 5000 // number of items to read at once
-	modTimeKey         = "mtime"
-	timeFormatIn       = time.RFC3339
-	timeFormatOut      = "2006-01-02T15:04:05.000000000Z07:00"
-	maxTotalParts      = 50000   // in multipart upload
-	maxUncommittedSize = 9 << 30 // can't upload bigger than this
+	apiVersion    = "2017-04-17"
+	minSleep      = 10 * time.Millisecond
+	maxSleep      = 10 * time.Second
+	decayConstant = 1    // bigger for slower decay, exponential
+	listChunkSize = 5000 // number of items to read at once
+	modTimeKey    = "mtime"
+	timeFormatIn  = time.RFC3339
+	timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
+	maxTotalParts = 50000 // in multipart upload
+	// maxUncommittedSize = 9 << 30 // can't upload bigger than this
 )
 
 // Globals
@@ -63,10 +60,16 @@ func init() {
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: "account",
-			Help: "Storage Account Name",
+			Help: "Storage Account Name (leave blank to use connection string or SAS URL)",
 		}, {
 			Name: "key",
-			Help: "Storage Account Key",
+			Help: "Storage Account Key (leave blank to use connection string or SAS URL)",
+		}, {
+			Name: "connection_string",
+			Help: "Connection string (leave blank if using account/key or SAS URL)",
+		}, {
+			Name: "sas_url",
+			Help: "SAS URL for container level access only\n(leave blank if using account/key or connection string)",
 		}, {
 			Name: "endpoint",
 			Help: "Endpoint for the service - leave blank normally.",
@@ -83,7 +86,6 @@ type Fs struct {
 	root             string       // the path we are working on if any
 	features         *fs.Features // optional features
 	account          string       // account name
-	key              []byte       // auth key
 	endpoint         string       // name of the starting api endpoint
 	bc               *storage.BlobStorageClient
 	cc               *storage.Container
@@ -99,7 +101,6 @@ type Fs struct {
 type Object struct {
 	fs       *Fs               // what this object is part of
 	remote   string            // The remote path
-	id       string            // azure id of the file
 	modTime  time.Time         // The modified time of the object if known
 	md5      string            // MD5 hash if known
 	size     int64             // Size of the object
@@ -188,36 +189,54 @@ func NewFs(name, root string) (fs.Fs, error) {
 		return nil, err
 	}
 	account := config.FileGet(name, "account")
-	if account == "" {
-		return nil, errors.New("account not found")
-	}
 	key := config.FileGet(name, "key")
-	if key == "" {
-		return nil, errors.New("key not found")
-	}
-	keyBytes, err := base64.StdEncoding.DecodeString(key)
-	if err != nil {
-		return nil, errors.Errorf("malformed storage account key: %v", err)
-	}
-
+	connectionString := config.FileGet(name, "connection_string")
+	sasURL := config.FileGet(name, "sas_url")
 	endpoint := config.FileGet(name, "endpoint", storage.DefaultBaseURL)
 
-	client, err := storage.NewClient(account, key, endpoint, apiVersion, true)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to make azure storage client")
+	var (
+		oclient storage.Client
+		client  = &oclient
+		cc      *storage.Container
+	)
+	switch {
+	case account != "" && key != "":
+		oclient, err = storage.NewClient(account, key, endpoint, apiVersion, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to make azure storage client from account/key")
+		}
+	case connectionString != "":
+		oclient, err = storage.NewClientFromConnectionString(connectionString)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to make azure storage client from connection string")
+		}
+	case sasURL != "":
+		URL, err := url.Parse(sasURL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse SAS URL")
+		}
+		cc, err = storage.GetContainerReferenceFromSASURI(*URL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to make azure storage client from SAS URL")
+		}
+		client = cc.Client()
+	default:
+		return nil, errors.New("Need account+key or connectionString or sasURL")
 	}
 	client.HTTPClient = fshttp.NewClient(fs.Config)
 	bc := client.GetBlobService()
+	if cc == nil {
+		cc = bc.GetContainerReference(container)
+	}
 
 	f := &Fs{
 		name:        name,
 		container:   container,
 		root:        directory,
 		account:     account,
-		key:         keyBytes,
 		endpoint:    endpoint,
 		bc:          &bc,
-		cc:          bc.GetContainerReference(container),
+		cc:          cc,
 		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
@@ -554,10 +573,19 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
+	// List the container to see if it exists
+	err := f.list("", false, 1, func(remote string, object *storage.Blob, isDirectory bool) error {
+		return nil
+	})
+	if err == nil {
+		f.markContainerOK()
+		return nil
+	}
+	// now try to create the container
 	options := storage.CreateContainerOptions{
 		Access: storage.ContainerAccessTypePrivate,
 	}
-	err := f.pacer.Call(func() (bool, error) {
+	err = f.pacer.Call(func() (bool, error) {
 		err := f.cc.Create(&options)
 		if err != nil {
 			if storageErr, ok := err.(storage.AzureStorageServiceError); ok {
@@ -850,16 +878,6 @@ func (o *Object) Storable() bool {
 	return true
 }
 
-// openFile represents an Object open for reading
-type openFile struct {
-	o     *Object        // Object we are reading for
-	resp  *http.Response // response of the GET
-	body  io.Reader      // reading from here
-	hash  gohash.Hash    // currently accumulating MD5
-	bytes int64          // number of bytes read on this connection
-	eof   bool           // whether we have read end of file
-}
-
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	getBlobOptions := storage.GetBlobOptions{}
@@ -927,20 +945,6 @@ func init() {
 	}
 }
 
-// urlEncode encodes in with % encoding
-func urlEncode(in string) string {
-	var out bytes.Buffer
-	for i := 0; i < len(in); i++ {
-		c := in[i]
-		if noNeedToEncode[c] {
-			_ = out.WriteByte(c)
-		} else {
-			_, _ = out.WriteString(fmt.Sprintf("%%%2X", c))
-		}
-	}
-	return out.String()
-}
-
 // uploadMultipart uploads a file using multipart upload
 //
 // Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.

backend/azureblob/azureblob_test.go

@@ -1,78 +1,17 @@
 // Test AzureBlob filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
-
-// +build go1.7
-
 package azureblob_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/azureblob"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*azureblob.Object)(nil))
-	fstests.RemoteName = "TestAzureBlob:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestAzureBlob:",
+		NilObject:  (*azureblob.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/azureblob/azureblob_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.7
-
-package azureblob

backend/b2/b2.go

@@ -1422,6 +1422,11 @@ func (o *Object) MimeType() string {
 	return o.mimeType
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}
@@ -1431,4 +1436,5 @@ var (
 	_ fs.ListRer     = &Fs{}
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
+	_ fs.IDer        = &Object{}
 )

backend/b2/b2_test.go

@@ -1,75 +1,17 @@
 // Test B2 filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package b2_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/b2"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*b2.Object)(nil))
-	fstests.RemoteName = "TestB2:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestB2:",
+		NilObject:  (*b2.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/b2/upload.go

@@ -357,7 +357,8 @@ outer:
 		buf := up.f.getUploadBlock()
 
 		// Read the chunk
-		n, err := io.ReadFull(up.in, buf)
+		var n int
+		n, err = io.ReadFull(up.in, buf)
 		if err == io.ErrUnexpectedEOF {
 			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
 			buf = buf[:n]
@@ -366,7 +367,6 @@ outer:
 		} else if err == io.EOF {
 			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
 			up.f.putUploadBlock(buf)
-			hasMoreParts = false
 			err = nil
 			break outer
 		} else if err != nil {

backend/box/api/types.go

@@ -74,18 +74,18 @@ const (
 
 // Item describes a folder or a file as returned by Get Folder Items and others
 type Item struct {
-	Type              string `json:"type"`
-	ID                string `json:"id"`
-	SequenceID        string `json:"sequence_id"`
-	Etag              string `json:"etag"`
-	SHA1              string `json:"sha1"`
-	Name              string `json:"name"`
-	Size              int64  `json:"size"`
-	CreatedAt         Time   `json:"created_at"`
-	ModifiedAt        Time   `json:"modified_at"`
-	ContentCreatedAt  Time   `json:"content_created_at"`
-	ContentModifiedAt Time   `json:"content_modified_at"`
-	ItemStatus        string `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
+	Type              string  `json:"type"`
+	ID                string  `json:"id"`
+	SequenceID        string  `json:"sequence_id"`
+	Etag              string  `json:"etag"`
+	SHA1              string  `json:"sha1"`
+	Name              string  `json:"name"`
+	Size              float64 `json:"size"` // box returns this in xEyy format for very large numbers - see #2261
+	CreatedAt         Time    `json:"created_at"`
+	ModifiedAt        Time    `json:"modified_at"`
+	ContentCreatedAt  Time    `json:"content_created_at"`
+	ContentModifiedAt Time    `json:"content_modified_at"`
+	ItemStatus        string  `json:"item_status"` // active, trashed if the file has been moved to the trash, and deleted if the file has been permanently deleted
 }
 
 // ModTime returns the modification time of the item

backend/box/box.go

@@ -16,7 +16,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -135,9 +134,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a box path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -883,7 +879,7 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 		return errors.Wrapf(fs.ErrorNotAFile, "%q is %q", o.remote, info.Type)
 	}
 	o.hasMetaData = true
-	o.size = info.Size
+	o.size = int64(info.Size)
 	o.sha1 = info.SHA1
 	o.modTime = info.ModTime()
 	o.id = info.ID
@@ -1052,6 +1048,11 @@ func (o *Object) Remove() error {
 	return o.fs.deleteObject(o.id)
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1062,4 +1063,5 @@ var (
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/box/box_test.go

@@ -1,75 +1,17 @@
 // Test Box filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package box_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/box"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*box.Object)(nil))
-	fstests.RemoteName = "TestBox:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestBox:",
+		NilObject:  (*box.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/cache/cache.go

@@ -1,20 +1,19 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 
 import (
+	"context"
 	"fmt"
 	"io"
+	"os"
+	"os/signal"
 	"path"
 	"path/filepath"
 	"strings"
 	"sync"
-	"time"
-
-	"os"
-
-	"os/signal"
 	"syscall"
+	"time"
 
 	"github.com/ncw/rclone/backend/crypt"
 	"github.com/ncw/rclone/fs"
@@ -26,7 +25,6 @@ import (
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/atexit"
 	"github.com/pkg/errors"
-	"golang.org/x/net/context"
 	"golang.org/x/time/rate"
 )
 
@@ -337,6 +335,9 @@ func NewFs(name, rootPath string) (fs.Fs, error) {
 	c := make(chan os.Signal, 1)
 	signal.Notify(c, syscall.SIGHUP)
 	atexit.Register(func() {
+		if plexURL != "" {
+			f.plexConnector.closeWebsocket()
+		}
 		f.StopBackgroundRunners()
 	})
 	go func() {
@@ -428,12 +429,37 @@ Purge a remote from the cache backend. Supports either a directory or a file.
 Params:
   - remote = path to remote (required)
   - withData = true/false to delete cached data (chunks) as well (optional)
+
+Eg
+
+    rclone rc cache/expire remote=path/to/sub/folder/
+    rclone rc cache/expire remote=/ withData=true 
+`,
+	})
+
+	rc.Add(rc.Call{
+		Path:  "cache/stats",
+		Fn:    f.httpStats,
+		Title: "Get cache stats",
+		Help: `
+Show statistics for the cache remote.
 `,
 	})
 
 	return f, fsErr
 }
 
+func (f *Fs) httpStats(in rc.Params) (out rc.Params, err error) {
+	out = make(rc.Params)
+	m, err := f.Stats()
+	if err != nil {
+		return out, errors.Errorf("error while getting cache stats")
+	}
+	out["status"] = "ok"
+	out["stats"] = m
+	return out, nil
+}
+
 func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 	out = make(rc.Params)
 	remoteInt, ok := in["remote"]
@@ -447,18 +473,20 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 		withData = true
 	}
 
-	// if it's wrapped by crypt we need to check what format we got
-	if cryptFs, yes := f.isWrappedByCrypt(); yes {
-		_, err := cryptFs.DecryptFileName(remote)
-		// if it failed to decrypt then it is a decrypted format and we need to encrypt it
-		if err != nil {
-			remote = cryptFs.EncryptFileName(remote)
+	if cleanPath(remote) != "" {
+		// if it's wrapped by crypt we need to check what format we got
+		if cryptFs, yes := f.isWrappedByCrypt(); yes {
+			_, err := cryptFs.DecryptFileName(remote)
+			// if it failed to decrypt then it is a decrypted format and we need to encrypt it
+			if err != nil {
+				remote = cryptFs.EncryptFileName(remote)
+			}
+			// else it's an encrypted format and we can use it as it is
 		}
-		// else it's an encrypted format and we can use it as it is
-	}
 
-	if !f.cache.HasEntry(path.Join(f.Root(), remote)) {
-		return out, errors.Errorf("%s doesn't exist in cache", remote)
+		if !f.cache.HasEntry(path.Join(f.Root(), remote)) {
+			return out, errors.Errorf("%s doesn't exist in cache", remote)
+		}
 	}
 
 	co := NewObject(f, remote)
@@ -476,17 +504,12 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 		return out, nil
 	}
 	// expire the entry
-	co.CacheTs = time.Now().Add(f.fileAge * -1)
-	err = f.cache.AddObject(co)
+	err = f.cache.ExpireObject(co, withData)
 	if err != nil {
 		return out, errors.WithMessage(err, "error expiring file")
 	}
 	// notify vfs too
 	f.notifyChangeUpstream(co.Remote(), fs.EntryObject)
-	if withData {
-		// safe to ignore as the file might not have been open
-		_ = os.RemoveAll(path.Join(f.cache.dataPath, co.abs()))
-	}
 
 	out["status"] = "ok"
 	out["message"] = fmt.Sprintf("cached file cleared: %v", remote)
@@ -495,7 +518,16 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 
 // receiveChangeNotify is a wrapper to notifications sent from the wrapped FS about changed files
 func (f *Fs) receiveChangeNotify(forgetPath string, entryType fs.EntryType) {
-	fs.Debugf(f, "notify: expiring cache for '%v'", forgetPath)
+	if crypt, yes := f.isWrappedByCrypt(); yes {
+		decryptedPath, err := crypt.DecryptFileName(forgetPath)
+		if err == nil {
+			fs.Infof(decryptedPath, "received cache expiry notification")
+		} else {
+			fs.Infof(forgetPath, "received cache expiry notification")
+		}
+	} else {
+		fs.Infof(forgetPath, "received cache expiry notification")
+	}
 	// notify upstreams too (vfs)
 	f.notifyChangeUpstream(forgetPath, entryType)
 
@@ -504,27 +536,22 @@ func (f *Fs) receiveChangeNotify(forgetPath string, entryType fs.EntryType) {
 		co := NewObject(f, forgetPath)
 		err := f.cache.GetObject(co)
 		if err != nil {
-			fs.Debugf(f, "ignoring change notification for non cached entry %v", co)
-			return
+			fs.Debugf(f, "got change notification for non cached entry %v", co)
 		}
-		// expire the entry
-		co.CacheTs = time.Now().Add(f.fileAge * -1)
-		err = f.cache.AddObject(co)
+		err = f.cache.ExpireObject(co, true)
 		if err != nil {
-			fs.Errorf(forgetPath, "notify: error expiring '%v': %v", co, err)
-		} else {
-			fs.Debugf(forgetPath, "notify: expired %v", co)
+			fs.Debugf(forgetPath, "notify: error expiring '%v': %v", co, err)
 		}
 		cd = NewDirectory(f, cleanPath(path.Dir(co.Remote())))
 	} else {
 		cd = NewDirectory(f, forgetPath)
-		// we expire the dir
-		err := f.cache.ExpireDir(cd)
-		if err != nil {
-			fs.Errorf(forgetPath, "notify: error expiring '%v': %v", cd, err)
-		} else {
-			fs.Debugf(forgetPath, "notify: expired '%v'", cd)
-		}
+	}
+	// we expire the dir
+	err := f.cache.ExpireDir(cd)
+	if err != nil {
+		fs.Debugf(forgetPath, "notify: error expiring '%v': %v", cd, err)
+	} else {
+		fs.Debugf(forgetPath, "notify: expired '%v'", cd)
 	}
 
 	f.notifiedMu.Lock()
@@ -618,7 +645,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 
 	// search for entry in source or temp fs
 	var obj fs.Object
-	err = nil
 	if f.tempWritePath != "" {
 		obj, err = f.tempFs.NewObject(remote)
 		// not found in temp fs
@@ -697,6 +723,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	fs.Debugf(dir, "list: source entries: %v", entries)
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
+	var batchDirectories []*Directory
 	for _, entry := range entries {
 		switch o := entry.(type) {
 		case fs.Object:
@@ -718,18 +745,19 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			cdd := DirectoryFromOriginal(f, o)
 			// check if the dir isn't expired and add it in cache if it isn't
 			if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(f.fileAge)) {
-				err := f.cache.AddDir(cdd)
-				if err != nil {
-					fs.Errorf(dir, "list: error caching dir from listing %v", o)
-				} else {
-					fs.Debugf(dir, "list: cached dir: %v", cdd)
-				}
+				batchDirectories = append(batchDirectories, cdd)
 			}
 			cachedEntries = append(cachedEntries, cdd)
 		default:
 			fs.Debugf(entry, "list: Unknown object type %T", entry)
 		}
 	}
+	err = f.cache.AddBatchDir(batchDirectories)
+	if err != nil {
+		fs.Errorf(dir, "list: error caching directories from listing %v", dir)
+	} else {
+		fs.Debugf(dir, "list: cached directories: %v", len(batchDirectories))
+	}
 
 	// cache dir meta
 	t := time.Now()
@@ -1139,8 +1167,14 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
 	}
 
 	// cache the new file
-	cachedObj := ObjectFromOriginal(f, obj).persist()
+	cachedObj := ObjectFromOriginal(f, obj)
+
+	// deleting cached chunks and info to be replaced with new ones
+	_ = f.cache.RemoveObject(cachedObj.abs())
+
+	cachedObj.persist()
 	fs.Debugf(cachedObj, "put: added to cache")
+
 	// expire parent
 	parentCd := NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote())))
 	err = f.cache.ExpireDir(parentCd)
@@ -1389,6 +1423,15 @@ func (f *Fs) CleanUp() error {
 	return do()
 }
 
+// About gets quota information from the Fs
+func (f *Fs) About() (*fs.Usage, error) {
+	do := f.Fs.Features().About
+	if do == nil {
+		return nil, errors.New("About not supported")
+	}
+	return do()
+}
+
 // Stats returns stats about the cache storage
 func (f *Fs) Stats() (map[string]map[string]interface{}, error) {
 	return f.cache.Stats()
@@ -1527,4 +1570,5 @@ var (
 	_ fs.Wrapper        = (*Fs)(nil)
 	_ fs.ListRer        = (*Fs)(nil)
 	_ fs.ChangeNotifier = (*Fs)(nil)
+	_ fs.Abouter        = (*Fs)(nil)
 )

backend/cache/cache_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache_test
 
@@ -50,6 +50,7 @@ const (
 	cryptedTextBase64  = "UkNMT05FAAC320i2xIee0BiNyknSPBn+Qcw3q9FhIFp3tvq6qlqvbsno3PnxmEFeJG3jDBnR/wku2gHWeQ=="     // one content
 	cryptedText2Base64 = "UkNMT05FAAATcQkVsgjBh8KafCKcr0wdTa1fMmV0U8hsCLGFoqcvxKVmvv7wx3Hf5EXxFcki2FFV4sdpmSrb9Q==" // updated content
 	cryptedText3Base64 = "UkNMT05FAAB/f7YtYKbPfmk9+OX/ffN3qG3OEdWT+z74kxCX9V/YZwJ4X2DN3HOnUC3gKQ4Gcoud5UtNvQ=="     // test content
+	letterBytes        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 )
 
 var (
@@ -132,6 +133,7 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	require.Len(t, listInner, 1)
 }
 
+/* TODO: is this testing something?
 func TestInternalVfsCache(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 30
 	testSize := int64(524288000)
@@ -229,6 +231,7 @@ func TestInternalVfsCache(t *testing.T) {
 	cacheCh <- true
 	readCh <- true
 }
+*/
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
@@ -308,7 +311,7 @@ func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, chunkSize*4+chunkSize/2)
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 
 	// write the object
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
@@ -385,27 +388,19 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// create some rand test data
 	testSize := chunkSize*4 + chunkSize/2
-	testData := runInstance.randomBytes(t, testSize)
+	testData := randStringBytes(int(testSize))
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
 	require.Equal(t, o.Size(), int64(testSize))
 	time.Sleep(time.Second * 3)
 
-	data2, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
 	require.NoError(t, err)
-	require.Equal(t, int64(len(data2)), o.Size())
-
-	// check sample of data from in-file
-	sampleStart := chunkSize / 2
-	sampleEnd := chunkSize
-	testSample := testData[sampleStart:sampleEnd]
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", sampleStart, sampleEnd, false)
-	require.NoError(t, err)
-	require.Equal(t, len(checkSample), len(testSample))
+	require.Equal(t, int64(len(checkSample)), o.Size())
 
 	for i := 0; i < len(checkSample); i++ {
-		require.Equal(t, testSample[i], checkSample[i])
+		require.Equal(t, testData[i], checkSample[i])
 	}
 }
 
@@ -424,7 +419,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 	// create some rand test data
 	testSize := chunkSize*10 + chunkSize/2
-	testData := runInstance.randomBytes(t, testSize)
+	testData := randStringBytes(int(testSize))
 
 	// write the object
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
@@ -447,7 +442,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
@@ -579,6 +574,93 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
+	id := fmt.Sprintf("tincep%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	if !runInstance.wrappedIsExternal {
+		t.Skipf("Not external")
+	}
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+
+	srcName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "test")
+	dstName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "test2")
+	// create some rand test data
+	var testData []byte
+	if runInstance.rootIsCrypt {
+		testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64)
+		require.NoError(t, err)
+	} else {
+		testData = []byte("test content")
+	}
+	err = rootFs.Mkdir("test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("test/one")
+	require.NoError(t, err)
+	srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
+
+	// list in mount
+	_, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	_, err = runInstance.list(t, rootFs, "test/one")
+	require.NoError(t, err)
+
+	found := boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.True(t, found)
+	boltDb.Purge()
+	found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.False(t, found)
+
+	// move file
+	_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
+	require.NoError(t, err)
+
+	err = runInstance.retryBlock(func() error {
+		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
+		if !found {
+			log.Printf("not found /test")
+			return errors.Errorf("not found /test")
+		}
+		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one")))
+		if !found {
+			log.Printf("not found /test/one")
+			return errors.Errorf("not found /test/one")
+		}
+		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"), runInstance.encryptRemoteIfNeeded(t, "test2")))
+		if !found {
+			log.Printf("not found /test/one/test2")
+			return errors.Errorf("not found /test/one/test2")
+		}
+		li, err := runInstance.list(t, rootFs, "test/one")
+		if err != nil {
+			log.Printf("err: %v", err)
+			return err
+		}
+		if len(li) != 1 {
+			log.Printf("not expected listing /test/one: %v", li)
+			return errors.Errorf("not expected listing /test/one: %v", li)
+		}
+		if fi, ok := li[0].(os.FileInfo); ok {
+			if fi.Name() != "test2" {
+				log.Printf("not expected name: %v", fi.Name())
+				return errors.Errorf("not expected name: %v", fi.Name())
+			}
+		} else if di, ok := li[0].(fs.DirEntry); ok {
+			if di.Remote() != "test/one/test2" {
+				log.Printf("not expected remote: %v", di.Remote())
+				return errors.Errorf("not expected remote: %v", di.Remote())
+			}
+		} else {
+			log.Printf("unexpected listing: %v", li)
+			return errors.Errorf("unexpected listing: %v", li)
+		}
+		log.Printf("complete listing /test/one/test2")
+		return nil
+	}, 12, time.Second*10)
+	require.NoError(t, err)
+}
+
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
@@ -589,7 +671,7 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
@@ -623,13 +705,16 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	if !runInstance.useMount {
 		t.Skipf("needs mount")
 	}
+	if !runInstance.wrappedIsExternal {
+		t.Skipf("needs drive")
+	}
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
@@ -660,6 +745,31 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
+	li1, err := runInstance.list(t, rootFs, "")
+
+	// create some rand test data
+	testData2 := randStringBytes(int(chunkSize))
+	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
+
+	// list should have 1 item only
+	li1, err = runInstance.list(t, rootFs, "")
+	require.Len(t, li1, 1)
+
+	m = make(map[string]string)
+	res2, err := http.Post("http://localhost:5572/cache/expire?remote=/", "application/json; charset=utf-8", strings.NewReader(""))
+	require.NoError(t, err)
+	defer func() {
+		_ = res2.Body.Close()
+	}()
+	_ = json.NewDecoder(res2.Body).Decode(&m)
+	require.Contains(t, m, "status")
+	require.Contains(t, m, "message")
+	require.Equal(t, "ok", m["status"])
+	require.Contains(t, m["message"], "cached directory cleared")
+
+	// list should have 2 items now
+	li2, err := runInstance.list(t, rootFs, "")
+	require.Len(t, li2, 2)
 }
 
 func TestInternalCacheWrites(t *testing.T) {
@@ -673,7 +783,7 @@ func TestInternalCacheWrites(t *testing.T) {
 
 	// create some rand test data
 	earliestTime := time.Now()
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 	expectedTs := time.Now()
 	ts, err := boltDb.GetChunkTs(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "data.bin")), 0)
@@ -692,7 +802,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 	totalChunks := 20
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (int64(totalChunks-1)*chunkSize + chunkSize/2))
+	testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 	o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
@@ -805,458 +915,6 @@ func TestInternalBug2117(t *testing.T) {
 	require.Len(t, di, 4)
 }
 
-func TestInternalUploadTempDirCreated(t *testing.T) {
-	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
-	require.NoError(t, err)
-}
-
-func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
-	// create some rand test data
-	testSize := int64(524288000)
-	testReader := runInstance.randomReader(t, testSize)
-	bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
-	runInstance.writeRemoteReader(t, rootFs, "one", testReader)
-	// validate that it exists in temp fs
-	ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
-	require.NoError(t, err)
-
-	if runInstance.rootIsCrypt {
-		require.Equal(t, int64(524416032), ti.Size())
-	} else {
-		require.Equal(t, testSize, ti.Size())
-	}
-	de1, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
-	require.Len(t, de1, 1)
-
-	runInstance.completeBackgroundUpload(t, "one", bu)
-	// check if it was removed from temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
-	require.True(t, os.IsNotExist(err))
-
-	// check if it can be read
-	data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
-	require.NoError(t, err)
-	require.Len(t, data2, 1024)
-}
-
-func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
-	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
-}
-
-func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
-	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
-}
-
-func TestInternalUploadMoveExistingFile(t *testing.T) {
-	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	err := rootFs.Mkdir("one")
-	require.NoError(t, err)
-	err = rootFs.Mkdir("one/test")
-	require.NoError(t, err)
-	err = rootFs.Mkdir("second")
-	require.NoError(t, err)
-
-	// create some rand test data
-	testSize := int64(10485760)
-	testReader := runInstance.randomReader(t, testSize)
-	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
-	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
-
-	de1, err := runInstance.list(t, rootFs, "one/test")
-	require.NoError(t, err)
-	require.Len(t, de1, 1)
-
-	time.Sleep(time.Second * 5)
-	_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
-	require.NoError(t, err)
-
-	err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
-	require.NoError(t, err)
-
-	// check if it can be read
-	de1, err = runInstance.list(t, rootFs, "second/test")
-	require.NoError(t, err)
-	require.Len(t, de1, 1)
-}
-
-func TestInternalUploadQueueMoreFiles(t *testing.T) {
-	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	err := rootFs.Mkdir("test")
-	require.NoError(t, err)
-	minSize := 5242880
-	maxSize := 10485760
-	totalFiles := 10
-	rand.Seed(time.Now().Unix())
-
-	lastFile := ""
-	for i := 0; i < totalFiles; i++ {
-		size := int64(rand.Intn(maxSize-minSize) + minSize)
-		testReader := runInstance.randomReader(t, size)
-		remote := "test/" + strconv.Itoa(i) + ".bin"
-		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
-
-		// validate that it exists in temp fs
-		ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
-		require.NoError(t, err)
-		require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
-
-		if runInstance.wrappedIsExternal && i < totalFiles-1 {
-			time.Sleep(time.Second * 3)
-		}
-		lastFile = remote
-	}
-
-	// check if cache lists all files, likely temp upload didn't finish yet
-	de1, err := runInstance.list(t, rootFs, "test")
-	require.NoError(t, err)
-	require.Len(t, de1, totalFiles)
-
-	// wait for background uploader to do its thing
-	runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
-
-	// retry until we have no more temp files and fail if they don't go down to 0
-	tf, err := ioutil.ReadDir(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
-	require.NoError(t, err)
-	require.Len(t, tf, 0)
-
-	// check if cache lists all files
-	de1, err = runInstance.list(t, rootFs, "test")
-	require.NoError(t, err)
-	require.Len(t, de1, totalFiles)
-}
-
-func TestInternalUploadTempFileOperations(t *testing.T) {
-	id := "tiutfo"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	boltDb.PurgeTempUploads()
-
-	// create some rand test data
-	runInstance.mkdir(t, rootFs, "test")
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// check if it can be read
-	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
-	require.NoError(t, err)
-	require.Equal(t, []byte("one content"), data1)
-	// validate that it exists in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-
-	// test DirMove - allowed
-	err = runInstance.dirMove(t, rootFs, "test", "second")
-	if err != errNotSupported {
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.Error(t, err)
-		_, err = rootFs.NewObject("second/one")
-		require.NoError(t, err)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.Error(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
-		require.NoError(t, err)
-		started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
-		require.Error(t, err)
-		started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
-		require.NoError(t, err)
-		require.False(t, started)
-		runInstance.mkdir(t, rootFs, "test")
-		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-	}
-
-	// test Rmdir - allowed
-	err = runInstance.rm(t, rootFs, "test")
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "directory not empty")
-	_, err = rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	// validate that it exists in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-	started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
-	require.False(t, started)
-	require.NoError(t, err)
-
-	// test Move/Rename -- allowed
-	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
-	if err != errNotSupported {
-		require.NoError(t, err)
-		// try to read from it
-		_, err = rootFs.NewObject("test/one")
-		require.Error(t, err)
-		_, err = rootFs.NewObject("test/second")
-		require.NoError(t, err)
-		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
-		require.NoError(t, err)
-		require.Equal(t, []byte("one content"), data2)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.Error(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
-		require.NoError(t, err)
-		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-	}
-
-	// test Copy -- allowed
-	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
-	if err != errNotSupported {
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/third")
-		require.NoError(t, err)
-		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
-		require.NoError(t, err)
-		require.Equal(t, []byte("one content"), data2)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
-		require.NoError(t, err)
-	}
-
-	// test Remove -- allowed
-	err = runInstance.rm(t, rootFs, "test/one")
-	require.NoError(t, err)
-	_, err = rootFs.NewObject("test/one")
-	require.Error(t, err)
-	// validate that it doesn't exist in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.Error(t, err)
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// test Update -- allowed
-	firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	require.NoError(t, err)
-	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
-	require.NoError(t, err)
-	obj2, err := rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
-	require.Equal(t, "one content updated", string(data2))
-	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-	if runInstance.rootIsCrypt {
-		require.Equal(t, int64(67), tmpInfo.Size())
-	} else {
-		require.Equal(t, int64(len(data2)), tmpInfo.Size())
-	}
-
-	// test SetModTime -- allowed
-	secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	require.NoError(t, err)
-	require.NotEqual(t, secondModTime, firstModTime)
-	require.NotEqual(t, time.Time{}, firstModTime)
-	require.NotEqual(t, time.Time{}, secondModTime)
-}
-
-func TestInternalUploadUploadingFileOperations(t *testing.T) {
-	id := "tiuufo"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	boltDb.PurgeTempUploads()
-
-	// create some rand test data
-	runInstance.mkdir(t, rootFs, "test")
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// check if it can be read
-	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
-	require.NoError(t, err)
-	require.Equal(t, []byte("one content"), data1)
-	// validate that it exists in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-
-	err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
-	require.NoError(t, err)
-
-	// test DirMove
-	err = runInstance.dirMove(t, rootFs, "test", "second")
-	if err != errNotSupported {
-		require.Error(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
-		require.Error(t, err)
-	}
-
-	// test Rmdir
-	err = runInstance.rm(t, rootFs, "test")
-	require.Error(t, err)
-	_, err = rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	// validate that it doesn't exist in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-
-	// test Move/Rename
-	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
-	if err != errNotSupported {
-		require.Error(t, err)
-		// try to read from it
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/second")
-		require.Error(t, err)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
-		require.Error(t, err)
-	}
-
-	// test Copy -- allowed
-	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
-	if err != errNotSupported {
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/third")
-		require.NoError(t, err)
-		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
-		require.NoError(t, err)
-		require.Equal(t, []byte("one content"), data2)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
-		require.NoError(t, err)
-	}
-
-	// test Remove
-	err = runInstance.rm(t, rootFs, "test/one")
-	require.Error(t, err)
-	_, err = rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	// validate that it doesn't exist in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// test Update - this seems to work. Why? FIXME
-	//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	//require.NoError(t, err)
-	//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
-	//	data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
-	//	require.Equal(t, "one content", string(data2))
-	//
-	//	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	//	require.NoError(t, err)
-	//	if runInstance.rootIsCrypt {
-	//		require.Equal(t, int64(67), tmpInfo.Size())
-	//	} else {
-	//		require.Equal(t, int64(len(data2)), tmpInfo.Size())
-	//	}
-	//})
-	//require.Error(t, err)
-
-	// test SetModTime -- seems to work cause of previous
-	//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	//require.NoError(t, err)
-	//require.Equal(t, secondModTime, firstModTime)
-	//require.NotEqual(t, time.Time{}, firstModTime)
-	//require.NotEqual(t, time.Time{}, secondModTime)
-}
-
-// FIXME, enable this when mount is sorted out
-//func TestInternalFilesMissingInMount1904(t *testing.T) {
-//	t.Skip("Not yet")
-//	if runtime.GOOS == "windows" {
-//		t.Skip("Not yet")
-//	}
-//	id := "tifm1904"
-//	rootFs, _ := newCacheFs(t, RemoteName, id, false,
-//		map[string]string{"chunk_size": "5M", "info_age": "1m", "chunk_total_size": "500M", "cache-writes": "true"})
-//	mntPoint := path.Join("/tmp", "tifm1904-mnt")
-//	testPoint := path.Join(mntPoint, id)
-//	checkOutput := "1 10 100 11 12 13 14 15 16 17 18 19 2 20 21 22 23 24 25 26 27 28 29 3 30 31 32 33 34 35 36 37 38 39 4 40 41 42 43 44 45 46 47 48 49 5 50 51 52 53 54 55 56 57 58 59 6 60 61 62 63 64 65 66 67 68 69 7 70 71 72 73 74 75 76 77 78 79 8 80 81 82 83 84 85 86 87 88 89 9 90 91 92 93 94 95 96 97 98 99 "
-//
-//	_ = os.MkdirAll(mntPoint, os.ModePerm)
-//
-//	list, err := rootFs.List("")
-//	require.NoError(t, err)
-//	found := false
-//	list.ForDir(func(d fs.Directory) {
-//		if strings.Contains(d.Remote(), id) {
-//			found = true
-//		}
-//	})
-//
-//	if !found {
-//		t.Skip("Test folder '%v' doesn't exist", id)
-//	}
-//
-//	mountFs(t, rootFs, mntPoint)
-//	defer unmountFs(t, mntPoint)
-//
-//	for i := 1; i <= 2; i++ {
-//		out, err := exec.Command("ls", testPoint).Output()
-//		require.NoError(t, err)
-//		require.Equal(t, checkOutput, strings.Replace(string(out), "\n", " ", -1))
-//		t.Logf("root path has all files")
-//		_ = writeObjectString(t, rootFs, path.Join(id, strconv.Itoa(i), strconv.Itoa(i), "one_file"), "one content")
-//
-//		for j := 1; j <= 100; j++ {
-//			out, err := exec.Command("ls", path.Join(testPoint, strconv.Itoa(j))).Output()
-//			require.NoError(t, err)
-//			require.Equal(t, checkOutput, strings.Replace(string(out), "\n", " ", -1), "'%v' doesn't match", j)
-//		}
-//		obj, err := rootFs.NewObject(path.Join(id, strconv.Itoa(i), strconv.Itoa(i), "one_file"))
-//		require.NoError(t, err)
-//		err = obj.Remove()
-//		require.NoError(t, err)
-//		t.Logf("folders contain all the files")
-//
-//		out, err = exec.Command("date").Output()
-//		require.NoError(t, err)
-//		t.Logf("check #%v date: '%v'", i, strings.Replace(string(out), "\n", " ", -1))
-//
-//		if i < 2 {
-//			time.Sleep(time.Second * 60)
-//		}
-//	}
-//}
-
 // run holds the remotes for a test run
 type run struct {
 	okDiff            time.Duration
@@ -1395,7 +1053,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		config.FileSet(remote, "type", "cache")
 		config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
 	} else {
-		remoteType := fs.ConfigFileGet(remote, "type", "")
+		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -1406,14 +1064,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				config.FileSet(remote, "password", cryptPassword1)
 				config.FileSet(remote, "password2", cryptPassword2)
 			}
-			remoteRemote := fs.ConfigFileGet(remote, "remote", "")
+			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := fs.ConfigFileGet(remoteWrapping, "type", "")
+			remoteType := config.FileGet(remoteWrapping, "type", "")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -1504,15 +1162,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
 	}
 }
 
-func (r *run) randomBytes(t *testing.T, size int64) []byte {
-	testData := make([]byte, size)
-	testSize, err := rand.Read(testData)
-	require.Equal(t, size, int64(len(testData)))
-	require.Equal(t, size, int64(testSize))
-	require.NoError(t, err)
-	return testData
-}
-
 func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
@@ -1521,12 +1170,12 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {
-		data := r.randomBytes(t, chunk)
+		data := randStringBytes(int(chunk))
 		_, _ = f.Write(data)
 	}
-	data := r.randomBytes(t, int64(left))
+	data := randStringBytes(int(left))
 	_, _ = f.Write(data)
-	_, _ = f.Seek(int64(0), 0)
+	_, _ = f.Seek(int64(0), io.SeekStart)
 	r.tempFiles = append(r.tempFiles, f)
 
 	return f
@@ -1535,7 +1184,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string {
 	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
 	// create some rand test data
-	testData := r.randomBytes(t, size)
+	testData := randStringBytes(int(size))
 
 	r.writeRemoteBytes(t, f, remote, testData)
 	return remote
@@ -1544,7 +1193,7 @@ func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64
 func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object {
 	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
 	// create some rand test data
-	testData := r.randomBytes(t, size)
+	testData := randStringBytes(int(size))
 
 	return r.writeObjectBytes(t, f, remote, testData)
 }
@@ -1653,7 +1302,7 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 		if err != nil {
 			return checkSample, err
 		}
-		_, _ = f.Seek(offset, 0)
+		_, _ = f.Seek(offset, io.SeekStart)
 		totalRead, err := io.ReadFull(f, checkSample)
 		checkSample = checkSample[:totalRead]
 		if err == io.EOF || err == io.ErrUnexpectedEOF {
@@ -1662,9 +1311,6 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 		if err != nil {
 			return checkSample, err
 		}
-		if !noLengthCheck && size != int64(totalRead) {
-			return checkSample, errors.Errorf("read size doesn't match expected: %v <> %v", totalRead, size)
-		}
 	} else {
 		co, err := f.NewObject(remote)
 		if err != nil {
@@ -1688,7 +1334,7 @@ func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLe
 		err = nil
 		checkSample = checkSample[:totalRead]
 	}
-	require.NoError(t, err)
+	require.NoError(t, err, "with string -%v-", string(checkSample))
 	_ = reader.Close()
 	return checkSample
 }
@@ -1897,16 +1543,11 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		if err != nil {
 			return err
 		}
-		_, err = f.WriteString(data + append)
-		if err != nil {
+		defer func() {
 			_ = f.Close()
-			return err
-		}
-		err = f.Close()
-		if err != nil {
-			return err
-		}
-		r.vfs.WaitForWriters(10 * time.Second)
+			r.vfs.WaitForWriters(10 * time.Second)
+		}()
+		_, err = f.WriteString(data + append)
 	} else {
 		obj1, err := rootFs.NewObject(src)
 		if err != nil {
@@ -1916,9 +1557,6 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		r := bytes.NewReader(data1)
 		objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
 		err = obj1.Update(r, objInfo1)
-		if err != nil {
-			return err
-		}
 	}
 
 	return err
@@ -2055,6 +1693,14 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	return nil, errors.New("didn't found a cache fs")
 }
 
+func randStringBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 var (
 	_ fs.Fs = (*cache.Fs)(nil)
 	_ fs.Fs = (*local.Fs)(nil)

backend/cache/cache_mount_unix_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,!windows,go1.7
+// +build !plan9,!windows
 
 package cache_test
 
@@ -23,7 +23,7 @@ func (r *run) mountFs(t *testing.T, f fs.Fs) {
 		fuse.FSName(device), fuse.VolumeName(device),
 		fuse.NoAppleDouble(),
 		fuse.NoAppleXattr(),
-		fuse.AllowOther(),
+		//fuse.AllowOther(),
 	}
 	err := os.MkdirAll(r.mntDir, os.ModePerm)
 	require.NoError(t, err)

backend/cache/cache_mount_windows_test.go

@@ -1,4 +1,4 @@
-// +build windows,go1.7
+// +build windows
 
 package cache_test
 

backend/cache/cache_test.go

@@ -1,9 +1,6 @@
 // Test Cache filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache_test
 
@@ -12,68 +9,13 @@ import (
 
 	"github.com/ncw/rclone/backend/cache"
 	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*cache.Object)(nil))
-	fstests.RemoteName = "TestCache:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestCache:",
+		NilObject:  (*cache.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/cache/cache_unsupported.go

@@ -1,6 +1,6 @@
 // Build for cache for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 !go1.7
+// +build plan9
 
 package cache

backend/cache/cache_upload_test.go

@@ -0,0 +1,455 @@
+// +build !plan9
+
+package cache_test
+
+import (
+	"math/rand"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+	"time"
+
+	"fmt"
+
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInternalUploadTempDirCreated(t *testing.T) {
+	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
+	require.NoError(t, err)
+}
+
+func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
+	// create some rand test data
+	testSize := int64(524288000)
+	testReader := runInstance.randomReader(t, testSize)
+	bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
+	runInstance.writeRemoteReader(t, rootFs, "one", testReader)
+	// validate that it exists in temp fs
+	ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.NoError(t, err)
+
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(524416032), ti.Size())
+	} else {
+		require.Equal(t, testSize, ti.Size())
+	}
+	de1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	runInstance.completeBackgroundUpload(t, "one", bu)
+	// check if it was removed from temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if it can be read
+	data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
+	require.NoError(t, err)
+	require.Len(t, data2, 1024)
+}
+
+func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadMoveExistingFile(t *testing.T) {
+	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(10485760)
+	testReader := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	time.Sleep(time.Second * 5)
+	//_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	//require.NoError(t, err)
+
+	err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
+	require.NoError(t, err)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadTempPathCleaned(t *testing.T) {
+	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(1048576)
+	testReader := runInstance.randomReader(t, testSize)
+	testReader2 := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2)
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second")))
+	require.False(t, os.IsNotExist(err))
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin")))
+	require.True(t, os.IsNotExist(err))
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadQueueMoreFiles(t *testing.T) {
+	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("test")
+	require.NoError(t, err)
+	minSize := 5242880
+	maxSize := 10485760
+	totalFiles := 10
+	rand.Seed(time.Now().Unix())
+
+	lastFile := ""
+	for i := 0; i < totalFiles; i++ {
+		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		testReader := runInstance.randomReader(t, size)
+		remote := "test/" + strconv.Itoa(i) + ".bin"
+		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
+
+		// validate that it exists in temp fs
+		ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
+		require.NoError(t, err)
+		require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
+
+		if runInstance.wrappedIsExternal && i < totalFiles-1 {
+			time.Sleep(time.Second * 3)
+		}
+		lastFile = remote
+	}
+
+	// check if cache lists all files, likely temp upload didn't finish yet
+	de1, err := runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+
+	// wait for background uploader to do its thing
+	runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
+
+	// retry until we have no more temp files and fail if they don't go down to 0
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if cache lists all files
+	de1, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+}
+
+func TestInternalUploadTempFileOperations(t *testing.T) {
+	id := "tiutfo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove - allowed
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("second/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.NoError(t, err)
+		_, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+		require.Error(t, err)
+		var started bool
+		started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
+		require.NoError(t, err)
+		require.False(t, started)
+		runInstance.mkdir(t, rootFs, "test")
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Rmdir - allowed
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "directory not empty")
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+	require.False(t, started)
+	require.NoError(t, err)
+
+	// test Move/Rename -- allowed
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.NoError(t, err)
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove -- allowed
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.NoError(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.Error(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.Error(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update -- allowed
+	firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
+	require.NoError(t, err)
+	obj2, err := rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
+	require.Equal(t, "one content updated", string(data2))
+	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(67), tmpInfo.Size())
+	} else {
+		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	}
+
+	// test SetModTime -- allowed
+	secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	require.NotEqual(t, secondModTime, firstModTime)
+	require.NotEqual(t, time.Time{}, firstModTime)
+	require.NotEqual(t, time.Time{}, secondModTime)
+}
+
+func TestInternalUploadUploadingFileOperations(t *testing.T) {
+	id := "tiuufo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.Error(t, err)
+	}
+
+	// test Rmdir
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test Move/Rename
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.Error(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.Error(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.Error(t, err)
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update - this seems to work. Why? FIXME
+	//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
+	//	data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
+	//	require.Equal(t, "one content", string(data2))
+	//
+	//	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	//	require.NoError(t, err)
+	//	if runInstance.rootIsCrypt {
+	//		require.Equal(t, int64(67), tmpInfo.Size())
+	//	} else {
+	//		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	//	}
+	//})
+	//require.Error(t, err)
+
+	// test SetModTime -- seems to work cause of previous
+	//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//require.Equal(t, secondModTime, firstModTime)
+	//require.NotEqual(t, time.Time{}, firstModTime)
+	//require.NotEqual(t, time.Time{}, secondModTime)
+}

backend/cache/directory.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 

backend/cache/handle.go

@@ -1,11 +1,10 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 
 import (
 	"fmt"
 	"io"
-	"os"
 	"sync"
 	"time"
 
@@ -146,34 +145,16 @@ func (r *Handle) scaleWorkers(desired int) {
 	}
 }
 
-func (r *Handle) requestExternalConfirmation() {
-	// if there's no external confirmation available
-	// then we skip this step
-	if len(r.workers) >= r.cacheFs().totalMaxWorkers ||
-		!r.cacheFs().plexConnector.isConnected() {
-		return
-	}
-	go r.cacheFs().plexConnector.isPlayingAsync(r.cachedObject, r.confirmReading)
-}
-
 func (r *Handle) confirmExternalReading() {
 	// if we have a max value of workers
-	// or there's no external confirmation available
 	// then we skip this step
-	if len(r.workers) >= r.cacheFs().totalMaxWorkers ||
-		!r.cacheFs().plexConnector.isConnected() {
+	if len(r.workers) > 1 ||
+		!r.cacheFs().plexConnector.isConfigured() {
 		return
 	}
-
-	select {
-	case confirmed := <-r.confirmReading:
-		if !confirmed {
-			return
-		}
-	default:
+	if !r.cacheFs().plexConnector.isPlaying(r.cachedObject) {
 		return
 	}
-
 	fs.Infof(r, "confirmed reading by external reader")
 	r.scaleWorkers(r.cacheFs().totalMaxWorkers)
 }
@@ -209,8 +190,6 @@ func (r *Handle) queueOffset(offset int64) {
 			r.seenOffsets[o] = true
 			r.preloadQueue <- o
 		}
-
-		r.requestExternalConfirmation()
 	}
 }
 
@@ -274,7 +253,7 @@ func (r *Handle) getChunk(chunkStart int64) ([]byte, error) {
 
 	// first chunk will be aligned with the start
 	if offset > 0 {
-		if offset >= int64(len(data)) {
+		if offset > int64(len(data)) {
 			fs.Errorf(r, "unexpected conditions during reading. current position: %v, current chunk position: %v, current chunk size: %v, offset: %v, chunk size: %v, file size: %v",
 				r.offset, chunkStart, len(data), offset, r.cacheFs().chunkSize, r.cachedObject.Size())
 			return nil, io.ErrUnexpectedEOF
@@ -294,7 +273,6 @@ func (r *Handle) Read(p []byte) (n int, err error) {
 	// first reading
 	if !r.reading {
 		r.reading = true
-		r.requestExternalConfirmation()
 	}
 	// reached EOF
 	if r.offset >= r.cachedObject.Size() {
@@ -302,8 +280,10 @@ func (r *Handle) Read(p []byte) (n int, err error) {
 	}
 	currentOffset := r.offset
 	buf, err = r.getChunk(currentOffset)
-	if err != nil && len(buf) == 0 {
+	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		fs.Errorf(r, "(%v/%v) error (%v) response", currentOffset, r.cachedObject.Size(), err)
+	}
+	if len(buf) == 0 && err != io.ErrUnexpectedEOF {
 		return 0, io.EOF
 	}
 	readSize := copy(p, buf)
@@ -332,6 +312,7 @@ func (r *Handle) Close() error {
 			waitIdx++
 		}
 	}
+	r.memory.db.Flush()
 
 	fs.Debugf(r, "cache reader closed %v", r.offset)
 	return nil
@@ -344,13 +325,13 @@ func (r *Handle) Seek(offset int64, whence int) (int64, error) {
 
 	var err error
 	switch whence {
-	case os.SEEK_SET:
+	case io.SeekStart:
 		fs.Debugf(r, "moving offset set from %v to %v", r.offset, offset)
 		r.offset = offset
-	case os.SEEK_CUR:
+	case io.SeekCurrent:
 		fs.Debugf(r, "moving offset cur from %v to %v", r.offset, r.offset+offset)
 		r.offset += offset
-	case os.SEEK_END:
+	case io.SeekEnd:
 		fs.Debugf(r, "moving offset end (%v) from %v to %v", r.cachedObject.Size(), r.offset, r.cachedObject.Size()+offset)
 		r.offset = r.cachedObject.Size() + offset
 	default:
@@ -399,10 +380,10 @@ func (w *worker) reader(offset, end int64, closeOpen bool) (io.ReadCloser, error
 
 	if !closeOpen {
 		if do, ok := r.(fs.RangeSeeker); ok {
-			_, err = do.RangeSeek(offset, os.SEEK_SET, end-offset)
+			_, err = do.RangeSeek(offset, io.SeekStart, end-offset)
 			return r, err
 		} else if do, ok := r.(io.Seeker); ok {
-			_, err = do.Seek(offset, os.SEEK_SET)
+			_, err = do.Seek(offset, io.SeekStart)
 			return r, err
 		}
 	}
@@ -464,7 +445,6 @@ func (w *worker) run() {
 					continue
 				}
 			}
-			err = nil
 		} else {
 			if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
 				continue
@@ -511,7 +491,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	}
 
 	data = make([]byte, chunkEnd-chunkStart)
-	sourceRead := 0
+	var sourceRead int
 	sourceRead, err = io.ReadFull(w.rc, data)
 	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err)
@@ -648,6 +628,23 @@ func (b *backgroundWriter) run() {
 			fs.Errorf(remote, "background upload: %v", err)
 			continue
 		}
+		// clean empty dirs up to root
+		thisDir := cleanPath(path.Dir(remote))
+		for thisDir != "" {
+			thisList, err := b.fs.tempFs.List(thisDir)
+			if err != nil {
+				break
+			}
+			if len(thisList) > 0 {
+				break
+			}
+			err = b.fs.tempFs.Rmdir(thisDir)
+			fs.Debugf(thisDir, "cleaned from temp path")
+			if err != nil {
+				break
+			}
+			thisDir = cleanPath(path.Dir(thisDir))
+		}
 		fs.Infof(remote, "background upload: uploaded entry")
 		err = b.fs.cache.removePendingUpload(absPath)
 		if err != nil && !strings.Contains(err.Error(), "pending upload not found") {

backend/cache/object.go

@@ -1,10 +1,9 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 
 import (
 	"io"
-	"os"
 	"path"
 	"sync"
 	"time"
@@ -223,7 +222,7 @@ func (o *Object) Open(options ...fs.OpenOption) (io.ReadCloser, error) {
 		case *fs.RangeOption:
 			offset, limit = x.Decode(o.Size())
 		}
-		_, err = cacheReader.Seek(offset, os.SEEK_SET)
+		_, err = cacheReader.Seek(offset, io.SeekStart)
 		if err != nil {
 			return nil, err
 		}
@@ -257,6 +256,8 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 
 	// deleting cached chunks and info to be replaced with new ones
 	_ = o.CacheFs.cache.RemoveObject(o.abs())
+	// advertise to ChangeNotify if wrapped doesn't do that
+	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)
 
 	o.CacheModTime = src.ModTime().UnixNano()
 	o.CacheSize = src.Size()

backend/cache/plex.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 
@@ -17,21 +17,49 @@ import (
 
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
+	"github.com/patrickmn/go-cache"
+	"golang.org/x/net/websocket"
 )
 
 const (
 	// defPlexLoginURL is the default URL for Plex login
-	defPlexLoginURL = "https://plex.tv/users/sign_in.json"
+	defPlexLoginURL        = "https://plex.tv/users/sign_in.json"
+	defPlexNotificationURL = "%s/:/websockets/notifications?X-Plex-Token=%s"
 )
 
+// PlaySessionStateNotification is part of the API response of Plex
+type PlaySessionStateNotification struct {
+	SessionKey       string `json:"sessionKey"`
+	GUID             string `json:"guid"`
+	Key              string `json:"key"`
+	ViewOffset       int64  `json:"viewOffset"`
+	State            string `json:"state"`
+	TranscodeSession string `json:"transcodeSession"`
+}
+
+// NotificationContainer is part of the API response of Plex
+type NotificationContainer struct {
+	Type             string                         `json:"type"`
+	Size             int                            `json:"size"`
+	PlaySessionState []PlaySessionStateNotification `json:"PlaySessionStateNotification"`
+}
+
+// PlexNotification is part of the API response of Plex
+type PlexNotification struct {
+	Container NotificationContainer `json:"NotificationContainer"`
+}
+
 // plexConnector is managing the cache integration with Plex
 type plexConnector struct {
-	url      *url.URL
-	username string
-	password string
-	token    string
-	f        *Fs
-	mu       sync.Mutex
+	url        *url.URL
+	username   string
+	password   string
+	token      string
+	f          *Fs
+	mu         sync.Mutex
+	running    bool
+	runningMu  sync.Mutex
+	stateCache *cache.Cache
 }
 
 // newPlexConnector connects to a Plex server and generates a token
@@ -42,11 +70,12 @@ func newPlexConnector(f *Fs, plexURL, username, password string) (*plexConnector
 	}
 
 	pc := &plexConnector{
-		f:        f,
-		url:      u,
-		username: username,
-		password: password,
-		token:    "",
+		f:          f,
+		url:        u,
+		username:   username,
+		password:   password,
+		token:      "",
+		stateCache: cache.New(time.Hour, time.Minute),
 	}
 
 	return pc, nil
@@ -60,14 +89,83 @@ func newPlexConnectorWithToken(f *Fs, plexURL, token string) (*plexConnector, er
 	}
 
 	pc := &plexConnector{
-		f:     f,
-		url:   u,
-		token: token,
+		f:          f,
+		url:        u,
+		token:      token,
+		stateCache: cache.New(time.Hour, time.Minute),
 	}
+	pc.listenWebsocket()
 
 	return pc, nil
 }
 
+func (p *plexConnector) closeWebsocket() {
+	p.runningMu.Lock()
+	defer p.runningMu.Unlock()
+	fs.Infof("plex", "stopped Plex watcher")
+	p.running = false
+}
+
+func (p *plexConnector) listenWebsocket() {
+	p.runningMu.Lock()
+	defer p.runningMu.Unlock()
+
+	u := strings.Replace(p.url.String(), "http://", "ws://", 1)
+	u = strings.Replace(u, "https://", "wss://", 1)
+	conn, err := websocket.Dial(fmt.Sprintf(defPlexNotificationURL, strings.TrimRight(u, "/"), p.token),
+		"", "http://localhost")
+	if err != nil {
+		fs.Errorf("plex", "%v", err)
+		return
+	}
+
+	p.running = true
+	go func() {
+		for {
+			if !p.isConnected() {
+				break
+			}
+
+			notif := &PlexNotification{}
+			err := websocket.JSON.Receive(conn, notif)
+			if err != nil {
+				fs.Debugf("plex", "%v", err)
+				p.closeWebsocket()
+				break
+			}
+			// we're only interested in play events
+			if notif.Container.Type == "playing" {
+				// we loop through each of them
+				for _, v := range notif.Container.PlaySessionState {
+					// event type of playing
+					if v.State == "playing" {
+						// if it's not cached get the details and cache them
+						if _, found := p.stateCache.Get(v.Key); !found {
+							req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", p.url.String(), v.Key), nil)
+							if err != nil {
+								continue
+							}
+							p.fillDefaultHeaders(req)
+							resp, err := http.DefaultClient.Do(req)
+							if err != nil {
+								continue
+							}
+							var data []byte
+							data, err = ioutil.ReadAll(resp.Body)
+							if err != nil {
+								continue
+							}
+							p.stateCache.Set(v.Key, data, cache.DefaultExpiration)
+						}
+					} else if v.State == "stopped" {
+						p.stateCache.Delete(v.Key)
+					}
+				}
+			}
+		}
+	}()
+}
+
 // fillDefaultHeaders will add common headers to requests
 func (p *plexConnector) fillDefaultHeaders(req *http.Request) {
 	req.Header.Add("X-Plex-Client-Identifier", fmt.Sprintf("rclone (%v)", p.f.String()))
@@ -115,13 +213,16 @@ func (p *plexConnector) authenticate() error {
 		config.SaveConfig()
 		fs.Infof(p.f.Name(), "Connected to Plex server: %v", p.url.String())
 	}
+	p.listenWebsocket()
 
 	return nil
 }
 
 // isConnected checks if this rclone is authenticated to Plex
 func (p *plexConnector) isConnected() bool {
-	return p.token != ""
+	p.runningMu.Lock()
+	defer p.runningMu.Unlock()
+	return p.running
 }
 
 // isConfigured checks if this rclone is configured to use a Plex server
@@ -131,6 +232,9 @@ func (p *plexConnector) isConfigured() bool {
 
 func (p *plexConnector) isPlaying(co *Object) bool {
 	var err error
+	if !p.isConnected() {
+		p.listenWebsocket()
+	}
 
 	remote := co.Remote()
 	if cr, yes := p.f.isWrappedByCrypt(); yes {
@@ -142,64 +246,8 @@ func (p *plexConnector) isPlaying(co *Object) bool {
 	}
 
 	isPlaying := false
-	req, err := http.NewRequest("GET", fmt.Sprintf("%s/status/sessions", p.url.String()), nil)
-	if err != nil {
-		return false
-	}
-	p.fillDefaultHeaders(req)
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return false
-	}
-	var data map[string]interface{}
-	err = json.NewDecoder(resp.Body).Decode(&data)
-	if err != nil {
-		return false
-	}
-	sizeGen, ok := get(data, "MediaContainer", "size")
-	if !ok {
-		return false
-	}
-	size, ok := sizeGen.(float64)
-	if !ok || size < float64(1) {
-		return false
-	}
-	videosGen, ok := get(data, "MediaContainer", "Video")
-	if !ok {
-		fs.Debugf("plex", "empty videos: %v", data)
-		return false
-	}
-	videos, ok := videosGen.([]interface{})
-	if !ok || len(videos) < 1 {
-		fs.Debugf("plex", "empty videos: %v", data)
-		return false
-	}
-	for _, v := range videos {
-		keyGen, ok := get(v, "key")
-		if !ok {
-			fs.Debugf("plex", "failed to find: key")
-			continue
-		}
-		key, ok := keyGen.(string)
-		if !ok {
-			fs.Debugf("plex", "failed to understand: key")
-			continue
-		}
-		req, err := http.NewRequest("GET", fmt.Sprintf("%s%s", p.url.String(), key), nil)
-		if err != nil {
-			return false
-		}
-		p.fillDefaultHeaders(req)
-		resp, err := http.DefaultClient.Do(req)
-		if err != nil {
-			return false
-		}
-		var data []byte
-		data, err = ioutil.ReadAll(resp.Body)
-		if err != nil {
-			return false
-		}
-		if bytes.Contains(data, []byte(remote)) {
+	for _, v := range p.stateCache.Items() {
+		if bytes.Contains(v.Object.([]byte), []byte(remote)) {
 			isPlaying = true
 			break
 		}
@@ -208,12 +256,6 @@ func (p *plexConnector) isPlaying(co *Object) bool {
 	return isPlaying
 }
 
-func (p *plexConnector) isPlayingAsync(co *Object, response chan bool) {
-	time.Sleep(time.Second) // FIXME random guess here
-	res := p.isPlaying(co)
-	response <- res
-}
-
 // adapted from: https://stackoverflow.com/a/28878037 (credit)
 func get(m interface{}, path ...interface{}) (interface{}, bool) {
 	for _, p := range path {

backend/cache/storage_memory.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 

backend/cache/storage_persistent.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.7
+// +build !plan9
 
 package cache
 
@@ -192,19 +192,46 @@ func (b *Persistent) GetDir(remote string) (*Directory, error) {
 
 // AddDir will update a CachedDirectory metadata and all its entries
 func (b *Persistent) AddDir(cachedDir *Directory) error {
+	return b.AddBatchDir([]*Directory{cachedDir})
+}
+
+// AddBatchDir will update a list of CachedDirectory metadata and all their entries
+func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error {
+	if len(cachedDirs) == 0 {
+		return nil
+	}
+
 	return b.db.Update(func(tx *bolt.Tx) error {
-		bucket := b.getBucket(cachedDir.abs(), true, tx)
+		var bucket *bolt.Bucket
+		if cachedDirs[0].Dir == "" {
+			bucket = tx.Bucket([]byte(RootBucket))
+		} else {
+			bucket = b.getBucket(cachedDirs[0].Dir, true, tx)
+		}
 		if bucket == nil {
-			return errors.Errorf("couldn't open bucket (%v)", cachedDir)
+			return errors.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir)
 		}
 
-		encoded, err := json.Marshal(cachedDir)
-		if err != nil {
-			return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
-		}
-		err = bucket.Put([]byte("."), encoded)
-		if err != nil {
-			return err
+		for _, cachedDir := range cachedDirs {
+			var b *bolt.Bucket
+			var err error
+			if cachedDir.Name == "" {
+				b = bucket
+			} else {
+				b, err = bucket.CreateBucketIfNotExists([]byte(cachedDir.Name))
+			}
+			if err != nil {
+				return err
+			}
+
+			encoded, err := json.Marshal(cachedDir)
+			if err != nil {
+				return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
+			}
+			err = b.Put([]byte("."), encoded)
+			if err != nil {
+				return err
+			}
 		}
 		return nil
 	})
@@ -400,6 +427,16 @@ func (b *Persistent) RemoveObject(fp string) error {
 	})
 }
 
+// ExpireObject will flush an Object and all its data if desired
+func (b *Persistent) ExpireObject(co *Object, withData bool) error {
+	co.CacheTs = time.Now().Add(co.CacheFs.fileAge * -1)
+	err := b.AddObject(co)
+	if withData {
+		_ = os.RemoveAll(path.Join(b.dataPath, co.abs()))
+	}
+	return err
+}
+
 // HasEntry confirms the existence of a single entry (dir or object)
 func (b *Persistent) HasEntry(remote string) bool {
 	dir, name := path.Split(remote)
@@ -1060,10 +1097,3 @@ func itob(v int64) []byte {
 func btoi(d []byte) int64 {
 	return int64(binary.BigEndian.Uint64(d))
 }
-
-// cloneBytes returns a copy of a given slice.
-func cloneBytes(v []byte) []byte {
-	var clone = make([]byte, len(v))
-	copy(clone, v)
-	return clone
-}

backend/crypt/cipher.go

@@ -781,7 +781,7 @@ func (c *cipher) newDecrypterSeek(open OpenRangeSeek, offset, limit int64) (fh *
 	}
 	fh.open = open // will be called by fh.RangeSeek
 	if doRangeSeek {
-		_, err = fh.RangeSeek(offset, 0, limit)
+		_, err = fh.RangeSeek(offset, io.SeekStart, limit)
 		if err != nil {
 			_ = fh.Close()
 			return nil, err
@@ -908,7 +908,7 @@ func (fh *decrypter) RangeSeek(offset int64, whence int, limit int64) (int64, er
 	if fh.open == nil {
 		return 0, fh.finish(errors.New("can't seek - not initialised with newDecrypterSeek"))
 	}
-	if whence != 0 {
+	if whence != io.SeekStart {
 		return 0, fh.finish(errors.New("can only seek from the start"))
 	}
 

backend/crypt/cipher_test.go

@@ -1016,7 +1016,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 			if offset+limit > len(plaintext) {
 				continue
 			}
-			_, err := fh.RangeSeek(int64(offset), 0, int64(limit))
+			_, err := fh.RangeSeek(int64(offset), io.SeekStart, int64(limit))
 			assert.NoError(t, err)
 
 			check(fh, offset, limit)
@@ -1083,7 +1083,7 @@ func TestNewDecrypterSeekLimit(t *testing.T) {
 		}
 		fh, err := c.DecryptDataSeek(testOpen, 0, -1)
 		assert.NoError(t, err)
-		gotOffset, err := fh.RangeSeek(test.offset, 0, test.limit)
+		gotOffset, err := fh.RangeSeek(test.offset, io.SeekStart, test.limit)
 		assert.NoError(t, err)
 		assert.Equal(t, gotOffset, test.offset)
 	}

backend/crypt/crypt.go

@@ -168,7 +168,6 @@ type Fs struct {
 	root     string
 	features *fs.Features // optional features
 	cipher   Cipher
-	mode     NameEncryptionMode
 }
 
 // Name of the remote (as passed into NewFs)
@@ -291,14 +290,48 @@ type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.O
 
 // put implements Put or PutStream
 func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	// Encrypt the data into wrappedIn
 	wrappedIn, err := f.cipher.EncryptData(in)
 	if err != nil {
 		return nil, err
 	}
+
+	// Find a hash the destination supports to compute a hash of
+	// the encrypted data
+	ht := f.Fs.Hashes().GetOne()
+	var hasher *hash.MultiHasher
+	if ht != hash.None {
+		hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+		if err != nil {
+			return nil, err
+		}
+		wrappedIn = io.TeeReader(wrappedIn, hasher)
+	}
+
+	// Transfer the data
 	o, err := put(wrappedIn, f.newObjectInfo(src), options...)
 	if err != nil {
 		return nil, err
 	}
+
+	// Check the hashes of the encrypted data if we were comparing them
+	if ht != hash.None && hasher != nil {
+		srcHash := hasher.Sums()[ht]
+		var dstHash string
+		dstHash, err = o.Hash(ht)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to read destination hash")
+		}
+		if srcHash != "" && dstHash != "" && srcHash != dstHash {
+			// remove object
+			err = o.Remove()
+			if err != nil {
+				fs.Errorf(o, "Failed to remove corrupted object: %v", err)
+			}
+			return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
+		}
+	}
+
 	return f.newObject(o), nil
 }
 
@@ -452,6 +485,15 @@ func (f *Fs) CleanUp() error {
 	return do()
 }
 
+// About gets quota information from the Fs
+func (f *Fs) About() (*fs.Usage, error) {
+	do := f.Fs.Features().About
+	if do == nil {
+		return nil, errors.New("About not supported")
+	}
+	return do()
+}
+
 // UnWrap returns the Fs that this Fs is wrapping
 func (f *Fs) UnWrap() fs.Fs {
 	return f.Fs
@@ -627,11 +669,11 @@ func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
 
 // Update in to the object with the modTime given of the given size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	wrappedIn, err := o.f.cipher.EncryptData(in)
-	if err != nil {
-		return err
+	update := func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+		return o.Object, o.Object.Update(in, src, options...)
 	}
-	return o.Object.Update(wrappedIn, o.f.newObjectInfo(src))
+	_, err := o.f.put(in, src, options, update)
+	return err
 }
 
 // newDir returns a dir with the Name decrypted
@@ -699,6 +741,7 @@ var (
 	_ fs.CleanUpper      = (*Fs)(nil)
 	_ fs.UnWrapper       = (*Fs)(nil)
 	_ fs.ListRer         = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.ObjectInfo      = (*ObjectInfo)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.ObjectUnWrapper = (*Object)(nil)

backend/crypt/crypt2_test.go

@@ -1,76 +0,0 @@
-// Test Crypt filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
-package crypt_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/backend/crypt"
-	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-func TestSetup2(t *testing.T) {
-	fstests.NilObject = fs.Object((*crypt.Object)(nil))
-	fstests.RemoteName = "TestCrypt2:"
-}
-
-// Generic tests for the Fs
-func TestInit2(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString2(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName2(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot2(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty2(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound2(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir2(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir2(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty2(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty2(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty2(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound2(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile12(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError2(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile22(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile12(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile22(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile22(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot2(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot2(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir2(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir2(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel22(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel22(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile12(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject2(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and22(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir2(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy2(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove2(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove2(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull2(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision2(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify2(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString2(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs2(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote2(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes2(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime2(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType2(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime2(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize2(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen2(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek2(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange2(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead2(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate2(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable2(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile2(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound2(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove2(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream2(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge2(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal2(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise2(t *testing.T)            { fstests.TestFinalise(t) }

backend/crypt/crypt3_test.go

@@ -1,76 +0,0 @@
-// Test Crypt filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
-package crypt_test
-
-import (
-	"testing"
-
-	"github.com/ncw/rclone/backend/crypt"
-	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fs"
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-func TestSetup3(t *testing.T) {
-	fstests.NilObject = fs.Object((*crypt.Object)(nil))
-	fstests.RemoteName = "TestCrypt3:"
-}
-
-// Generic tests for the Fs
-func TestInit3(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString3(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName3(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot3(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty3(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound3(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir3(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir3(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty3(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty3(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty3(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound3(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile13(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError3(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile23(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile13(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile23(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile23(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot3(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot3(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir3(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir3(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel23(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel23(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile13(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject3(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and23(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir3(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy3(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove3(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove3(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull3(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision3(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify3(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString3(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs3(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote3(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes3(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime3(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType3(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime3(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize3(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen3(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek3(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange3(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead3(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate3(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable3(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile3(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound3(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove3(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream3(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge3(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal3(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise3(t *testing.T)            { fstests.TestFinalise(t) }

backend/crypt/crypt_config_test.go

@@ -1,34 +0,0 @@
-package crypt_test
-
-import (
-	"os"
-	"path/filepath"
-
-	"github.com/ncw/rclone/fs/config/obscure"
-	"github.com/ncw/rclone/fstest/fstests"
-)
-
-// Create the TestCrypt: remote
-func init() {
-	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
-	name := "TestCrypt"
-	tempdir2 := filepath.Join(os.TempDir(), "rclone-crypt-test-off")
-	name2 := name + "2"
-	tempdir3 := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
-	name3 := name + "3"
-	fstests.ExtraConfig = []fstests.ExtraConfigItem{
-		{Name: name, Key: "type", Value: "crypt"},
-		{Name: name, Key: "remote", Value: tempdir},
-		{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
-		{Name: name, Key: "filename_encryption", Value: "standard"},
-		{Name: name2, Key: "type", Value: "crypt"},
-		{Name: name2, Key: "remote", Value: tempdir2},
-		{Name: name2, Key: "password", Value: obscure.MustObscure("potato2")},
-		{Name: name2, Key: "filename_encryption", Value: "off"},
-		{Name: name3, Key: "type", Value: "crypt"},
-		{Name: name3, Key: "remote", Value: tempdir3},
-		{Name: name3, Key: "password", Value: obscure.MustObscure("potato2")},
-		{Name: name3, Key: "filename_encryption", Value: "obfuscate"},
-	}
-	fstests.SkipBadWindowsCharacters[name3+":"] = true
-}

backend/crypt/crypt_test.go

@@ -1,76 +1,62 @@
 // Test Crypt filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package crypt_test
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/ncw/rclone/backend/crypt"
 	_ "github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*crypt.Object)(nil))
-	fstests.RemoteName = "TestCrypt:"
+// TestStandard runs integration tests against the remote
+func TestStandard(t *testing.T) {
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-standard")
+	name := "TestCrypt"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato")},
+			{Name: name, Key: "filename_encryption", Value: "standard"},
+		},
+	})
 }
 
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }
+// TestOff runs integration tests against the remote
+func TestOff(t *testing.T) {
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-off")
+	name := "TestCrypt2"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
+			{Name: name, Key: "filename_encryption", Value: "off"},
+		},
+	})
+}
+
+// TestObfuscate runs integration tests against the remote
+func TestObfuscate(t *testing.T) {
+	tempdir := filepath.Join(os.TempDir(), "rclone-crypt-test-obfuscate")
+	name := "TestCrypt3"
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: name + ":",
+		NilObject:  (*crypt.Object)(nil),
+		ExtraConfig: []fstests.ExtraConfigItem{
+			{Name: name, Key: "type", Value: "crypt"},
+			{Name: name, Key: "remote", Value: tempdir},
+			{Name: name, Key: "password", Value: obscure.MustObscure("potato2")},
+			{Name: name, Key: "filename_encryption", Value: "obfuscate"},
+		},
+		SkipBadWindowsCharacters: true,
+	})
+}

backend/drive/drive.go

@@ -34,7 +34,7 @@ import (
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
-	"google.golang.org/api/drive/v3"
+	drive "google.golang.org/api/drive/v3"
 	"google.golang.org/api/googleapi"
 )
 
@@ -54,15 +54,18 @@ const (
 // Globals
 var (
 	// Flags
-	driveAuthOwnerOnly  = flags.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user.")
-	driveUseTrash       = flags.BoolP("drive-use-trash", "", true, "Send files to the trash instead of deleting permanently.")
-	driveSkipGdocs      = flags.BoolP("drive-skip-gdocs", "", false, "Skip google documents in all listings.")
-	driveSharedWithMe   = flags.BoolP("drive-shared-with-me", "", false, "Only show files that are shared with me")
-	driveTrashedOnly    = flags.BoolP("drive-trashed-only", "", false, "Only show files that are in the trash")
-	driveExtensions     = flags.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
-	driveUseCreatedDate = flags.BoolP("drive-use-created-date", "", false, "Use created date instead of modified date.")
-	driveListChunk      = flags.Int64P("drive-list-chunk", "", 1000, "Size of listing chunk 100-1000. 0 to disable.")
-	driveImpersonate    = flags.StringP("drive-impersonate", "", "", "Impersonate this user when using a service account.")
+	driveAuthOwnerOnly       = flags.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user.")
+	driveUseTrash            = flags.BoolP("drive-use-trash", "", true, "Send files to the trash instead of deleting permanently.")
+	driveSkipGdocs           = flags.BoolP("drive-skip-gdocs", "", false, "Skip google documents in all listings.")
+	driveSharedWithMe        = flags.BoolP("drive-shared-with-me", "", false, "Only show files that are shared with me")
+	driveTrashedOnly         = flags.BoolP("drive-trashed-only", "", false, "Only show files that are in the trash")
+	driveExtensions          = flags.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
+	driveUseCreatedDate      = flags.BoolP("drive-use-created-date", "", false, "Use created date instead of modified date.")
+	driveListChunk           = flags.Int64P("drive-list-chunk", "", 1000, "Size of listing chunk 100-1000. 0 to disable.")
+	driveImpersonate         = flags.StringP("drive-impersonate", "", "", "Impersonate this user when using a service account.")
+	driveAlternateExport     = flags.BoolP("drive-alternate-export", "", false, "Use alternate export URLs for google documents export.")
+	driveAcknowledgeAbuse    = flags.BoolP("drive-acknowledge-abuse", "", false, "Set to allow files which return cannotDownloadAbusiveFile to be downloaded.")
+	driveKeepRevisionForever = flags.BoolP("drive-keep-revision-forever", "", false, "Keep new head revision forever.")
 	// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	chunkSize         = fs.SizeSuffix(8 * 1024 * 1024)
@@ -292,8 +295,7 @@ func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bo
 		searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
 		// Convert ／ to / for search
 		searchTitle = strings.Replace(searchTitle, "／", "/", -1)
-		// use contains to work around #1675
-		query = append(query, fmt.Sprintf("name contains '%s'", searchTitle))
+		query = append(query, fmt.Sprintf("name='%s'", searchTitle))
 	}
 	if directoriesOnly {
 		query = append(query, fmt.Sprintf("mimeType='%s'", driveFolderType))
@@ -341,7 +343,8 @@ OUTER:
 		for _, item := range files.Files {
 			// Convert / to ／ for listing purposes
 			item.Name = strings.Replace(item.Name, "/", "／", -1)
-			// skip items introduced by workaround (#1675)
+			// Check the case of items is correct since
+			// the `=` operator is case insensitive.
 			if title != "" && title != item.Name {
 				continue
 			}
@@ -399,7 +402,7 @@ func configTeamDrive(name string) error {
 	} else {
 		fmt.Printf("Change current team drive ID %q?\n", teamDrive)
 	}
-	if !config.Confirm() {
+	if !config.ConfirmWithDefault(false) {
 		return nil
 	}
 	client, err := createOAuthClient(name)
@@ -446,12 +449,8 @@ func newPacer() *pacer.Pacer {
 	return pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer)
 }
 
-func getServiceAccountClient(keyJsonfilePath string) (*http.Client, error) {
-	data, err := ioutil.ReadFile(os.ExpandEnv(keyJsonfilePath))
-	if err != nil {
-		return nil, errors.Wrap(err, "error opening credentials file")
-	}
-	conf, err := google.JWTConfigFromJSON(data, driveConfig.Scopes...)
+func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
+	conf, err := google.JWTConfigFromJSON(credentialsData, driveConfig.Scopes...)
 	if err != nil {
 		return nil, errors.Wrap(err, "error processing credentials")
 	}
@@ -466,9 +465,18 @@ func createOAuthClient(name string) (*http.Client, error) {
 	var oAuthClient *http.Client
 	var err error
 
+	// try loading service account credentials from env variable, then from a file
+	serviceAccountCreds := []byte(config.FileGet(name, "service_account_credentials"))
 	serviceAccountPath := config.FileGet(name, "service_account_file")
-	if serviceAccountPath != "" {
-		oAuthClient, err = getServiceAccountClient(serviceAccountPath)
+	if len(serviceAccountCreds) == 0 && serviceAccountPath != "" {
+		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(serviceAccountPath))
+		if err != nil {
+			return nil, errors.Wrap(err, "error opening service account credentials file")
+		}
+		serviceAccountCreds = loadedCreds
+	}
+	if len(serviceAccountCreds) > 0 {
+		oAuthClient, err = getServiceAccountClient(serviceAccountCreds)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create oauth client from service account")
 		}
@@ -551,24 +559,28 @@ func NewFs(name, path string) (fs.Fs, error) {
 	if err != nil {
 		// Assume it is a file
 		newRoot, remote := dircache.SplitPath(root)
-		newF := *f
-		newF.dirCache = dircache.New(newRoot, f.rootFolderID, &newF)
-		newF.root = newRoot
+		tempF := *f
+		tempF.dirCache = dircache.New(newRoot, f.rootFolderID, &tempF)
+		tempF.root = newRoot
 		// Make new Fs which is the parent
-		err = newF.dirCache.FindRoot(false)
+		err = tempF.dirCache.FindRoot(false)
 		if err != nil {
 			// No root so return old f
 			return f, nil
 		}
-		entries, err := newF.List("")
+		entries, err := tempF.List("")
 		if err != nil {
 			// unable to list folder so return old f
 			return f, nil
 		}
 		for _, e := range entries {
 			if _, isObject := e.(fs.Object); isObject && e.Remote() == remote {
-				// return an error with an fs which points to the parent
-				return &newF, fs.ErrorIsFile
+				// XXX: update the old f here instead of returning tempF, since
+				// `features` were already filled with functions having *f as a receiver.
+				// See https://github.com/ncw/rclone/issues/2182
+				f.dirCache = tempF.dirCache
+				f.root = tempF.root
+				return f, fs.ErrorIsFile
 			}
 		}
 		// File doesn't exist so return old f
@@ -748,6 +760,18 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			}
 			obj := o.(*Object)
 			obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
+			if *driveAlternateExport {
+				switch item.MimeType {
+				case "application/vnd.google-apps.drawing":
+					obj.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", item.Id, extension)
+				case "application/vnd.google-apps.document":
+					obj.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", item.Id, extension)
+				case "application/vnd.google-apps.spreadsheet":
+					obj.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", item.Id, extension)
+				case "application/vnd.google-apps.presentation":
+					obj.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", item.Id, extension)
+				}
+			}
 			obj.isDocument = true
 			obj.mimeType = exportMimeType
 			obj.bytes = -1
@@ -834,7 +858,7 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt
 		// Make the API request to upload metadata and file data.
 		// Don't retry, return a retry error instead
 		err = f.pacer.CallNoRetry(func() (bool, error) {
-			info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do()
+			info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -881,9 +905,9 @@ func (f *Fs) MergeDirs(dirs []fs.Directory) error {
 			}
 		}
 		// rmdir (into trash) the now empty source directory
+		fs.Infof(srcDir, "removing empty directory")
 		err = f.rmdir(srcDir.ID(), true)
 		if err != nil {
-			fs.Infof(srcDir, "removing empty directory")
 			return errors.Wrapf(err, "MergDirs move failed to rmdir %q", srcDir)
 		}
 	}
@@ -991,7 +1015,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 
 	var info *drive.File
 	err = o.fs.pacer.Call(func() (bool, error) {
-		info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do()
+		info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -1046,6 +1070,34 @@ func (f *Fs) CleanUp() error {
 	return nil
 }
 
+// About gets quota information
+func (f *Fs) About() (*fs.Usage, error) {
+	if f.isTeamDrive {
+		// Teamdrives don't appear to have a usage API so just return empty
+		return &fs.Usage{}, nil
+	}
+	var about *drive.About
+	var err error
+	err = f.pacer.Call(func() (bool, error) {
+		about, err = f.svc.About.Get().Fields("storageQuota").Do()
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get Drive storageQuota")
+	}
+	q := about.StorageQuota
+	usage := &fs.Usage{
+		Used:    fs.NewUsageValue(q.UsageInDrive),           // bytes in use
+		Trashed: fs.NewUsageValue(q.UsageInDriveTrash),      // bytes in trash
+		Other:   fs.NewUsageValue(q.Usage - q.UsageInDrive), // other usage eg gmail in drive
+	}
+	if q.Limit > 0 {
+		usage.Total = fs.NewUsageValue(q.Limit)          // quota of bytes that can be used
+		usage.Free = fs.NewUsageValue(q.Limit - q.Usage) // bytes which can be uploaded before reaching the quota
+	}
+	return usage, nil
+}
+
 // Move src to this remote using server side move operations.
 //
 // This is stored with the remote path given
@@ -1091,6 +1143,41 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	return dstObj, nil
 }
 
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func (f *Fs) PublicLink(remote string) (link string, err error) {
+	id, err := f.dirCache.FindDir(remote, false)
+	if err == nil {
+		fs.Debugf(f, "attempting to share directory '%s'", remote)
+	} else {
+		fs.Debugf(f, "attempting to share single file '%s'", remote)
+		o := &Object{
+			fs:     f,
+			remote: remote,
+		}
+		if err = o.readMetaData(); err != nil {
+			return
+		}
+		id = o.id
+	}
+
+	permission := &drive.Permission{
+		AllowFileDiscovery: false,
+		Role:               "reader",
+		Type:               "anyone",
+	}
+
+	err = f.pacer.Call(func() (bool, error) {
+		// TODO: On TeamDrives this might fail if lacking permissions to change ACLs.
+		// Need to either check `canShare` attribute on the object or see if a sufficient permission is already present.
+		_, err = f.svc.Permissions.Create(id, permission).Fields(googleapi.Field("id")).SupportsTeamDrives(f.isTeamDrive).Do()
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("https://drive.google.com/open?id=%s", id), nil
+}
+
 // DirMove moves src, srcRemote to this remote at dstRemote
 // using server side move operations.
 //
@@ -1156,10 +1243,16 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	}
 
 	// Find ID of src parent
-	_, srcDirectoryID, err := srcFs.dirCache.FindPath(srcRemote, false)
+	var srcDirectoryID string
+	if srcRemote == "" {
+		srcDirectoryID, err = srcFs.dirCache.RootParentID()
+	} else {
+		_, srcDirectoryID, err = srcFs.dirCache.FindPath(srcRemote, false)
+	}
 	if err != nil {
 		return err
 	}
+
 	// Find ID of src
 	srcID, err := srcFs.dirCache.FindDir(srcRemote, false)
 	if err != nil {
@@ -1224,9 +1317,14 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
 		err = f.pacer.Call(func() (bool, error) {
 			changesCall := f.svc.Changes.List(pageToken).Fields("nextPageToken,newStartPageToken,changes(fileId,file(name,parents,mimeType))")
 			if *driveListChunk > 0 {
-				changesCall = changesCall.PageSize(*driveListChunk)
+				changesCall.PageSize(*driveListChunk)
 			}
-			changeList, err = changesCall.SupportsTeamDrives(f.isTeamDrive).Do()
+			if f.isTeamDrive {
+				changesCall.TeamDriveId(f.teamDriveID)
+				changesCall.SupportsTeamDrives(true)
+				changesCall.IncludeTeamDriveItems(true)
+			}
+			changeList, err = changesCall.Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -1249,7 +1347,12 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
 				continue
 			}
 
-			if change.File != nil && change.File.MimeType != driveFolderType {
+			if change.File != nil {
+				changeType := fs.EntryDirectory
+				if change.File.MimeType != driveFolderType {
+					changeType = fs.EntryObject
+				}
+
 				// translate the parent dir of this object
 				if len(change.File.Parents) > 0 {
 					if path, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
@@ -1260,10 +1363,10 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
 							path = change.File.Name
 						}
 						// this will now clear the actual file too
-						pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject})
+						pathsToClear = append(pathsToClear, entryType{path: path, entryType: changeType})
 					}
 				} else { // a true root object that is changed
-					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: fs.EntryObject})
+					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
 				}
 			}
 		}
@@ -1448,6 +1551,12 @@ func (o *Object) httpResponse(method string, options []fs.OpenOption) (req *http
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
 	err = o.fs.pacer.Call(func() (bool, error) {
 		res, err = o.fs.client.Do(req)
+		if err == nil {
+			err = googleapi.CheckResponse(res)
+			if err != nil {
+				_ = res.Body.Close() // ignore error
+			}
+		}
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -1491,16 +1600,39 @@ func (file *openFile) Close() (err error) {
 // Check it satisfies the interfaces
 var _ io.ReadCloser = &openFile{}
 
+// Checks to see if err is a googleapi.Error with of type what
+func isGoogleError(err error, what string) bool {
+	if gerr, ok := err.(*googleapi.Error); ok {
+		for _, error := range gerr.Errors {
+			if error.Reason == what {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
-	req, res, err := o.httpResponse("GET", options)
+	_, res, err := o.httpResponse("GET", options)
 	if err != nil {
-		return nil, err
-	}
-	_, isRanging := req.Header["Range"]
-	if !(res.StatusCode == http.StatusOK || (isRanging && res.StatusCode == http.StatusPartialContent)) {
-		_ = res.Body.Close() // ignore error
-		return nil, errors.Errorf("bad response: %d: %s", res.StatusCode, res.Status)
+		if isGoogleError(err, "cannotDownloadAbusiveFile") {
+			if *driveAcknowledgeAbuse {
+				// Retry acknowledging abuse
+				if strings.ContainsRune(o.url, '?') {
+					o.url += "&"
+				} else {
+					o.url += "?"
+				}
+				o.url += "acknowledgeAbuse=true"
+				_, res, err = o.httpResponse("GET", options)
+			} else {
+				err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file")
+			}
+		}
+		if err != nil {
+			return nil, errors.Wrap(err, "open file failed")
+		}
 	}
 	// If it is a document, update the size with what we are
 	// reading as it can change from the HEAD in the listing to
@@ -1534,7 +1666,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if size == 0 || size < int64(driveUploadCutoff) {
 		// Don't retry, return a retry error instead
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).Do()
+			info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -1581,6 +1713,11 @@ func (o *Object) MimeType() string {
 	return o.mimeType
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1593,7 +1730,10 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.PutUncheckeder  = (*Fs)(nil)
+	_ fs.PublicLinker    = (*Fs)(nil)
 	_ fs.MergeDirser     = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
-	_ fs.MimeTyper       = &Object{}
+	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/drive/drive_test.go

@@ -1,75 +1,17 @@
 // Test Drive filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package drive_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/drive"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*drive.Object)(nil))
-	fstests.RemoteName = "TestDrive:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestDrive:",
+		NilObject:  (*drive.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/drive/upload.go

@@ -30,9 +30,6 @@ import (
 const (
 	// statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete.
 	statusResumeIncomplete = 308
-
-	// Number of times to try each chunk
-	maxTries = 10
 )
 
 // resumableUpload is used by the generated APIs to provide resumable uploads.
@@ -61,6 +58,9 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string,
 	if f.isTeamDrive {
 		params.Set("supportsTeamDrives", "true")
 	}
+	if *driveKeepRevisionForever {
+		params.Set("keepRevisionForever", "true")
+	}
 	urls := "https://www.googleapis.com/upload/drive/v3/files"
 	method := "POST"
 	if fileID != "" {
@@ -159,7 +159,7 @@ func (rx *resumableUpload) transferStatus() (start int64, err error) {
 
 // Transfer a chunk - caller must call googleapi.CloseBody(res) if err == nil || res != nil
 func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunkSize int64) (int, error) {
-	_, _ = chunk.Seek(0, 0)
+	_, _ = chunk.Seek(0, io.SeekStart)
 	req := rx.makeRequest(start, chunk, chunkSize)
 	res, err := rx.f.client.Do(req)
 	if err != nil {
@@ -192,7 +192,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 }
 
 // Upload uploads the chunks from the input
-// It retries each chunk maxTries times (with a pause of uploadPause between attempts).
+// It retries each chunk using the pacer and --low-level-retries
 func (rx *resumableUpload) Upload() (*drive.File, error) {
 	start := int64(0)
 	var StatusCode int

backend/dropbox/dropbox.go

@@ -1,7 +1,4 @@
 // Package dropbox provides an interface to Dropbox object storage
-
-// +build go1.7
-
 package dropbox
 
 // FIXME dropbox for business would be quite easy to add
@@ -25,7 +22,6 @@ of path_display and all will be well.
 */
 
 import (
-	"crypto/md5"
 	"fmt"
 	"io"
 	"log"
@@ -35,7 +31,10 @@ import (
 	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/common"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/sharing"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/flags"
@@ -126,13 +125,16 @@ func init() {
 
 // Fs represents a remote dropbox server
 type Fs struct {
-	name           string       // name of this remote
-	root           string       // the path we are working on
-	features       *fs.Features // optional features
-	srv            files.Client // the connection to the dropbox server
-	slashRoot      string       // root with "/" prefix, lowercase
-	slashRootSlash string       // root with "/" prefix and postfix, lowercase
-	pacer          *pacer.Pacer // To pace the API calls
+	name           string         // name of this remote
+	root           string         // the path we are working on
+	features       *fs.Features   // optional features
+	srv            files.Client   // the connection to the dropbox server
+	sharing        sharing.Client // as above, but for generating sharing links
+	users          users.Client   // as above, but for accessing user information
+	slashRoot      string         // root with "/" prefix, lowercase
+	slashRootSlash string         // root with "/" prefix and postfix, lowercase
+	pacer          *pacer.Pacer   // To pace the API calls
+	ns             string         // The namespace we are using or "" for none
 }
 
 // Object describes a dropbox object
@@ -202,20 +204,21 @@ func NewFs(name, root string) (fs.Fs, error) {
 
 	oAuthClient, _, err := oauthutil.NewClient(name, dropboxConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure dropbox: %v", err)
+		return nil, errors.Wrap(err, "failed to configure dropbox")
 	}
 
-	config := dropbox.Config{
-		LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
-		Client:   oAuthClient,    // maybe???
-	}
-	srv := files.New(config)
-
 	f := &Fs{
 		name:  name,
-		srv:   srv,
 		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
+	config := dropbox.Config{
+		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
+		Client:          oAuthClient,    // maybe???
+		HeaderGenerator: f.headerGenerator,
+	}
+	f.srv = files.New(config)
+	f.sharing = sharing.New(config)
+	f.users = users.New(config)
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
 		ReadMimeType:            true,
@@ -223,6 +226,27 @@ func NewFs(name, root string) (fs.Fs, error) {
 	}).Fill(f)
 	f.setRoot(root)
 
+	// If root starts with / then use the actual root
+	if strings.HasPrefix(root, "/") {
+		var acc *users.FullAccount
+		err = f.pacer.Call(func() (bool, error) {
+			acc, err = f.users.GetCurrentAccount()
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "get current account failed")
+		}
+		switch x := acc.RootInfo.(type) {
+		case *common.TeamRootInfo:
+			f.ns = x.RootNamespaceId
+		case *common.UserRootInfo:
+			f.ns = x.RootNamespaceId
+		default:
+			return nil, errors.Errorf("unknown RootInfo type %v %T", acc.RootInfo, acc.RootInfo)
+		}
+		fs.Debugf(f, "Using root namespace %q", f.ns)
+	}
+
 	// See if the root is actually an object
 	_, err = f.getFileMetadata(f.slashRoot)
 	if err == nil {
@@ -237,14 +261,22 @@ func NewFs(name, root string) (fs.Fs, error) {
 	return f, nil
 }
 
+// headerGenerator for dropbox sdk
+func (f *Fs) headerGenerator(hostType string, style string, namespace string, route string) map[string]string {
+	if f.ns == "" {
+		return map[string]string{}
+	}
+	return map[string]string{
+		"Dropbox-API-Path-Root": `{".tag": "namespace_id", "namespace_id": "` + f.ns + `"}`,
+	}
+}
+
 // Sets root in f
 func (f *Fs) setRoot(root string) {
 	f.root = strings.Trim(root, "/")
-	lowerCaseRoot := strings.ToLower(f.root)
-
-	f.slashRoot = "/" + lowerCaseRoot
+	f.slashRoot = "/" + f.root
 	f.slashRootSlash = f.slashRoot
-	if lowerCaseRoot != "" {
+	if f.root != "" {
 		f.slashRootSlash += "/"
 	}
 }
@@ -417,21 +449,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-// A read closer which doesn't close the input
-type readCloser struct {
-	in io.Reader
-}
-
-// Read bytes from the object - see io.Reader
-func (rc *readCloser) Read(p []byte) (n int, err error) {
-	return rc.in.Read(p)
-}
-
-// Dummy close function
-func (rc *readCloser) Close() error {
-	return nil
-}
-
 // Put the object
 //
 // Copy the reader in to the new object which is returned
@@ -640,6 +657,52 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
 	return dstObj, nil
 }
 
+// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
+func (f *Fs) PublicLink(remote string) (link string, err error) {
+	absPath := "/" + path.Join(f.Root(), remote)
+	fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
+	createArg := sharing.CreateSharedLinkWithSettingsArg{
+		Path: absPath,
+	}
+	var linkRes sharing.IsSharedLinkMetadata
+	err = f.pacer.Call(func() (bool, error) {
+		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
+		return shouldRetry(err)
+	})
+
+	if err != nil && strings.Contains(err.Error(), sharing.CreateSharedLinkWithSettingsErrorSharedLinkAlreadyExists) {
+		fs.Debugf(absPath, "has a public link already, attempting to retrieve it")
+		listArg := sharing.ListSharedLinksArg{
+			Path:       absPath,
+			DirectOnly: true,
+		}
+		var listRes *sharing.ListSharedLinksResult
+		err = f.pacer.Call(func() (bool, error) {
+			listRes, err = f.sharing.ListSharedLinks(&listArg)
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return
+		}
+		if len(listRes.Links) == 0 {
+			err = errors.New("Dropbox says the sharing link already exists, but list came back empty")
+			return
+		}
+		linkRes = listRes.Links[0]
+	}
+	if err == nil {
+		switch res := linkRes.(type) {
+		case *sharing.FileLinkMetadata:
+			link = res.Url
+		case *sharing.FolderLinkMetadata:
+			link = res.Url
+		default:
+			err = fmt.Errorf("Don't know how to extract link, response has unknown format: %T", res)
+		}
+	}
+	return
+}
+
 // DirMove moves src, srcRemote to this remote at dstRemote
 // using server side move operations.
 //
@@ -683,6 +746,33 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
 	return nil
 }
 
+// About gets quota information
+func (f *Fs) About() (usage *fs.Usage, err error) {
+	var q *users.SpaceUsage
+	err = f.pacer.Call(func() (bool, error) {
+		q, err = f.users.GetSpaceUsage()
+		return shouldRetry(err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "about failed")
+	}
+	var total uint64
+	if q.Allocation != nil {
+		if q.Allocation.Individual != nil {
+			total += q.Allocation.Individual.Allocated
+		}
+		if q.Allocation.Team != nil {
+			total += q.Allocation.Team.Allocated
+		}
+	}
+	usage = &fs.Usage{
+		Total: fs.NewUsageValue(int64(total)),          // quota of bytes that can be used
+		Used:  fs.NewUsageValue(int64(q.Used)),         // bytes in use
+		Free:  fs.NewUsageValue(int64(total - q.Used)), // bytes which can be uploaded before reaching the quota
+	}
+	return usage, nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.Dropbox)
@@ -758,20 +848,6 @@ func (o *Object) remotePath() string {
 	return o.fs.slashRootSlash + o.remote
 }
 
-// Returns the key for the metadata database for a given path
-func metadataKey(path string) string {
-	// NB File system is case insensitive
-	path = strings.ToLower(path)
-	hash := md5.New()
-	_, _ = hash.Write([]byte(path))
-	return fmt.Sprintf("%x", hash.Sum(nil))
-}
-
-// Returns the key for the metadata database
-func (o *Object) metadataKey() string {
-	return metadataKey(o.remotePath())
-}
-
 // readMetaData gets the info if it hasn't already been fetched
 func (o *Object) readMetaData() (err error) {
 	if !o.modTime.IsZero() {
@@ -801,7 +877,7 @@ func (o *Object) SetModTime(modTime time.Time) error {
 	// Dropbox doesn't have a way of doing this so returning this
 	// error will cause the file to be deleted first then
 	// re-uploaded to set the time.
-	return fs.ErrorCantSetModTime
+	return fs.ErrorCantSetModTimeWithoutDelete
 }
 
 // Storable returns whether this object is storable
@@ -859,7 +935,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 	chunk := readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
 	err = o.fs.pacer.Call(func() (bool, error) {
 		// seek to the start in case this is a retry
-		if _, err = chunk.Seek(0, 0); err != nil {
+		if _, err = chunk.Seek(0, io.SeekStart); err != nil {
 			return false, nil
 		}
 		res, err = o.fs.srv.UploadSessionStart(&files.UploadSessionStartArg{}, chunk)
@@ -895,7 +971,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 		chunk = readers.NewRepeatableLimitReaderBuffer(in, buf, chunkSize)
 		err = o.fs.pacer.Call(func() (bool, error) {
 			// seek to the start in case this is a retry
-			if _, err = chunk.Seek(0, 0); err != nil {
+			if _, err = chunk.Seek(0, io.SeekStart); err != nil {
 				return false, nil
 			}
 			err = o.fs.srv.UploadSessionAppendV2(&appendArg, chunk)
@@ -918,7 +994,7 @@ func (o *Object) uploadChunked(in0 io.Reader, commitInfo *files.CommitInfo, size
 	chunk = readers.NewRepeatableReaderBuffer(in, buf)
 	err = o.fs.pacer.Call(func() (bool, error) {
 		// seek to the start in case this is a retry
-		if _, err = chunk.Seek(0, 0); err != nil {
+		if _, err = chunk.Seek(0, io.SeekStart); err != nil {
 			return false, nil
 		}
 		entry, err = o.fs.srv.UploadSessionFinish(args, chunk)
@@ -975,11 +1051,13 @@ func (o *Object) Remove() (err error) {
 
 // Check the interfaces are satisfied
 var (
-	_ fs.Fs          = (*Fs)(nil)
-	_ fs.Copier      = (*Fs)(nil)
-	_ fs.Purger      = (*Fs)(nil)
-	_ fs.PutStreamer = (*Fs)(nil)
-	_ fs.Mover       = (*Fs)(nil)
-	_ fs.DirMover    = (*Fs)(nil)
-	_ fs.Object      = (*Object)(nil)
+	_ fs.Fs           = (*Fs)(nil)
+	_ fs.Copier       = (*Fs)(nil)
+	_ fs.Purger       = (*Fs)(nil)
+	_ fs.PutStreamer  = (*Fs)(nil)
+	_ fs.Mover        = (*Fs)(nil)
+	_ fs.PublicLinker = (*Fs)(nil)
+	_ fs.DirMover     = (*Fs)(nil)
+	_ fs.Abouter      = (*Fs)(nil)
+	_ fs.Object       = (*Object)(nil)
 )

backend/dropbox/dropbox_test.go

@@ -1,78 +1,17 @@
 // Test Dropbox filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
-
-// +build go1.7
-
 package dropbox_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/dropbox"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*dropbox.Object)(nil))
-	fstests.RemoteName = "TestDropbox:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestDropbox:",
+		NilObject:  (*dropbox.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/dropbox/dropbox_unsupported.go

@@ -1,6 +0,0 @@
-// Build for unsupported platforms to stop go complaining about "no
-// buildable Go source files "
-
-// +build !go1.7
-
-package dropbox

backend/ftp/ftp.go

@@ -247,7 +247,7 @@ func translateErrorFile(err error) error {
 	switch errX := err.(type) {
 	case *textproto.Error:
 		switch errX.Code {
-		case ftp.StatusFileUnavailable:
+		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorObjectNotFound
 		}
 	}
@@ -259,16 +259,15 @@ func translateErrorDir(err error) error {
 	switch errX := err.(type) {
 	case *textproto.Error:
 		switch errX.Code {
-		case ftp.StatusFileUnavailable:
+		case ftp.StatusFileUnavailable, ftp.StatusFileActionIgnored:
 			err = fs.ErrorDirNotFound
 		}
 	}
 	return err
 }
 
-// NewObject finds the Object at remote.  If it can't be found
-// it returns the error fs.ErrorObjectNotFound.
-func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
+// findItem finds a directory entry for the name in its parent directory
+func (f *Fs) findItem(remote string) (entry *ftp.Entry, err error) {
 	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
 	fullPath := path.Join(f.root, remote)
 	dir := path.Dir(fullPath)
@@ -276,32 +275,58 @@ func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
 
 	c, err := f.getFtpConnection()
 	if err != nil {
-		return nil, errors.Wrap(err, "NewObject")
+		return nil, errors.Wrap(err, "findItem")
 	}
 	files, err := c.List(dir)
 	f.putFtpConnection(&c, err)
 	if err != nil {
 		return nil, translateErrorFile(err)
 	}
-	for i, file := range files {
-		if file.Type != ftp.EntryTypeFolder && file.Name == base {
-			o := &Object{
-				fs:     f,
-				remote: remote,
-			}
-			info := &FileInfo{
-				Name:    remote,
-				Size:    files[i].Size,
-				ModTime: files[i].Time,
-			}
-			o.info = info
-
-			return o, nil
+	for _, file := range files {
+		if file.Name == base {
+			return file, nil
 		}
 	}
+	return nil, nil
+}
+
+// NewObject finds the Object at remote.  If it can't be found
+// it returns the error fs.ErrorObjectNotFound.
+func (f *Fs) NewObject(remote string) (o fs.Object, err error) {
+	// defer fs.Trace(remote, "")("o=%v, err=%v", &o, &err)
+	entry, err := f.findItem(remote)
+	if err != nil {
+		return nil, err
+	}
+	if entry != nil && entry.Type != ftp.EntryTypeFolder {
+		o := &Object{
+			fs:     f,
+			remote: remote,
+		}
+		info := &FileInfo{
+			Name:    remote,
+			Size:    entry.Size,
+			ModTime: entry.Time,
+		}
+		o.info = info
+
+		return o, nil
+	}
 	return nil, fs.ErrorObjectNotFound
 }
 
+// dirExists checks the directory pointed to by remote exists or not
+func (f *Fs) dirExists(remote string) (exists bool, err error) {
+	entry, err := f.findItem(remote)
+	if err != nil {
+		return false, errors.Wrap(err, "dirExists")
+	}
+	if entry != nil && entry.Type == ftp.EntryTypeFolder {
+		return true, nil
+	}
+	return false, nil
+}
+
 // List the objects and directories in dir into entries.  The
 // entries can be returned in any order but should be for a
 // complete directory.
@@ -322,6 +347,18 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	if err != nil {
 		return nil, translateErrorDir(err)
 	}
+	// Annoyingly FTP returns success for a directory which
+	// doesn't exist, so check it really doesn't exist if no
+	// entries found.
+	if len(files) == 0 {
+		exists, err := f.dirExists(dir)
+		if err != nil {
+			return nil, errors.Wrap(err, "list")
+		}
+		if !exists {
+			return nil, fs.ErrorDirNotFound
+		}
+	}
 	for i := range files {
 		object := files[i]
 		newremote := path.Join(dir, object.Name)
@@ -438,6 +475,15 @@ func (f *Fs) mkdir(abspath string) error {
 	}
 	err = c.MakeDir(abspath)
 	f.putFtpConnection(&c, err)
+	switch errX := err.(type) {
+	case *textproto.Error:
+		switch errX.Code {
+		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
+			err = nil
+		case 521: // dir already exists: error number according to RFC 959: issue #2363
+			err = nil
+		}
+	}
 	return err
 }
 

backend/ftp/ftp_test.go

@@ -1,75 +1,17 @@
 // Test FTP filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package ftp_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/ftp"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*ftp.Object)(nil))
-	fstests.RemoteName = "TestFTP:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestFTP:",
+		NilObject:  (*ftp.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/googlecloudstorage/googlecloudstorage.go

@@ -31,10 +31,12 @@ import (
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/flags"
 	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/oauthutil"
+	"github.com/ncw/rclone/lib/pacer"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -49,6 +51,7 @@ const (
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
 	metaMtime                   = "mtime" // key to store mtime under in metadata
 	listChunks                  = 1000    // chunk size to read directory listings
+	minSleep                    = 10 * time.Millisecond
 )
 
 var (
@@ -219,6 +222,7 @@ type Fs struct {
 	bucketACL     string           // used when creating new buckets
 	location      string           // location of new buckets
 	storageClass  string           // storage class of new buckets
+	pacer         *pacer.Pacer     // To pace the API calls
 }
 
 // Object describes a storage object
@@ -262,6 +266,30 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
+// shouldRetry determines whehter a given err rates being retried
+func shouldRetry(err error) (again bool, errOut error) {
+	again = false
+	if err != nil {
+		if fserrors.ShouldRetry(err) {
+			again = true
+		} else {
+			switch gerr := err.(type) {
+			case *googleapi.Error:
+				if gerr.Code >= 500 && gerr.Code < 600 {
+					// All 5xx errors should be retried
+					again = true
+				} else if len(gerr.Errors) > 0 {
+					reason := gerr.Errors[0].Reason
+					if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
+						again = true
+					}
+				}
+			}
+		}
+	}
+	return again, err
+}
+
 // Pattern to match a storage path
 var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
 
@@ -277,12 +305,8 @@ func parsePath(path string) (bucket, directory string, err error) {
 	return
 }
 
-func getServiceAccountClient(keyJsonfilePath string) (*http.Client, error) {
-	data, err := ioutil.ReadFile(os.ExpandEnv(keyJsonfilePath))
-	if err != nil {
-		return nil, errors.Wrap(err, "error opening credentials file")
-	}
-	conf, err := google.JWTConfigFromJSON(data, storageConfig.Scopes...)
+func getServiceAccountClient(credentialsData []byte) (*http.Client, error) {
+	conf, err := google.JWTConfigFromJSON(credentialsData, storageConfig.Scopes...)
 	if err != nil {
 		return nil, errors.Wrap(err, "error processing credentials")
 	}
@@ -295,16 +319,25 @@ func NewFs(name, root string) (fs.Fs, error) {
 	var oAuthClient *http.Client
 	var err error
 
+	// try loading service account credentials from env variable, then from a file
+	serviceAccountCreds := []byte(config.FileGet(name, "service_account_credentials"))
 	serviceAccountPath := config.FileGet(name, "service_account_file")
-	if serviceAccountPath != "" {
-		oAuthClient, err = getServiceAccountClient(serviceAccountPath)
+	if len(serviceAccountCreds) == 0 && serviceAccountPath != "" {
+		loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(serviceAccountPath))
 		if err != nil {
-			log.Fatalf("Failed configuring Google Cloud Storage Service Account: %v", err)
+			return nil, errors.Wrap(err, "error opening service account credentials file")
+		}
+		serviceAccountCreds = loadedCreds
+	}
+	if len(serviceAccountCreds) > 0 {
+		oAuthClient, err = getServiceAccountClient(serviceAccountCreds)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")
 		}
 	} else {
 		oAuthClient, _, err = oauthutil.NewClient(name, storageConfig)
 		if err != nil {
-			log.Fatalf("Failed to configure Google Cloud Storage: %v", err)
+			return nil, errors.Wrap(err, "failed to configure Google Cloud Storage")
 		}
 	}
 
@@ -322,6 +355,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 		bucketACL:     config.FileGet(name, "bucket_acl"),
 		location:      config.FileGet(name, "location"),
 		storageClass:  config.FileGet(name, "storage_class"),
+		pacer:         pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -351,7 +385,10 @@ func NewFs(name, root string) (fs.Fs, error) {
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the object exists
-		_, err = f.svc.Objects.Get(bucket, directory).Do()
+		err = f.pacer.Call(func() (bool, error) {
+			_, err = f.svc.Objects.Get(bucket, directory).Do()
+			return shouldRetry(err)
+		})
 		if err == nil {
 			f.root = path.Dir(directory)
 			if f.root == "." {
@@ -399,7 +436,7 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
 // dir is the starting directory, "" for root
 //
 // Set recurse to read sub directories
-func (f *Fs) list(dir string, recurse bool, fn listFn) error {
+func (f *Fs) list(dir string, recurse bool, fn listFn) (err error) {
 	root := f.root
 	rootLength := len(root)
 	if dir != "" {
@@ -410,7 +447,11 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 		list = list.Delimiter("/")
 	}
 	for {
-		objects, err := list.Do()
+		var objects *storage.Objects
+		err = f.pacer.Call(func() (bool, error) {
+			objects, err = list.Do()
+			return shouldRetry(err)
+		})
 		if err != nil {
 			if gErr, ok := err.(*googleapi.Error); ok {
 				if gErr.Code == http.StatusNotFound {
@@ -437,6 +478,17 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 				continue
 			}
 			remote := object.Name[rootLength:]
+			// is this a directory marker?
+			if (strings.HasSuffix(remote, "/") || remote == "") && object.Size == 0 {
+				if recurse && remote != "" {
+					// add a directory in if --fast-list since will have no prefixes
+					err = fn(remote[:len(remote)-1], object, true)
+					if err != nil {
+						return err
+					}
+				}
+				continue // skip directory marker
+			}
 			err = fn(remote, object, false)
 			if err != nil {
 				return err
@@ -503,7 +555,11 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	}
 	listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
 	for {
-		buckets, err := listBuckets.Do()
+		var buckets *storage.Buckets
+		err = f.pacer.Call(func() (bool, error) {
+			buckets, err = listBuckets.Do()
+			return shouldRetry(err)
+		})
 		if err != nil {
 			return nil, err
 		}
@@ -591,13 +647,19 @@ func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption
 }
 
 // Mkdir creates the bucket if it doesn't exist
-func (f *Fs) Mkdir(dir string) error {
+func (f *Fs) Mkdir(dir string) (err error) {
 	f.bucketOKMu.Lock()
 	defer f.bucketOKMu.Unlock()
 	if f.bucketOK {
 		return nil
 	}
-	_, err := f.svc.Buckets.Get(f.bucket).Do()
+	// List something from the bucket to see if it exists.  Doing it like this enables the use of a
+	// service account that only has the "Storage Object Admin" role.  See #2193 for details.
+
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.svc.Objects.List(f.bucket).MaxResults(1).Do()
+		return shouldRetry(err)
+	})
 	if err == nil {
 		// Bucket already exists
 		f.bucketOK = true
@@ -619,7 +681,10 @@ func (f *Fs) Mkdir(dir string) error {
 		Location:     f.location,
 		StorageClass: f.storageClass,
 	}
-	_, err = f.svc.Buckets.Insert(f.projectNumber, &bucket).PredefinedAcl(f.bucketACL).Do()
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.svc.Buckets.Insert(f.projectNumber, &bucket).PredefinedAcl(f.bucketACL).Do()
+		return shouldRetry(err)
+	})
 	if err == nil {
 		f.bucketOK = true
 	}
@@ -630,13 +695,16 @@ func (f *Fs) Mkdir(dir string) error {
 //
 // Returns an error if it isn't empty: Error 409: The bucket you tried
 // to delete was not empty.
-func (f *Fs) Rmdir(dir string) error {
+func (f *Fs) Rmdir(dir string) (err error) {
 	f.bucketOKMu.Lock()
 	defer f.bucketOKMu.Unlock()
 	if f.root != "" || dir != "" {
 		return nil
 	}
-	err := f.svc.Buckets.Delete(f.bucket).Do()
+	err = f.pacer.Call(func() (bool, error) {
+		err = f.svc.Buckets.Delete(f.bucket).Do()
+		return shouldRetry(err)
+	})
 	if err == nil {
 		f.bucketOK = false
 	}
@@ -678,7 +746,11 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	srcObject := srcObj.fs.root + srcObj.remote
 	dstBucket := f.bucket
 	dstObject := f.root + remote
-	newObject, err := f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
+	var newObject *storage.Object
+	err = f.pacer.Call(func() (bool, error) {
+		newObject, err = f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -766,7 +838,11 @@ func (o *Object) readMetaData() (err error) {
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	object, err := o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
+	var object *storage.Object
+	err = o.fs.pacer.Call(func() (bool, error) {
+		object, err = o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		if gErr, ok := err.(*googleapi.Error); ok {
 			if gErr.Code == http.StatusNotFound {
@@ -800,14 +876,18 @@ func metadataFromModTime(modTime time.Time) map[string]string {
 }
 
 // SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(modTime time.Time) error {
+func (o *Object) SetModTime(modTime time.Time) (err error) {
 	// This only adds metadata so will perserve other metadata
 	object := storage.Object{
 		Bucket:   o.fs.bucket,
 		Name:     o.fs.root + o.remote,
 		Metadata: metadataFromModTime(modTime),
 	}
-	newObject, err := o.fs.svc.Objects.Patch(o.fs.bucket, o.fs.root+o.remote, &object).Do()
+	var newObject *storage.Object
+	err = o.fs.pacer.Call(func() (bool, error) {
+		newObject, err = o.fs.svc.Objects.Patch(o.fs.bucket, o.fs.root+o.remote, &object).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return err
 	}
@@ -827,7 +907,17 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		return nil, err
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
-	res, err := o.fs.client.Do(req)
+	var res *http.Response
+	err = o.fs.pacer.Call(func() (bool, error) {
+		res, err = o.fs.client.Do(req)
+		if err == nil {
+			err = googleapi.CheckResponse(res)
+			if err != nil {
+				_ = res.Body.Close() // ignore error
+			}
+		}
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -856,7 +946,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		Updated:     modTime.Format(timeFormatOut), // Doesn't get set
 		Metadata:    metadataFromModTime(modTime),
 	}
-	newObject, err := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.objectACL).Do()
+	var newObject *storage.Object
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.objectACL).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return err
 	}
@@ -866,8 +960,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 }
 
 // Remove an object
-func (o *Object) Remove() error {
-	return o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do()
+func (o *Object) Remove() (err error) {
+	err = o.fs.pacer.Call(func() (bool, error) {
+		err = o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do()
+		return shouldRetry(err)
+	})
+	return err
 }
 
 // MimeType of an Object if known, "" otherwise

backend/googlecloudstorage/googlecloudstorage_test.go

@@ -1,75 +1,17 @@
 // Test GoogleCloudStorage filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package googlecloudstorage_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/googlecloudstorage"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*googlecloudstorage.Object)(nil))
-	fstests.RemoteName = "TestGoogleCloudStorage:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestGoogleCloudStorage:",
+		NilObject:  (*googlecloudstorage.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/http/http.go

@@ -2,9 +2,6 @@
 //
 // It treats HTML pages served from the endpoint as directory
 // listings, and includes any links found as files.
-
-// +build go1.7
-
 package http
 
 import (
@@ -192,10 +189,11 @@ func (f *Fs) url(remote string) string {
 	return f.endpointURL + rest.URLPathEscape(remote)
 }
 
-func parseInt64(s string) int64 {
+// parse s into an int64, on failure return def
+func parseInt64(s string, def int64) int64 {
 	n, e := strconv.ParseInt(s, 10, 64)
 	if e != nil {
-		return 0
+		return def
 	}
 	return n
 }
@@ -412,7 +410,7 @@ func (o *Object) stat() error {
 	if err != nil {
 		t = timeUnset
 	}
-	o.size = parseInt64(res.Header.Get("Content-Length"))
+	o.size = parseInt64(res.Header.Get("Content-Length"), -1)
 	o.modTime = t
 	o.contentType = res.Header.Get("Content-Type")
 	return nil

backend/http/http_internal_test.go

@@ -151,6 +151,7 @@ func TestOpen(t *testing.T) {
 	fd, err := o.Open()
 	require.NoError(t, err)
 	data, err := ioutil.ReadAll(fd)
+	require.NoError(t, err)
 	require.NoError(t, fd.Close())
 	assert.Equal(t, "beetroot\n", string(data))
 
@@ -158,6 +159,7 @@ func TestOpen(t *testing.T) {
 	fd, err = o.Open(&fs.RangeOption{Start: 1, End: 5})
 	require.NoError(t, err)
 	data, err = ioutil.ReadAll(fd)
+	require.NoError(t, err)
 	require.NoError(t, fd.Close())
 	assert.Equal(t, "eetro", string(data))
 }

backend/http/http_unsupported.go

@@ -1,6 +0,0 @@
-// Build for mount for unsupported platforms to stop go complaining
-// about "no buildable Go source files "
-
-// +build !go1.7
-
-package http

backend/hubic/auth.go

@@ -36,7 +36,7 @@ func (a *auth) Response(resp *http.Response) error {
 
 // The public storage URL - set Internal to true to read
 // internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string {
+func (a *auth) StorageUrl(Internal bool) string { // nolint
 	return a.f.credentials.Endpoint
 }
 
@@ -46,7 +46,7 @@ func (a *auth) Token() string {
 }
 
 // The CDN url if available
-func (a *auth) CdnUrl() string {
+func (a *auth) CdnUrl() string { // nolint
 	return ""
 }
 

backend/hubic/hubic_test.go

@@ -1,75 +1,17 @@
 // Test Hubic filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package hubic_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/hubic"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*hubic.Object)(nil))
-	fstests.RemoteName = "TestHubic:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestHubic:",
+		NilObject:  (*hubic.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/local/about_unix.go

@@ -0,0 +1,29 @@
+// +build darwin dragonfly freebsd linux
+
+package local
+
+import (
+	"syscall"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+// About gets quota information
+func (f *Fs) About() (*fs.Usage, error) {
+	var s syscall.Statfs_t
+	err := syscall.Statfs(f.root, &s)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read disk usage")
+	}
+	bs := int64(s.Bsize)
+	usage := &fs.Usage{
+		Total: fs.NewUsageValue(bs * int64(s.Blocks)),         // quota of bytes that can be used
+		Used:  fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use
+		Free:  fs.NewUsageValue(bs * int64(s.Bavail)),         // bytes which can be uploaded before reaching the quota
+	}
+	return usage, nil
+}
+
+// check interface
+var _ fs.Abouter = &Fs{}

backend/local/about_windows.go

@@ -0,0 +1,36 @@
+// +build windows
+
+package local
+
+import (
+	"syscall"
+	"unsafe"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/pkg/errors"
+)
+
+var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
+
+// About gets quota information
+func (f *Fs) About() (*fs.Usage, error) {
+	var available, total, free int64
+	_, _, e1 := getFreeDiskSpace.Call(
+		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(f.root))),
+		uintptr(unsafe.Pointer(&available)), // lpFreeBytesAvailable - for this user
+		uintptr(unsafe.Pointer(&total)),     // lpTotalNumberOfBytes
+		uintptr(unsafe.Pointer(&free)),      // lpTotalNumberOfFreeBytes
+	)
+	if e1 != syscall.Errno(0) {
+		return nil, errors.Wrap(e1, "failed to read disk usage")
+	}
+	usage := &fs.Usage{
+		Total: fs.NewUsageValue(total),        // quota of bytes that can be used
+		Used:  fs.NewUsageValue(total - free), // bytes in use
+		Free:  fs.NewUsageValue(available),    // bytes which can be uploaded before reaching the quota
+	}
+	return usage, nil
+}
+
+// check interface
+var _ fs.Abouter = &Fs{}

backend/local/local.go

@@ -28,6 +28,7 @@ var (
 	followSymlinks = flags.BoolP("copy-links", "L", false, "Follow symlinks and copy the pointed to item.")
 	skipSymlinks   = flags.BoolP("skip-links", "", false, "Don't warn about skipped symlinks.")
 	noUTFNorm      = flags.BoolP("local-no-unicode-normalization", "", false, "Don't apply unicode normalization to paths and filenames")
+	noCheckUpdated = flags.BoolP("local-no-check-updated", "", false, "Don't check to see if the files change during upload")
 )
 
 // Constants
@@ -421,7 +422,7 @@ func (f *Fs) readPrecision() (precision time.Duration) {
 
 		// If it matches - have found the precision
 		// fmt.Println("compare", fi.ModTime(), t)
-		if fi.ModTime() == t {
+		if fi.ModTime().Equal(t) {
 			// fmt.Println("Precision detected as", duration)
 			return duration
 		}
@@ -592,7 +593,6 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		hashes = make(map[hash.Type]string)
 		in, err := os.Open(o.path)
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
@@ -642,11 +642,6 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	// On windows a file with os.ModeSymlink represents a file with reparse points
-	if runtime.GOOS == "windows" && (mode&os.ModeSymlink) != 0 {
-		fs.Debugf(o, "Clearing symlink bit to allow a file with reparse points to be copied")
-		mode &^= os.ModeSymlink
-	}
 	if mode&os.ModeSymlink != 0 {
 		if !*skipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")
@@ -673,13 +668,18 @@ type localOpenFile struct {
 
 // Read bytes from the object - see io.Reader
 func (file *localOpenFile) Read(p []byte) (n int, err error) {
-	// Check if file has the same size and modTime
-	fi, err := file.fd.Stat()
-	if err != nil {
-		return 0, errors.Wrap(err, "can't read status of source file while transferring")
-	}
-	if file.o.size != fi.Size() || file.o.modTime != fi.ModTime() {
-		return 0, errors.New("can't copy - source file is being updated")
+	if !*noCheckUpdated {
+		// Check if file has the same size and modTime
+		fi, err := file.fd.Stat()
+		if err != nil {
+			return 0, errors.Wrap(err, "can't read status of source file while transferring")
+		}
+		if file.o.size != fi.Size() {
+			return 0, errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size())
+		}
+		if !file.o.modTime.Equal(fi.ModTime()) {
+			return 0, errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime())
+		}
 	}
 
 	n, err = file.in.Read(p)
@@ -729,7 +729,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	wrappedFd := readers.NewLimitedReadCloser(fd, limit)
 	if offset != 0 {
 		// seek the object
-		_, err = fd.Seek(offset, 0)
+		_, err = fd.Seek(offset, io.SeekStart)
 		// don't attempt to make checksums
 		return wrappedFd, err
 	}
@@ -834,7 +834,7 @@ func (o *Object) lstat() error {
 
 // Remove an object
 func (o *Object) Remove() error {
-	return os.Remove(o.path)
+	return remove(o.path)
 }
 
 // Return the directory and file from an OS path. Assumes

backend/local/local_internal_test.go

@@ -29,8 +29,10 @@ func TestMapper(t *testing.T) {
 	})
 	assert.Equal(t, "potato", m.Load("potato"))
 	assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö"))
+}
 
-	// Test copy with source file that's updating
+// Test copy with source file that's updating
+func TestUpdatingCheck(t *testing.T) {
 	r := fstest.NewRun(t)
 	defer r.Finalise()
 	filePath := "sub dir/local test"
@@ -42,9 +44,11 @@ func TestMapper(t *testing.T) {
 	}
 
 	fi, err := fd.Stat()
+	require.NoError(t, err)
 	o := &Object{size: fi.Size(), modTime: fi.ModTime()}
 	wrappedFd := readers.NewLimitedReadCloser(fd, -1)
 	hash, err := hash.NewMultiHasherTypes(hash.Supported)
+	require.NoError(t, err)
 	in := localOpenFile{
 		o:    o,
 		in:   wrappedFd,
@@ -59,4 +63,16 @@ func TestMapper(t *testing.T) {
 	r.WriteFile(filePath, "content updated", time.Now())
 	_, err = in.Read(buf)
 	require.Errorf(t, err, "can't copy - source file is being updated")
+
+	// turn the checking off and try again
+
+	*noCheckUpdated = true
+	defer func() {
+		*noCheckUpdated = false
+	}()
+
+	r.WriteFile(filePath, "content updated", time.Now())
+	_, err = in.Read(buf)
+	require.NoError(t, err)
+
 }

backend/local/local_test.go

@@ -1,75 +1,17 @@
 // Test Local filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package local_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/local"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*local.Object)(nil))
-	fstests.RemoteName = ""
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "",
+		NilObject:  (*local.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/local/remove_other.go

@@ -0,0 +1,10 @@
+//+build !windows
+
+package local
+
+import "os"
+
+// Removes name, retrying on a sharing violation
+func remove(name string) error {
+	return os.Remove(name)
+}

backend/local/remove_test.go

@@ -0,0 +1,50 @@
+package local
+
+import (
+	"io/ioutil"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Check we can remove an open file
+func TestRemove(t *testing.T) {
+	fd, err := ioutil.TempFile("", "rclone-remove-test")
+	require.NoError(t, err)
+	name := fd.Name()
+	defer func() {
+		_ = os.Remove(name)
+	}()
+
+	exists := func() bool {
+		_, err := os.Stat(name)
+		if err == nil {
+			return true
+		} else if os.IsNotExist(err) {
+			return false
+		}
+		require.NoError(t, err)
+		return false
+	}
+
+	assert.True(t, exists())
+	// close the file in the background
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		time.Sleep(250 * time.Millisecond)
+		require.NoError(t, fd.Close())
+	}()
+	// delete the open file
+	err = remove(name)
+	require.NoError(t, err)
+	// check it no longer exists
+	assert.False(t, exists())
+	// wait for background close
+	wg.Wait()
+}

backend/local/remove_windows.go

@@ -0,0 +1,38 @@
+//+build windows
+
+package local
+
+import (
+	"os"
+	"syscall"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+)
+
+const (
+	ERROR_SHARING_VIOLATION syscall.Errno = 32
+)
+
+// Removes name, retrying on a sharing violation
+func remove(name string) (err error) {
+	const maxTries = 10
+	var sleepTime = 1 * time.Millisecond
+	for i := 0; i < maxTries; i++ {
+		err = os.Remove(name)
+		if err == nil {
+			break
+		}
+		pathErr, ok := err.(*os.PathError)
+		if !ok {
+			break
+		}
+		if pathErr.Err != ERROR_SHARING_VIOLATION {
+			break
+		}
+		fs.Logf(name, "Remove detected sharing violation - retry %d/%d sleeping %v", i+1, maxTries, sleepTime)
+		time.Sleep(sleepTime)
+		sleepTime <<= 1
+	}
+	return err
+}

backend/mega/mega_test.go

@@ -0,0 +1,17 @@
+// Test Mega filesystem interface
+package mega_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/backend/mega"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestMega:",
+		NilObject:  (*mega.Object)(nil),
+	})
+}

backend/onedrive/api/types.go

@@ -50,10 +50,10 @@ type IdentitySet struct {
 
 // Quota groups storage space quota-related information on OneDrive into a single structure.
 type Quota struct {
-	Total     int    `json:"total"`
-	Used      int    `json:"used"`
-	Remaining int    `json:"remaining"`
-	Deleted   int    `json:"deleted"`
+	Total     int64  `json:"total"`
+	Used      int64  `json:"used"`
+	Remaining int64  `json:"remaining"`
+	Deleted   int64  `json:"deleted"`
 	State     string `json:"state"` // normal | nearing | critical | exceeded
 }
 
@@ -100,8 +100,9 @@ type FolderFacet struct {
 
 // HashesType groups different types of hashes into a single structure, for an item on OneDrive.
 type HashesType struct {
-	Sha1Hash  string `json:"sha1Hash"`  // base64 encoded SHA1 hash for the contents of the file (if available)
-	Crc32Hash string `json:"crc32Hash"` // base64 encoded CRC32 value of the file (if available)
+	Sha1Hash     string `json:"sha1Hash"`     // hex encoded SHA1 hash for the contents of the file (if available)
+	Crc32Hash    string `json:"crc32Hash"`    // hex encoded CRC32 value of the file (if available)
+	QuickXorHash string `json:"quickXorHash"` // base64 encoded QuickXorHash value of the file (if available)
 }
 
 // FileFacet groups file-related data on OneDrive into a single structure.

backend/onedrive/onedrive.go

@@ -3,6 +3,8 @@
 package onedrive
 
 import (
+	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -10,7 +12,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -96,7 +97,15 @@ func init() {
 					log.Fatalf("Failed to configure token: %v", err)
 				}
 			} else {
-				err := oauthutil.Config("onedrive", name, oauthBusinessConfig, oauthBusinessResource)
+				err := oauthutil.ConfigErrorCheck("onedrive", name, func(req *http.Request) oauthutil.AuthError {
+					var resp oauthutil.AuthError
+
+					resp.Name = req.URL.Query().Get("error")
+					resp.Code = strings.Split(req.URL.Query().Get("error_description"), ":")[0] // error_description begins with XXXXXXXXXXXX:
+					resp.Description = strings.Join(strings.Split(req.URL.Query().Get("error_description"), ":")[1:], ":")
+					resp.HelpURL = "https://rclone.org/onedrive/#troubleshooting"
+					return resp
+				}, oauthBusinessConfig, oauthBusinessResource)
 				if err != nil {
 					log.Fatalf("Failed to configure token: %v", err)
 					return
@@ -245,14 +254,15 @@ type Fs struct {
 //
 // Will definitely have info but maybe not meta
 type Object struct {
-	fs          *Fs       // what this object is part of
-	remote      string    // The remote path
-	hasMetaData bool      // whether info below has been set
-	size        int64     // size of the object
-	modTime     time.Time // modification time of the object
-	id          string    // ID of the object
-	sha1        string    // SHA-1 of the object content
-	mimeType    string    // Content-Type of object from server (may not be as uploaded)
+	fs           *Fs       // what this object is part of
+	remote       string    // The remote path
+	hasMetaData  bool      // whether info below has been set
+	size         int64     // size of the object
+	modTime      time.Time // modification time of the object
+	id           string    // ID of the object
+	sha1         string    // SHA-1 of the object content
+	quickxorhash string    // QuickXorHash of the object content
+	mimeType     string    // Content-Type of object from server (may not be as uploaded)
 }
 
 // ------------------------------------------------------------
@@ -277,9 +287,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a one drive path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an one drive 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -922,8 +929,36 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
+// About gets quota information
+func (f *Fs) About() (usage *fs.Usage, err error) {
+	var drive api.Drive
+	opts := rest.Opts{
+		Method: "GET",
+		Path:   "",
+	}
+	var resp *http.Response
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(&opts, nil, &drive)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "about failed")
+	}
+	q := drive.Quota
+	usage = &fs.Usage{
+		Total:   fs.NewUsageValue(q.Total),     // quota of bytes that can be used
+		Used:    fs.NewUsageValue(q.Used),      // bytes in use
+		Trashed: fs.NewUsageValue(q.Deleted),   // bytes in trash
+		Free:    fs.NewUsageValue(q.Remaining), // bytes which can be uploaded before reaching the quota
+	}
+	return usage, nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
+	if f.isBusiness {
+		return hash.Set(hash.QuickXorHash)
+	}
 	return hash.Set(hash.SHA1)
 }
 
@@ -954,6 +989,12 @@ func (o *Object) srvPath() string {
 
 // Hash returns the SHA-1 of an object returning a lowercase hex string
 func (o *Object) Hash(t hash.Type) (string, error) {
+	if o.fs.isBusiness {
+		if t != hash.QuickXorHash {
+			return "", hash.ErrUnsupported
+		}
+		return o.quickxorhash, nil
+	}
 	if t != hash.SHA1 {
 		return "", hash.ErrUnsupported
 	}
@@ -978,18 +1019,22 @@ func (o *Object) setMetaData(info *api.Item) (err error) {
 	o.hasMetaData = true
 	o.size = info.Size
 
-	// Docs: https://dev.onedrive.com/facets/hashes_facet.htm
+	// Docs: https://docs.microsoft.com/en-us/onedrive/developer/rest-api/resources/hashes
 	//
-	// The docs state both that the hashes are returned as hex
-	// strings, and as base64 strings. Testing reveals they are in
-	// fact uppercase hex strings.
-	//
-	// In OneDrive for Business, SHA1 and CRC32 hash values are not returned for files.
+	// We use SHA1 for onedrive personal and QuickXorHash for onedrive for business
 	if info.File != nil {
 		o.mimeType = info.File.MimeType
 		if info.File.Hashes.Sha1Hash != "" {
 			o.sha1 = strings.ToLower(info.File.Hashes.Sha1Hash)
 		}
+		if info.File.Hashes.QuickXorHash != "" {
+			h, err := base64.StdEncoding.DecodeString(info.File.Hashes.QuickXorHash)
+			if err != nil {
+				fs.Errorf(o, "Failed to decode QuickXorHash %q: %v", info.File.Hashes.QuickXorHash, err)
+			} else {
+				o.quickxorhash = hex.EncodeToString(h)
+			}
+		}
 	}
 	if info.FileSystemInfo != nil {
 		o.modTime = time.Time(info.FileSystemInfo.LastModifiedDateTime)
@@ -1123,14 +1168,16 @@ func (o *Object) uploadFragment(url string, start int64, totalSize int64, chunk
 	//	var response api.UploadFragmentResponse
 	var resp *http.Response
 	err = o.fs.pacer.Call(func() (bool, error) {
-		_, _ = chunk.Seek(0, 0)
+		_, _ = chunk.Seek(0, io.SeekStart)
 		resp, err = o.fs.srv.Call(&opts)
+		if resp != nil {
+			defer fs.CheckClose(resp.Body, &err)
+		}
 		retry, err := shouldRetry(resp, err)
 		if !retry && resp != nil {
 			if resp.StatusCode == 200 || resp.StatusCode == 201 {
 				// we are done :)
 				// read the item
-				defer fs.CheckClose(resp.Body, &err)
 				info = &api.Item{}
 				return false, json.NewDecoder(resp.Body).Decode(info)
 			}
@@ -1263,6 +1310,11 @@ func (o *Object) MimeType() string {
 	return o.mimeType
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs     = (*Fs)(nil)
@@ -1271,6 +1323,8 @@ var (
 	_ fs.Mover  = (*Fs)(nil)
 	// _ fs.DirMover = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = &Object{}
+	_ fs.IDer            = &Object{}
 )

backend/onedrive/onedrive_test.go

@@ -1,75 +1,17 @@
 // Test OneDrive filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package onedrive_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/onedrive"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*onedrive.Object)(nil))
-	fstests.RemoteName = "TestOneDrive:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestOneDrive:",
+		NilObject:  (*onedrive.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/onedrive/quickxorhash/quickxorhash.go

@@ -0,0 +1,202 @@
+// Package quickxorhash provides the quickXorHash algorithm which is a
+// quick, simple non-cryptographic hash algorithm that works by XORing
+// the bytes in a circular-shifting fashion.
+//
+// It is used by Microsoft Onedrive for Business to hash data.
+//
+// See: https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
+package quickxorhash
+
+// This code was ported from the code snippet linked from
+// https://docs.microsoft.com/en-us/onedrive/developer/code-snippets/quickxorhash
+// Which has the copyright
+
+// ------------------------------------------------------------------------------
+//  Copyright (c) 2016 Microsoft Corporation
+//
+//  Permission is hereby granted, free of charge, to any person obtaining a copy
+//  of this software and associated documentation files (the "Software"), to deal
+//  in the Software without restriction, including without limitation the rights
+//  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+//  copies of the Software, and to permit persons to whom the Software is
+//  furnished to do so, subject to the following conditions:
+//
+//  The above copyright notice and this permission notice shall be included in
+//  all copies or substantial portions of the Software.
+//
+//  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+//  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+//  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+//  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+//  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+//  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+//  THE SOFTWARE.
+// ------------------------------------------------------------------------------
+
+import (
+	"hash"
+)
+
+const (
+	// BlockSize is the preferred size for hashing
+	BlockSize = 64
+	// Size of the output checksum
+	Size           = 20
+	bitsInLastCell = 32
+	shift          = 11
+	widthInBits    = 8 * Size
+	dataSize       = (widthInBits-1)/64 + 1
+)
+
+type quickXorHash struct {
+	data        [dataSize]uint64
+	lengthSoFar uint64
+	shiftSoFar  int
+}
+
+// New returns a new hash.Hash computing the quickXorHash checksum.
+func New() hash.Hash {
+	return &quickXorHash{}
+}
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// It never returns an error.
+//
+// Write writes len(p) bytes from p to the underlying data stream. It returns
+// the number of bytes written from p (0 <= n <= len(p)) and any error
+// encountered that caused the write to stop early. Write must return a non-nil
+// error if it returns n < len(p). Write must not modify the slice data, even
+// temporarily.
+//
+// Implementations must not retain p.
+func (q *quickXorHash) Write(p []byte) (n int, err error) {
+	currentshift := q.shiftSoFar
+
+	// The bitvector where we'll start xoring
+	vectorArrayIndex := currentshift / 64
+
+	// The position within the bit vector at which we begin xoring
+	vectorOffset := currentshift % 64
+	iterations := len(p)
+	if iterations > widthInBits {
+		iterations = widthInBits
+	}
+
+	for i := 0; i < iterations; i++ {
+		isLastCell := vectorArrayIndex == len(q.data)-1
+		var bitsInVectorCell int
+		if isLastCell {
+			bitsInVectorCell = bitsInLastCell
+		} else {
+			bitsInVectorCell = 64
+		}
+
+		// There's at least 2 bitvectors before we reach the end of the array
+		if vectorOffset <= bitsInVectorCell-8 {
+			for j := i; j < len(p); j += widthInBits {
+				q.data[vectorArrayIndex] ^= uint64(p[j]) << uint(vectorOffset)
+			}
+		} else {
+			index1 := vectorArrayIndex
+			var index2 int
+			if isLastCell {
+				index2 = 0
+			} else {
+				index2 = vectorArrayIndex + 1
+			}
+			low := byte(bitsInVectorCell - vectorOffset)
+
+			xoredByte := byte(0)
+			for j := i; j < len(p); j += widthInBits {
+				xoredByte ^= p[j]
+			}
+			q.data[index1] ^= uint64(xoredByte) << uint(vectorOffset)
+			q.data[index2] ^= uint64(xoredByte) >> low
+		}
+		vectorOffset += shift
+		for vectorOffset >= bitsInVectorCell {
+			if isLastCell {
+				vectorArrayIndex = 0
+			} else {
+				vectorArrayIndex = vectorArrayIndex + 1
+			}
+			vectorOffset -= bitsInVectorCell
+		}
+	}
+
+	// Update the starting position in a circular shift pattern
+	q.shiftSoFar = (q.shiftSoFar + shift*(len(p)%widthInBits)) % widthInBits
+
+	q.lengthSoFar += uint64(len(p))
+
+	return len(p), nil
+}
+
+// Calculate the current checksum
+func (q *quickXorHash) checkSum() (h [Size]byte) {
+	// Output the data as little endian bytes
+	ph := 0
+	for _, d := range q.data[:len(q.data)-1] {
+		_ = h[ph+7] // bounds check
+		h[ph+0] = byte(d >> (8 * 0))
+		h[ph+1] = byte(d >> (8 * 1))
+		h[ph+2] = byte(d >> (8 * 2))
+		h[ph+3] = byte(d >> (8 * 3))
+		h[ph+4] = byte(d >> (8 * 4))
+		h[ph+5] = byte(d >> (8 * 5))
+		h[ph+6] = byte(d >> (8 * 6))
+		h[ph+7] = byte(d >> (8 * 7))
+		ph += 8
+	}
+	// remaining 32 bits
+	d := q.data[len(q.data)-1]
+	h[Size-4] = byte(d >> (8 * 0))
+	h[Size-3] = byte(d >> (8 * 1))
+	h[Size-2] = byte(d >> (8 * 2))
+	h[Size-1] = byte(d >> (8 * 3))
+
+	// XOR the file length with the least significant bits in little endian format
+	d = q.lengthSoFar
+	h[Size-8] ^= byte(d >> (8 * 0))
+	h[Size-7] ^= byte(d >> (8 * 1))
+	h[Size-6] ^= byte(d >> (8 * 2))
+	h[Size-5] ^= byte(d >> (8 * 3))
+	h[Size-4] ^= byte(d >> (8 * 4))
+	h[Size-3] ^= byte(d >> (8 * 5))
+	h[Size-2] ^= byte(d >> (8 * 6))
+	h[Size-1] ^= byte(d >> (8 * 7))
+
+	return h
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (q *quickXorHash) Sum(b []byte) []byte {
+	hash := q.checkSum()
+	return append(b, hash[:]...)
+}
+
+// Reset resets the Hash to its initial state.
+func (q *quickXorHash) Reset() {
+	*q = quickXorHash{}
+}
+
+// Size returns the number of bytes Sum will return.
+func (q *quickXorHash) Size() int {
+	return Size
+}
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (q *quickXorHash) BlockSize() int {
+	return BlockSize
+}
+
+// Sum returns the quickXorHash checksum of the data.
+func Sum(data []byte) [Size]byte {
+	var d quickXorHash
+	_, _ = d.Write(data)
+	return d.checkSum()
+}

backend/onedrive/quickxorhash/quickxorhash_test.go

@@ -0,0 +1,168 @@
+package quickxorhash
+
+import (
+	"encoding/base64"
+	"fmt"
+	"hash"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var testVectors = []struct {
+	size int
+	in   string
+	out  string
+}{
+	{0, ``, "AAAAAAAAAAAAAAAAAAAAAAAAAAA="},
+	{1, `Sg==`, "SgAAAAAAAAAAAAAAAQAAAAAAAAA="},
+	{2, `tbQ=`, "taAFAAAAAAAAAAAAAgAAAAAAAAA="},
+	{3, `0pZP`, "0rDEEwAAAAAAAAAAAwAAAAAAAAA="},
+	{4, `jRRDVA==`, "jaDAEKgAAAAAAAAABAAAAAAAAAA="},
+	{5, `eAV52qE=`, "eChAHrQRCgAAAAAABQAAAAAAAAA="},
+	{6, `luBZlaT6`, "lgBHFipBCn0AAAAABgAAAAAAAAA="},
+	{7, `qaApEj66lw==`, "qQBFCiTgA11cAgAABwAAAAAAAAA="},
+	{8, `/aNzzCFPS/A=`, "/RjFHJgRgicsAR4ACAAAAAAAAAA="},
+	{9, `n6Neh7p6fFgm`, "nxiFFw6hCz3wAQsmCQAAAAAAAAA="},
+	{10, `J9iPGCbfZSTNyw==`, "J8DGIzBggm+UgQTNUgYAAAAAAAA="},
+	{11, `i+UZyUGJKh+ISbk=`, "iyhHBpIRhESo4AOIQ0IuAAAAAAA="},
+	{12, `h490d57Pqz5q2rtT`, "h3gEHe7giWeswgdq3MYupgAAAAA="},
+	{13, `vPgoDjOfO6fm71RxLw==`, "vMAHChwwg0/s4BTmdQcV4vACAAA="},
+	{14, `XoJ1AsoR4fDYJrDqYs4=`, "XhBEHQSgjAiEAx7YPgEs1CEGZwA="},
+	{15, `gQaybEqS/4UlDc8e4IJm`, "gDCALNigBEn8oxAlZ8AzPAAOQZg="},
+	{16, `2fuxhBJXtpWFe8dOfdGeHw==`, "O9tHLAghgSvYohKFyMMxnNCHaHg="},
+	{17, `XBV6YKU9V7yMakZnFIxIkuU=`, "HbplHsBQih5cgReMQYMRzkABRiA="},
+	{18, `XJZSOiNO2bmfKnTKD7fztcQX`, "/6ZArHQwAidkIxefQgEdlPGAW8w="},
+	{19, `g8VtAh+2Kf4k0kY5tzji2i2zmA==`, "wDNrgwHWAVukwB8kg4YRcnALHIg="},
+	{20, `T6LYJIfDh81JrAK309H2JMJTXis=`, "zBTHrspn3mEcohlJdIUAbjGNaNg="},
+	{21, `DWAAX5/CIfrmErgZa8ot6ZraeSbu`, "LR2Z0PjuRYGKQB/mhQAuMrAGZbQ="},
+	{22, `N9abi3qy/mC1THZuVLHPpx7SgwtLOA==`, "1KTYttCBEen8Hwy1doId3ECFWDw="},
+	{23, `LlUe7wHerLqEtbSZLZgZa9u0m7hbiFs=`, "TqVZpxs3cN61BnuFvwUtMtECTGQ="},
+	{24, `bU2j/0XYdgfPFD4691jV0AOUEUPR4Z5E`, "bnLBiLpVgnxVkXhNsIAPdHAPLFQ="},
+	{25, `lScPwPsyUsH2T1Qsr31wXtP55Wqbe47Uyg==`, "VDMSy8eI26nBHCB0e8gVWPCKPsA="},
+	{26, `rJaKh1dLR1k+4hynliTZMGf8Nd4qKKoZiAM=`, "r7bjwkl8OYQeNaMcCY8fTmEJEmQ="},
+	{27, `pPsT0CPmHrd3Frsnva1pB/z1ytARLeHEYRCo`, "Rdg7rCcDomL59pL0s6GuTvqLVqQ="},
+	{28, `wSRChaqmrsnMrfB2yqI43eRWbro+f9kBvh+01w==`, "YTtloIi6frI7HX3vdLvE7I2iUOA="},
+	{29, `apL67KMIRxQeE9k1/RuW09ppPjbF1WeQpTjSWtI=`, "CIpedls+ZlSQ654fl+X26+Q7LVU="},
+	{30, `53yx0/QgMTVb7OOzHRHbkS7ghyRc+sIXxi7XHKgT`, "zfJtLGFgR9DB3Q64fAFIp+S5iOY="},
+	{31, `PwXNnutoLLmxD8TTog52k8cQkukmT87TTnDipKLHQw==`, "PTaGs7yV3FUyBy/SfU6xJRlCJlI="},
+	{32, `NbYXsp5/K6mR+NmHwExjvWeWDJFnXTKWVlzYHoesp2E=`, "wjuAuWDiq04qDt1R8hHWDDcwVoQ="},
+	{33, `qQ70RB++JAR5ljNv3lJt1PpqETPsckopfonItu18Cr3E`, "FkJaeg/0Z5+euShYlLpE2tJh+Lo="},
+	{34, `RhzSatQTQ9/RFvpHyQa1WLdkr3nIk6MjJUma998YRtp44A==`, "SPN2D29reImAqJezlqV2DLbi8tk="},
+	{35, `DND1u1uZ5SqZVpRUk6NxSUdVo7IjjL9zs4A1evDNCDLcXWc=`, "S6lBk2hxI2SWBfn7nbEl7D19UUs="},
+	{36, `jEi62utFz69JMYHjg1iXy7oO6ZpZSLcVd2B+pjm6BGsv/CWi`, "s0lYU9tr/bp9xsnrrjYgRS5EvV8="},
+	{37, `hfS3DZZnhy0hv7nJdXLv/oJOtIgAuP9SInt/v8KeuO4/IvVh4A==`, "CV+HQCdd2A/e/vdi12f2UU55GLA="},
+	{38, `EkPQAC6ymuRrYjIXD/LT/4Vb+7aTjYVZOHzC8GPCEtYDP0+T3Nc=`, "kE9H9sEmr3vHBYUiPbvsrcDgSEo="},
+	{39, `vtBOGIENG7yQ/N7xNWPNIgy66Gk/I2Ur/ZhdFNUK9/1FCZuu/KeS`, "+Fgp3HBimtCzUAyiinj3pkarYTk="},
+	{40, `YnF4smoy9hox2jBlJ3VUa4qyCRhOZbWcmFGIiszTT4zAdYHsqJazyg==`, "arkIn+ELddmE8N34J9ydyFKW+9w="},
+	{41, `0n7nl3YJtipy6yeUbVPWtc2h45WbF9u8hTz5tNwj3dZZwfXWkk+GN3g=`, "YJLNK7JR64j9aODWfqDvEe/u6NU="},
+	{42, `FnIIPHayc1pHkY4Lh8+zhWwG8xk6Knk/D3cZU1/fOUmRAoJ6CeztvMOL`, "22RPOylMtdk7xO/QEQiMli4ql0k="},
+	{43, `J82VT7ND0Eg1MorSfJMUhn+qocF7PsUpdQAMrDiHJ2JcPZAHZ2nyuwjoKg==`, "pOR5eYfwCLRJbJsidpc1rIJYwtM="},
+	{44, `Zbu+78+e35ZIymV5KTDdub5McyI3FEO8fDxs62uWHQ9U3Oh3ZqgaZ30SnmQ=`, "DbvbTkgNTgWRqRidA9r1jhtUjro="},
+	{45, `lgybK3Da7LEeY5aeeNrqcdHvv6mD1W4cuQ3/rUj2C/CNcSI0cAMw6vtpVY3y`, "700RQByn1lRQSSme9npQB/Ye+bY="},
+	{46, `jStZgKHv4QyJLvF2bYbIUZi/FscHALfKHAssTXkrV1byVR9eACwW9DNZQRHQwg==`, "uwN55He8xgE4g93dH9163xPew4U="},
+	{47, `V1PSud3giF5WW72JB/bgtltsWtEB5V+a+wUALOJOGuqztzVXUZYrvoP3XV++gM0=`, "U+3ZfUF/6mwOoHJcSHkQkckfTDA="},
+	{48, `VXs4t4tfXGiWAL6dlhEMm0YQF0f2w9rzX0CvIVeuW56o6/ec2auMpKeU2VeteEK5`, "sq24lSf7wXLH8eigHl07X+qPTps="},
+	{49, `bLUn3jLH+HFUsG3ptWTHgNvtr3eEv9lfKBf0jm6uhpqhRwtbEQ7Ovj/hYQf42zfdtQ==`, "uC8xrnopGiHebGuwgq607WRQyxQ="},
+	{50, `4SVmjtXIL8BB8SfkbR5Cpaljm2jpyUfAhIBf65XmKxHlz9dy5XixgiE/q1lv+esZW/E=`, "wxZ0rxkMQEnRNAp8ZgEZLT4RdLM="},
+	{51, `pMljctlXeFUqbG3BppyiNbojQO3ygg6nZPeUZaQcVyJ+Clgiw3Q8ntLe8+02ZSfyCc39`, "aZEPmNvOXnTt7z7wt+ewV7QGMlg="},
+	{52, `C16uQlxsHxMWnV2gJhFPuJ2/guZ4N1YgmNvAwL1yrouGQtwieGx8WvZsmYRnX72JnbVtTw==`, "QtlSNqXhVij64MMhKJ3EsDFB/z8="},
+	{53, `7ZVDOywvrl3L0GyKjjcNg2CcTI81n2CeUbzdYWcZOSCEnA/xrNHpiK01HOcGh3BbxuS4S6g=`, "4NznNJc4nmXeApfiCFTq/H5LbHw="},
+	{54, `JXm2tTVqpYuuz2Cc+ZnPusUb8vccPGrzWK2oVwLLl/FjpFoxO9FxGlhnB08iu8Q/XQSdzHn+`, "IwE5+2pKNcK366I2k2BzZYPibSI="},
+	{55, `TiiU1mxzYBSGZuE+TX0l9USWBilQ7dEml5lLrzNPh75xmhjIK8SGqVAkvIMgAmcMB+raXdMPZg==`, "yECGHtgR128ScP4XlvF96eLbIBE="},
+	{56, `zz+Q4zi6wh0fCJUFU9yUOqEVxlIA93gybXHOtXIPwQQ44pW4fyh6BRgc1bOneRuSWp85hwlTJl8=`, "+3Ef4D6yuoC8J+rbFqU1cegverE="},
+	{57, `sa6SHK9z/G505bysK5KgRO2z2cTksDkLoFc7sv0tWBmf2G2mCiozf2Ce6EIO+W1fRsrrtn/eeOAV`, "xZg1CwMNAjN0AIXw2yh4+1N3oos="},
+	{58, `0qx0xdyTHhnKJ22IeTlAjRpWw6y2sOOWFP75XJ7cleGJQiV2kyrmQOST4DGHIL0qqA7sMOdzKyTV
+iw==`, "bS0tRYPkP1Gfc+ZsBm9PMzPunG8="},
+	{59, `QuzaF0+5ooig6OLEWeibZUENl8EaiXAQvK9UjBEauMeuFFDCtNcGs25BDtJGGbX90gH4VZvCCDNC
+q4s=`, "rggokuJq1OGNOfB6aDp2g4rdPgw="},
+	{60, `+wg2x23GZQmMLkdv9MeAdettIWDmyK6Wr+ba23XD+Pvvq1lIMn9QIQT4Z7QHJE3iC/ZMFgaId9VA
+yY3d`, "ahQbTmOdiKUNdhYRHgv5/Ky+Y6k="},
+	{61, `y0ydRgreRQwP95vpNP92ioI+7wFiyldHRbr1SfoPNdbKGFA0lBREaBEGNhf9yixmfE+Azo2AuROx
+b7Yc7g==`, "cJKFc0dXfiN4hMg1lcMf5E4gqvo="},
+	{62, `LxlVvGXSQlSubK8r0pGf9zf7s/3RHe75a2WlSXQf3gZFR/BtRnR7fCIcaG//CbGfodBFp06DBx/S
+9hUV8Bk=`, "NwuwhhRWX8QZ/vhWKWgQ1+rNomI="},
+	{63, `L+LSB8kmGMnHaWVA5P/+qFnfQliXvgJW7d2JGAgT6+koi5NQujFW1bwQVoXrBVyob/gBxGizUoJM
+gid5gGNo`, "ndX/KZBtFoeO3xKeo1ajO/Jy+rY="},
+	{64, `Mb7EGva2rEE5fENDL85P+BsapHEEjv2/siVhKjvAQe02feExVOQSkfmuYzU/kTF1MaKjPmKF/w+c
+bvwfdWL8aQ==`, "n1anP5NfvD4XDYWIeRPW3ZkPv1Y="},
+	{111, `jyibxJSzO6ZiZ0O1qe3tG/bvIAYssvukh9suIT5wEy1JBINVgPiqdsTW0cOpP0aUfP7mgqLfADkz
+I/m/GgCuVhr8oFLrOCoTx1/psBOWwhltCbhUx51Icm9aH8tY4Z3ccU+6BKpYQkLCy0B/A9Zc`, "hZfLIilSITC6N3e3tQ/iSgEzkto="},
+	{128, `ikwCorI7PKWz17EI50jZCGbV9JU2E8bXVfxNMg5zdmqSZ2NlsQPp0kqYIPjzwTg1MBtfWPg53k0h
+0P2naJNEVgrqpoHTfV2b3pJ4m0zYPTJmUX4Bg/lOxcnCxAYKU29Y5F0U8Quz7ZXFBEweftXxJ7RS
+4r6N7BzJrPsLhY7hgck=`, "imAoFvCWlDn4yVw3/oq1PDbbm6U="},
+	{222, `PfxMcUd0vIW6VbHG/uj/Y0W6qEoKmyBD0nYebEKazKaKG+UaDqBEcmQjbfQeVnVLuodMoPp7P7TR
+1htX5n2VnkHh22xDyoJ8C/ZQKiSNqQfXvh83judf4RVr9exJCud8Uvgip6aVZTaPrJHVjQhMCp/d
+EnGvqg0oN5OVkM2qqAXvA0teKUDhgNM71sDBVBCGXxNOR2bpbD1iM4dnuT0ey4L+loXEHTL0fqMe
+UcEi2asgImnlNakwenDzz0x57aBwyq3AspCFGB1ncX4yYCr/OaCcS5OKi/00WH+wNQU3`, "QX/YEpG0gDsmhEpCdWhsxDzsfVE="},
+	{256, `qwGf2ESubE5jOUHHyc94ORczFYYbc2OmEzo+hBIyzJiNwAzC8PvJqtTzwkWkSslgHFGWQZR2BV5+
+uYTrYT7HVwRM40vqfj0dBgeDENyTenIOL1LHkjtDKoXEnQ0mXAHoJ8PjbNC93zi5TovVRXTNzfGE
+s5dpWVqxUzb5lc7dwkyvOluBw482mQ4xrzYyIY1t+//OrNi1ObGXuUw2jBQOFfJVj2Y6BOyYmfB1
+y36eBxi3zxeG5d5NYjm2GSh6e08QMAwu3zrINcqIzLOuNIiGXBtl7DjKt7b5wqi4oFiRpZsCyx2s
+mhSrdrtK/CkdU6nDN+34vSR/M8rZpWQdBE7a8g==`, "WYT9JY3JIo/pEBp+tIM6Gt2nyTM="},
+	{333, `w0LGhqU1WXFbdavqDE4kAjEzWLGGzmTNikzqnsiXHx2KRReKVTxkv27u3UcEz9+lbMvYl4xFf2Z4
+aE1xRBBNd1Ke5C0zToSaYw5o4B/7X99nKK2/XaUX1byLow2aju2XJl2OpKpJg+tSJ2fmjIJTkfuY
+Uz574dFX6/VXxSxwGH/xQEAKS5TCsBK3CwnuG1p5SAsQq3gGVozDWyjEBcWDMdy8/AIFrj/y03Lf
+c/RNRCQTAfZbnf2QwV7sluw4fH3XJr07UoD0YqN+7XZzidtrwqMY26fpLZnyZjnBEt1FAZWO7RnK
+G5asg8xRk9YaDdedXdQSJAOy6bWEWlABj+tVAigBxavaluUH8LOj+yfCFldJjNLdi90fVHkUD/m4
+Mr5OtmupNMXPwuG3EQlqWUVpQoYpUYKLsk7a5Mvg6UFkiH596y5IbJEVCI1Kb3D1`, "e3+wo77iKcILiZegnzyUNcjCdoQ="},
+}
+
+func TestQuickXorHash(t *testing.T) {
+	for _, test := range testVectors {
+		what := fmt.Sprintf("test size %d", test.size)
+		in, err := base64.StdEncoding.DecodeString(test.in)
+		require.NoError(t, err, what)
+		got := Sum(in)
+		want, err := base64.StdEncoding.DecodeString(test.out)
+		require.NoError(t, err, what)
+		assert.Equal(t, want, got[:], what)
+	}
+}
+
+func TestQuickXorHashByBlock(t *testing.T) {
+	for _, blockSize := range []int{1, 2, 4, 7, 8, 16, 32, 64, 128, 256, 512} {
+		for _, test := range testVectors {
+			what := fmt.Sprintf("test size %d blockSize %d", test.size, blockSize)
+			in, err := base64.StdEncoding.DecodeString(test.in)
+			require.NoError(t, err, what)
+			h := New()
+			for i := 0; i < len(in); i += blockSize {
+				end := i + blockSize
+				if end > len(in) {
+					end = len(in)
+				}
+				n, err := h.Write(in[i:end])
+				require.Equal(t, end-i, n, what)
+				require.NoError(t, err, what)
+			}
+			got := h.Sum(nil)
+			want, err := base64.StdEncoding.DecodeString(test.out)
+			require.NoError(t, err, what)
+			assert.Equal(t, want, got[:], test.size, what)
+		}
+	}
+}
+
+func TestSize(t *testing.T) {
+	d := New()
+	assert.Equal(t, 20, d.Size())
+}
+
+func TestBlockSize(t *testing.T) {
+	d := New()
+	assert.Equal(t, 64, d.BlockSize())
+}
+
+func TestReset(t *testing.T) {
+	d := New()
+	zeroHash := d.Sum(nil)
+	_, _ = d.Write([]byte{1})
+	assert.NotEqual(t, zeroHash, d.Sum(nil))
+	d.Reset()
+	assert.Equal(t, zeroHash, d.Sum(nil))
+}
+
+// check interface
+var _ hash.Hash = (*quickXorHash)(nil)

backend/opendrive/opendrive_test.go

@@ -0,0 +1,17 @@
+// Test Opendrive filesystem interface
+package opendrive_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/backend/opendrive"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestOpenDrive:",
+		NilObject:  (*opendrive.Object)(nil),
+	})
+}

backend/opendrive/replace.go

@@ -0,0 +1,78 @@
+/*
+Translate file names for OpenDrive
+
+OpenDrive reserved characters
+
+The following characters are OpenDrive reserved characters, and can't
+be used in OpenDrive folder and file names.
+
+\ / : * ? " < > |
+
+OpenDrive files and folders can't have leading or trailing spaces also.
+
+*/
+
+package opendrive
+
+import (
+	"regexp"
+	"strings"
+)
+
+// charMap holds replacements for characters
+//
+// OpenDrive has a restricted set of characters compared to other cloud
+// storage systems, so we to map these to the FULLWIDTH unicode
+// equivalents
+//
+// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
+var (
+	charMap = map[rune]rune{
+		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+		':':  '：', // FULLWIDTH COLON
+		'*':  '＊', // FULLWIDTH ASTERISK
+		'?':  '？', // FULLWIDTH QUESTION MARK
+		'"':  '＂', // FULLWIDTH QUOTATION MARK
+		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
+		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
+		'|':  '｜', // FULLWIDTH VERTICAL LINE
+		' ':  '␠', // SYMBOL FOR SPACE
+	}
+	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
+	fixEndingWithSpace   = regexp.MustCompile(` (/|$)`)
+	invCharMap           map[rune]rune
+)
+
+func init() {
+	// Create inverse charMap
+	invCharMap = make(map[rune]rune, len(charMap))
+	for k, v := range charMap {
+		invCharMap[v] = k
+	}
+}
+
+// replaceReservedChars takes a path and substitutes any reserved
+// characters in it
+func replaceReservedChars(in string) string {
+	// Filenames can't start with space
+	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
+	// Filenames can't end with space
+	in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
+	return strings.Map(func(c rune) rune {
+		if replacement, ok := charMap[c]; ok && c != ' ' {
+			return replacement
+		}
+		return c
+	}, in)
+}
+
+// restoreReservedChars takes a path and undoes any substitutions
+// made by replaceReservedChars
+func restoreReservedChars(in string) string {
+	return strings.Map(func(c rune) rune {
+		if replacement, ok := invCharMap[c]; ok {
+			return replacement
+		}
+		return c
+	}, in)
+}

backend/opendrive/replace_test.go

@@ -0,0 +1,28 @@
+package opendrive
+
+import "testing"
+
+func TestReplace(t *testing.T) {
+	for _, test := range []struct {
+		in  string
+		out string
+	}{
+		{"", ""},
+		{"abc 123", "abc 123"},
+		{`\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~`},
+		{`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~/＼＊＜＞？：｜#%＂.~`},
+		{" leading space", "␠leading space"},
+		{" path/  leading spaces", "␠path/␠ leading spaces"},
+		{"trailing space ", "trailing space␠"},
+		{"trailing spaces  /path ", "trailing spaces ␠/path␠"},
+	} {
+		got := replaceReservedChars(test.in)
+		if got != test.out {
+			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
+		}
+		got2 := restoreReservedChars(got)
+		if got2 != test.in {
+			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
+		}
+	}
+}

backend/opendrive/types.go

@@ -0,0 +1,214 @@
+package opendrive
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// Error describes an openDRIVE error response
+type Error struct {
+	Info struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	} `json:"error"`
+}
+
+// Error statisfies the error interface
+func (e *Error) Error() string {
+	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
+}
+
+// Account describes a OpenDRIVE account
+type Account struct {
+	Username string `json:"username"`
+	Password string `json:"passwd"`
+}
+
+// UserSessionInfo describes a OpenDRIVE session
+type UserSessionInfo struct {
+	Username string `json:"username"`
+	Password string `json:"passwd"`
+
+	SessionID          string          `json:"SessionID"`
+	UserName           string          `json:"UserName"`
+	UserFirstName      string          `json:"UserFirstName"`
+	UserLastName       string          `json:"UserLastName"`
+	AccType            string          `json:"AccType"`
+	UserLang           string          `json:"UserLang"`
+	UserID             string          `json:"UserID"`
+	IsAccountUser      json.RawMessage `json:"IsAccountUser"`
+	DriveName          string          `json:"DriveName"`
+	UserLevel          string          `json:"UserLevel"`
+	UserPlan           string          `json:"UserPlan"`
+	FVersioning        string          `json:"FVersioning"`
+	UserDomain         string          `json:"UserDomain"`
+	PartnerUsersDomain string          `json:"PartnerUsersDomain"`
+}
+
+// FolderList describes a OpenDRIVE listing
+type FolderList struct {
+	// DirUpdateTime    string   `json:"DirUpdateTime,string"`
+	Name             string   `json:"Name"`
+	ParentFolderID   string   `json:"ParentFolderID"`
+	DirectFolderLink string   `json:"DirectFolderLink"`
+	ResponseType     int      `json:"ResponseType"`
+	Folders          []Folder `json:"Folders"`
+	Files            []File   `json:"Files"`
+}
+
+// Folder describes a OpenDRIVE folder
+type Folder struct {
+	FolderID      string `json:"FolderID"`
+	Name          string `json:"Name"`
+	DateCreated   int    `json:"DateCreated"`
+	DirUpdateTime int    `json:"DirUpdateTime"`
+	Access        int    `json:"Access"`
+	DateModified  int64  `json:"DateModified"`
+	Shared        string `json:"Shared"`
+	ChildFolders  int    `json:"ChildFolders"`
+	Link          string `json:"Link"`
+	Encrypted     string `json:"Encrypted"`
+}
+
+type createFolder struct {
+	SessionID           string `json:"session_id"`
+	FolderName          string `json:"folder_name"`
+	FolderSubParent     string `json:"folder_sub_parent"`
+	FolderIsPublic      int64  `json:"folder_is_public"`      // (0 = private, 1 = public, 2 = hidden)
+	FolderPublicUpl     int64  `json:"folder_public_upl"`     // (0 = disabled, 1 = enabled)
+	FolderPublicDisplay int64  `json:"folder_public_display"` // (0 = disabled, 1 = enabled)
+	FolderPublicDnl     int64  `json:"folder_public_dnl"`     // (0 = disabled, 1 = enabled).
+}
+
+type createFolderResponse struct {
+	FolderID      string `json:"FolderID"`
+	Name          string `json:"Name"`
+	DateCreated   int    `json:"DateCreated"`
+	DirUpdateTime int    `json:"DirUpdateTime"`
+	Access        int    `json:"Access"`
+	DateModified  int    `json:"DateModified"`
+	Shared        string `json:"Shared"`
+	Description   string `json:"Description"`
+	Link          string `json:"Link"`
+}
+
+type moveCopyFolder struct {
+	SessionID     string `json:"session_id"`
+	FolderID      string `json:"folder_id"`
+	DstFolderID   string `json:"dst_folder_id"`
+	Move          string `json:"move"`
+	NewFolderName string `json:"new_folder_name"` // New name for destination folder.
+}
+
+type moveCopyFolderResponse struct {
+	FolderID string `json:"FolderID"`
+}
+
+type removeFolder struct {
+	SessionID string `json:"session_id"`
+	FolderID  string `json:"folder_id"`
+}
+
+// File describes a OpenDRIVE file
+type File struct {
+	FileID            string `json:"FileId"`
+	FileHash          string `json:"FileHash"`
+	Name              string `json:"Name"`
+	GroupID           int    `json:"GroupID"`
+	Extension         string `json:"Extension"`
+	Size              int64  `json:"Size,string"`
+	Views             string `json:"Views"`
+	Version           string `json:"Version"`
+	Downloads         string `json:"Downloads"`
+	DateModified      int64  `json:"DateModified,string"`
+	Access            string `json:"Access"`
+	Link              string `json:"Link"`
+	DownloadLink      string `json:"DownloadLink"`
+	StreamingLink     string `json:"StreamingLink"`
+	TempStreamingLink string `json:"TempStreamingLink"`
+	EditLink          string `json:"EditLink"`
+	ThumbLink         string `json:"ThumbLink"`
+	Password          string `json:"Password"`
+	EditOnline        int    `json:"EditOnline"`
+}
+
+type moveCopyFile struct {
+	SessionID         string `json:"session_id"`
+	SrcFileID         string `json:"src_file_id"`
+	DstFolderID       string `json:"dst_folder_id"`
+	Move              string `json:"move"`
+	OverwriteIfExists string `json:"overwrite_if_exists"`
+	NewFileName       string `json:"new_file_name"` // New name for destination file.
+}
+
+type moveCopyFileResponse struct {
+	FileID string `json:"FileID"`
+	Size   string `json:"Size"`
+}
+
+type createFile struct {
+	SessionID string `json:"session_id"`
+	FolderID  string `json:"folder_id"`
+	Name      string `json:"file_name"`
+}
+
+type createFileResponse struct {
+	FileID             string `json:"FileId"`
+	Name               string `json:"Name"`
+	GroupID            int    `json:"GroupID"`
+	Extension          string `json:"Extension"`
+	Size               string `json:"Size"`
+	Views              string `json:"Views"`
+	Downloads          string `json:"Downloads"`
+	DateModified       string `json:"DateModified"`
+	Access             string `json:"Access"`
+	Link               string `json:"Link"`
+	DownloadLink       string `json:"DownloadLink"`
+	StreamingLink      string `json:"StreamingLink"`
+	TempStreamingLink  string `json:"TempStreamingLink"`
+	DirUpdateTime      int    `json:"DirUpdateTime"`
+	TempLocation       string `json:"TempLocation"`
+	SpeedLimit         int    `json:"SpeedLimit"`
+	RequireCompression int    `json:"RequireCompression"`
+	RequireHash        int    `json:"RequireHash"`
+	RequireHashOnly    int    `json:"RequireHashOnly"`
+}
+
+type modTimeFile struct {
+	SessionID            string `json:"session_id"`
+	FileID               string `json:"file_id"`
+	FileModificationTime string `json:"file_modification_time"`
+}
+
+type openUpload struct {
+	SessionID string `json:"session_id"`
+	FileID    string `json:"file_id"`
+	Size      int64  `json:"file_size"`
+}
+
+type openUploadResponse struct {
+	TempLocation       string `json:"TempLocation"`
+	RequireCompression bool   `json:"RequireCompression"`
+	RequireHash        bool   `json:"RequireHash"`
+	RequireHashOnly    bool   `json:"RequireHashOnly"`
+	SpeedLimit         int    `json:"SpeedLimit"`
+}
+
+type closeUpload struct {
+	SessionID    string `json:"session_id"`
+	FileID       string `json:"file_id"`
+	Size         int64  `json:"file_size"`
+	TempLocation string `json:"temp_location"`
+}
+
+type closeUploadResponse struct {
+	FileID   string `json:"FileID"`
+	FileHash string `json:"FileHash"`
+	Size     int64  `json:"Size"`
+}
+
+type permissions struct {
+	SessionID    string `json:"session_id"`
+	FileID       string `json:"file_id"`
+	FileIsPublic int64  `json:"file_ispublic"`
+}

backend/pcloud/api/types.go

@@ -151,3 +151,35 @@ type ChecksumFileResult struct {
 	Hashes
 	Metadata Item `json:"metadata"`
 }
+
+// UserInfo is returned from /userinfo
+type UserInfo struct {
+	Error
+	Cryptosetup           bool   `json:"cryptosetup"`
+	Plan                  int    `json:"plan"`
+	CryptoSubscription    bool   `json:"cryptosubscription"`
+	PublicLinkQuota       int64  `json:"publiclinkquota"`
+	Email                 string `json:"email"`
+	UserID                int    `json:"userid"`
+	Result                int    `json:"result"`
+	Quota                 int64  `json:"quota"`
+	TrashRevretentionDays int    `json:"trashrevretentiondays"`
+	Premium               bool   `json:"premium"`
+	PremiumLifetime       bool   `json:"premiumlifetime"`
+	EmailVerified         bool   `json:"emailverified"`
+	UsedQuota             int64  `json:"usedquota"`
+	Language              string `json:"language"`
+	Business              bool   `json:"business"`
+	CryptoLifetime        bool   `json:"cryptolifetime"`
+	Registered            string `json:"registered"`
+	Journey               struct {
+		Claimed bool `json:"claimed"`
+		Steps   struct {
+			VerifyMail    bool `json:"verifymail"`
+			UploadFile    bool `json:"uploadfile"`
+			AutoUpload    bool `json:"autoupload"`
+			DownloadApp   bool `json:"downloadapp"`
+			DownloadDrive bool `json:"downloaddrive"`
+		} `json:"steps"`
+	} `json:"journey"`
+}

backend/pcloud/pcloud.go

@@ -17,7 +17,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -130,9 +129,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a pcloud path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an pcloud 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -806,6 +802,30 @@ func (f *Fs) DirCacheFlush() {
 	f.dirCache.ResetRoot()
 }
 
+// About gets quota information
+func (f *Fs) About() (usage *fs.Usage, err error) {
+	opts := rest.Opts{
+		Method: "POST",
+		Path:   "/userinfo",
+	}
+	var resp *http.Response
+	var q api.UserInfo
+	err = f.pacer.Call(func() (bool, error) {
+		resp, err = f.srv.CallJSON(&opts, nil, &q)
+		err = q.Error.Update(err)
+		return shouldRetry(resp, err)
+	})
+	if err != nil {
+		return nil, errors.Wrap(err, "about failed")
+	}
+	usage = &fs.Usage{
+		Total: fs.NewUsageValue(q.Quota),               // quota of bytes that can be used
+		Used:  fs.NewUsageValue(q.UsedQuota),           // bytes in use
+		Free:  fs.NewUsageValue(q.Quota - q.UsedQuota), // bytes which can be uploaded before reaching the quota
+	}
+	return usage, nil
+}
+
 // Hashes returns the supported hash sets.
 func (f *Fs) Hashes() hash.Set {
 	return hash.Set(hash.MD5 | hash.SHA1)
@@ -1098,6 +1118,11 @@ func (o *Object) Remove() error {
 	})
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1107,5 +1132,7 @@ var (
 	_ fs.Mover           = (*Fs)(nil)
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
+	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/pcloud/pcloud_test.go

@@ -1,75 +1,17 @@
 // Test Pcloud filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package pcloud_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/pcloud"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*pcloud.Object)(nil))
-	fstests.RemoteName = "TestPcloud:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestPcloud:",
+		NilObject:  (*pcloud.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/qingstor/qingstor.go

@@ -1,7 +1,7 @@
 // Package qingstor provides an interface to QingStor object storage
 // Home: https://www.qingcloud.com/
 
-// +build !plan9,go1.7
+// +build !plan9
 
 package qingstor
 

backend/qingstor/qingstor_test.go

@@ -1,9 +1,6 @@
 // Test QingStor filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 
-// +build !plan9,go1.7
+// +build !plan9
 
 package qingstor_test
 
@@ -11,68 +8,13 @@ import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/qingstor"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*qingstor.Object)(nil))
-	fstests.RemoteName = "TestQingStor:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestQingStor:",
+		NilObject:  (*qingstor.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/qingstor/qingstor_unsupported.go

@@ -1,6 +1,6 @@
 // Build for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 !go1.7
+// +build plan9
 
 package qingstor

backend/qingstor/upload.go

@@ -1,6 +1,6 @@
 // Upload object to QingStor
 
-// +build !plan9,go1.7
+// +build !plan9
 
 package qingstor
 
@@ -19,10 +19,10 @@ import (
 )
 
 const (
-	maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
-	maxMultiPartSize  = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
-	minMultiPartSize  = 1024 * 1024 * 4        // The minimum allowed part size when uploading a part to QingStor
-	maxMultiParts     = 10000                  // The maximum allowed number of parts in an multi-part upload
+	// maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
+	// maxMultiPartSize = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
+	minMultiPartSize = 1024 * 1024 * 4 // The minimum allowed part size when uploading a part to QingStor
+	maxMultiParts    = 10000           // The maximum allowed number of parts in an multi-part upload
 )
 
 const (
@@ -130,12 +130,12 @@ func (u *uploader) init() {
 	u.totalSize = -1
 	switch r := u.cfg.body.(type) {
 	case io.Seeker:
-		pos, _ := r.Seek(0, 1)
+		pos, _ := r.Seek(0, io.SeekCurrent)
 		defer func() {
-			_, _ = r.Seek(pos, 0)
+			_, _ = r.Seek(pos, io.SeekStart)
 		}()
 
-		n, err := r.Seek(0, 2)
+		n, err := r.Seek(0, io.SeekEnd)
 		if err != nil {
 			return
 		}

backend/s3/s3.go

@@ -51,21 +51,46 @@ import (
 func init() {
 	fs.Register(&fs.RegInfo{
 		Name:        "s3",
-		Description: "Amazon S3 (also Dreamhost, Ceph, Minio, IBM COS)",
+		Description: "Amazon S3 Compliant Storage Providers (AWS, Ceph, Dreamhost, IBM COS, Minio)",
 		NewFs:       NewFs,
-		// AWS endpoints: http://docs.amazonwebservices.com/general/latest/gr/rande.html#s3_region
 		Options: []fs.Option{{
+			Name: fs.ConfigProvider,
+			Help: "Choose your S3 provider.",
+			Examples: []fs.OptionExample{{
+				Value: "AWS",
+				Help:  "Amazon Web Services (AWS) S3",
+			}, {
+				Value: "Ceph",
+				Help:  "Ceph Object Storage",
+			}, {
+				Value: "DigitalOcean",
+				Help:  "Digital Ocean Spaces",
+			}, {
+				Value: "Dreamhost",
+				Help:  "Dreamhost DreamObjects",
+			}, {
+				Value: "IBMCOS",
+				Help:  "IBM COS S3",
+			}, {
+				Value: "Minio",
+				Help:  "Minio Object Storage",
+			}, {
+				Value: "Wasabi",
+				Help:  "Wasabi Object Storage",
+			}, {
+				Value: "Other",
+				Help:  "Any other S3 compatible provider",
+			}},
+		}, {
 			Name: "env_auth",
 			Help: "Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.",
-			Examples: []fs.OptionExample{
-				{
-					Value: "false",
-					Help:  "Enter AWS credentials in the next step",
-				}, {
-					Value: "true",
-					Help:  "Get AWS credentials from the environment (env vars or IAM)",
-				},
-			},
+			Examples: []fs.OptionExample{{
+				Value: "false",
+				Help:  "Enter AWS credentials in the next step",
+			}, {
+				Value: "true",
+				Help:  "Get AWS credentials from the environment (env vars or IAM)",
+			}},
 		}, {
 			Name: "access_key_id",
 			Help: "AWS Access Key ID - leave blank for anonymous access or runtime credentials.",
@@ -73,8 +98,9 @@ func init() {
 			Name: "secret_access_key",
 			Help: "AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.",
 		}, {
-			Name: "region",
-			Help: "Region to connect to.  Leave blank if you are using an S3 clone and you don't have a region.",
+			Name:     "region",
+			Help:     "Region to connect to.",
+			Provider: "AWS",
 			Examples: []fs.OptionExample{{
 				Value: "us-east-1",
 				Help:  "The default endpoint - a good choice if you are unsure.\nUS Region, Northern Virginia or Pacific Northwest.\nLeave location constraint empty.",
@@ -117,16 +143,158 @@ func init() {
 			}, {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
-			}, {
-				Value: "other-v2-signature",
-				Help:  "Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.\nSet this and make sure you set the endpoint.",
 			}},
 		}, {
-			Name: "endpoint",
-			Help: "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.\nSpecify if using an S3 clone such as Ceph.",
+			Name:     "region",
+			Help:     "Region to connect to.  Leave blank if you are using an S3 clone and you don't have a region.",
+			Provider: "!AWS",
+			Examples: []fs.OptionExample{{
+				Value: "",
+				Help:  "Use this if unsure. Will use v4 signatures and an empty region.",
+			}, {
+				Value: "other-v2-signature",
+				Help:  "Use this only if v4 signatures don't work, eg pre Jewel/v10 CEPH.",
+			}},
 		}, {
-			Name: "location_constraint",
-			Help: "Location constraint - must be set to match the Region. Used when creating buckets only.",
+			Name:     "endpoint",
+			Help:     "Endpoint for S3 API.\nLeave blank if using AWS to use the default endpoint for the region.",
+			Provider: "AWS",
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for IBM COS S3 API.\nSpecify if using an IBM COS On Premise.",
+			Provider: "IBMCOS",
+			Examples: []fs.OptionExample{{
+				Value: "s3-api.us-geo.objectstorage.softlayer.net",
+				Help:  "US Cross Region Endpoint",
+			}, {
+				Value: "s3-api.dal.us-geo.objectstorage.softlayer.net",
+				Help:  "US Cross Region Dallas Endpoint",
+			}, {
+				Value: "s3-api.wdc-us-geo.objectstorage.softlayer.net",
+				Help:  "US Cross Region Washington DC Endpoint",
+			}, {
+				Value: "s3-api.sjc-us-geo.objectstorage.softlayer.net",
+				Help:  "US Cross Region San Jose Endpoint",
+			}, {
+				Value: "s3-api.us-geo.objectstorage.service.networklayer.com",
+				Help:  "US Cross Region Private Endpoint",
+			}, {
+				Value: "s3-api.dal-us-geo.objectstorage.service.networklayer.com",
+				Help:  "US Cross Region Dallas Private Endpoint",
+			}, {
+				Value: "s3-api.wdc-us-geo.objectstorage.service.networklayer.com",
+				Help:  "US Cross Region Washington DC Private Endpoint",
+			}, {
+				Value: "s3-api.sjc-us-geo.objectstorage.service.networklayer.com",
+				Help:  "US Cross Region San Jose Private Endpoint",
+			}, {
+				Value: "s3.us-east.objectstorage.softlayer.net",
+				Help:  "US Region East Endpoint",
+			}, {
+				Value: "s3.us-east.objectstorage.service.networklayer.com",
+				Help:  "US Region East Private Endpoint",
+			}, {
+				Value: "s3.us-south.objectstorage.softlayer.net",
+				Help:  "US Region South Endpoint",
+			}, {
+				Value: "s3.us-south.objectstorage.service.networklayer.com",
+				Help:  "US Region South Private Endpoint",
+			}, {
+				Value: "s3.eu-geo.objectstorage.softlayer.net",
+				Help:  "EU Cross Region Endpoint",
+			}, {
+				Value: "s3.fra-eu-geo.objectstorage.softlayer.net",
+				Help:  "EU Cross Region Frankfurt Endpoint",
+			}, {
+				Value: "s3.mil-eu-geo.objectstorage.softlayer.net",
+				Help:  "EU Cross Region Milan Endpoint",
+			}, {
+				Value: "s3.ams-eu-geo.objectstorage.softlayer.net",
+				Help:  "EU Cross Region Amsterdam Endpoint",
+			}, {
+				Value: "s3.eu-geo.objectstorage.service.networklayer.com",
+				Help:  "EU Cross Region Private Endpoint",
+			}, {
+				Value: "s3.fra-eu-geo.objectstorage.service.networklayer.com",
+				Help:  "EU Cross Region Frankfurt Private Endpoint",
+			}, {
+				Value: "s3.mil-eu-geo.objectstorage.service.networklayer.com",
+				Help:  "EU Cross Region Milan Private Endpoint",
+			}, {
+				Value: "s3.ams-eu-geo.objectstorage.service.networklayer.com",
+				Help:  "EU Cross Region Amsterdam Private Endpoint",
+			}, {
+				Value: "s3.eu-gb.objectstorage.softlayer.net",
+				Help:  "Great Britan Endpoint",
+			}, {
+				Value: "s3.eu-gb.objectstorage.service.networklayer.com",
+				Help:  "Great Britan Private Endpoint",
+			}, {
+				Value: "s3.ap-geo.objectstorage.softlayer.net",
+				Help:  "APAC Cross Regional Endpoint",
+			}, {
+				Value: "s3.tok-ap-geo.objectstorage.softlayer.net",
+				Help:  "APAC Cross Regional Tokyo Endpoint",
+			}, {
+				Value: "s3.hkg-ap-geo.objectstorage.softlayer.net",
+				Help:  "APAC Cross Regional HongKong Endpoint",
+			}, {
+				Value: "s3.seo-ap-geo.objectstorage.softlayer.net",
+				Help:  "APAC Cross Regional Seoul Endpoint",
+			}, {
+				Value: "s3.ap-geo.objectstorage.service.networklayer.com",
+				Help:  "APAC Cross Regional Private Endpoint",
+			}, {
+				Value: "s3.tok-ap-geo.objectstorage.service.networklayer.com",
+				Help:  "APAC Cross Regional Tokyo Private Endpoint",
+			}, {
+				Value: "s3.hkg-ap-geo.objectstorage.service.networklayer.com",
+				Help:  "APAC Cross Regional HongKong Private Endpoint",
+			}, {
+				Value: "s3.seo-ap-geo.objectstorage.service.networklayer.com",
+				Help:  "APAC Cross Regional Seoul Private Endpoint",
+			}, {
+				Value: "s3.mel01.objectstorage.softlayer.net",
+				Help:  "Melbourne Single Site Endpoint",
+			}, {
+				Value: "s3.mel01.objectstorage.service.networklayer.com",
+				Help:  "Melbourne Single Site Private Endpoint",
+			}, {
+				Value: "s3.tor01.objectstorage.softlayer.net",
+				Help:  "Toronto Single Site Endpoint",
+			}, {
+				Value: "s3.tor01.objectstorage.service.networklayer.com",
+				Help:  "Toronto Single Site Private Endpoint",
+			}},
+		}, {
+			Name:     "endpoint",
+			Help:     "Endpoint for S3 API.\nRequired when using an S3 clone.",
+			Provider: "!AWS,IBMCOS",
+			Examples: []fs.OptionExample{{
+				Value:    "objects-us-west-1.dream.io",
+				Help:     "Dream Objects endpoint",
+				Provider: "Dreamhost",
+			}, {
+				Value:    "nyc3.digitaloceanspaces.com",
+				Help:     "Digital Ocean Spaces New York 3",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "ams3.digitaloceanspaces.com",
+				Help:     "Digital Ocean Spaces Amsterdam 3",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "sgp1.digitaloceanspaces.com",
+				Help:     "Digital Ocean Spaces Singapore 1",
+				Provider: "DigitalOcean",
+			}, {
+				Value:    "s3.wasabisys.com",
+				Help:     "Wasabi Object Storage",
+				Provider: "Wasabi",
+			}},
+		}, {
+			Name:     "location_constraint",
+			Help:     "Location constraint - must be set to match the Region. Used when creating buckets only.",
+			Provider: "AWS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Empty for US Region, Northern Virginia or Pacific Northwest.",
@@ -170,31 +338,159 @@ func init() {
 				Value: "sa-east-1",
 				Help:  "South America (Sao Paulo) Region.",
 			}},
+		}, {
+			Name:     "location_constraint",
+			Help:     "Location constraint - must match endpoint when using IBM Cloud Public. For on-prem COS, do not make a selection from this list, hit enter",
+			Provider: "IBMCOS",
+			Examples: []fs.OptionExample{{
+				Value: "us-standard",
+				Help:  "US Cross Region Standard",
+			}, {
+				Value: "us-vault",
+				Help:  "US Cross Region Vault",
+			}, {
+				Value: "us-cold",
+				Help:  "US Cross Region Cold",
+			}, {
+				Value: "us-flex",
+				Help:  "US Cross Region Flex",
+			}, {
+				Value: "us-east-standard",
+				Help:  "US East Region Standard",
+			}, {
+				Value: "us-east-vault",
+				Help:  "US East Region Vault",
+			}, {
+				Value: "us-east-cold",
+				Help:  "US East Region Cold",
+			}, {
+				Value: "us-east-flex",
+				Help:  "US East Region Flex",
+			}, {
+				Value: "us-south-standard",
+				Help:  "US Sout hRegion Standard",
+			}, {
+				Value: "us-south-vault",
+				Help:  "US South Region Vault",
+			}, {
+				Value: "us-south-cold",
+				Help:  "US South Region Cold",
+			}, {
+				Value: "us-south-flex",
+				Help:  "US South Region Flex",
+			}, {
+				Value: "eu-standard",
+				Help:  "EU Cross Region Standard",
+			}, {
+				Value: "eu-vault",
+				Help:  "EU Cross Region Vault",
+			}, {
+				Value: "eu-cold",
+				Help:  "EU Cross Region Cold",
+			}, {
+				Value: "eu-flex",
+				Help:  "EU Cross Region Flex",
+			}, {
+				Value: "eu-gb-standard",
+				Help:  "Great Britan Standard",
+			}, {
+				Value: "eu-gb-vault",
+				Help:  "Great Britan Vault",
+			}, {
+				Value: "eu-gb-cold",
+				Help:  "Great Britan Cold",
+			}, {
+				Value: "eu-gb-flex",
+				Help:  "Great Britan Flex",
+			}, {
+				Value: "ap-standard",
+				Help:  "APAC Standard",
+			}, {
+				Value: "ap-vault",
+				Help:  "APAC Vault",
+			}, {
+				Value: "ap-cold",
+				Help:  "APAC Cold",
+			}, {
+				Value: "ap-flex",
+				Help:  "APAC Flex",
+			}, {
+				Value: "mel01-standard",
+				Help:  "Melbourne Standard",
+			}, {
+				Value: "mel01-vault",
+				Help:  "Melbourne Vault",
+			}, {
+				Value: "mel01-cold",
+				Help:  "Melbourne Cold",
+			}, {
+				Value: "mel01-flex",
+				Help:  "Melbourne Flex",
+			}, {
+				Value: "tor01-standard",
+				Help:  "Toronto Standard",
+			}, {
+				Value: "tor01-vault",
+				Help:  "Toronto Vault",
+			}, {
+				Value: "tor01-cold",
+				Help:  "Toronto Cold",
+			}, {
+				Value: "tor01-flex",
+				Help:  "Toronto Flex",
+			}},
+		}, {
+			Name:     "location_constraint",
+			Help:     "Location constraint - must be set to match the Region. Leave blank if not sure. Used when creating buckets only.",
+			Provider: "!AWS,IBMCOS",
 		}, {
 			Name: "acl",
 			Help: "Canned ACL used when creating buckets and/or storing objects in S3.\nFor more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl",
 			Examples: []fs.OptionExample{{
-				Value: "private",
-				Help:  "Owner gets FULL_CONTROL. No one else has access rights (default).",
+				Value:    "private",
+				Help:     "Owner gets FULL_CONTROL. No one else has access rights (default).",
+				Provider: "!IBMCOS",
 			}, {
-				Value: "public-read",
-				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
+				Value:    "public-read",
+				Help:     "Owner gets FULL_CONTROL. The AllUsers group gets READ access.",
+				Provider: "!IBMCOS",
 			}, {
-				Value: "public-read-write",
-				Help:  "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.",
+				Value:    "public-read-write",
+				Help:     "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access.\nGranting this on a bucket is generally not recommended.",
+				Provider: "!IBMCOS",
 			}, {
-				Value: "authenticated-read",
-				Help:  "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.",
+				Value:    "authenticated-read",
+				Help:     "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access.",
+				Provider: "!IBMCOS",
 			}, {
-				Value: "bucket-owner-read",
-				Help:  "Object owner gets FULL_CONTROL. Bucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.",
+				Value:    "bucket-owner-read",
+				Help:     "Object owner gets FULL_CONTROL. Bucket owner gets READ access.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.",
+				Provider: "!IBMCOS",
 			}, {
-				Value: "bucket-owner-full-control",
-				Help:  "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.",
+				Value:    "bucket-owner-full-control",
+				Help:     "Both the object owner and the bucket owner get FULL_CONTROL over the object.\nIf you specify this canned ACL when creating a bucket, Amazon S3 ignores it.",
+				Provider: "!IBMCOS",
+			}, {
+				Value:    "private",
+				Help:     "Owner gets FULL_CONTROL. No one else has access rights (default). This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise COS",
+				Provider: "IBMCOS",
+			}, {
+				Value:    "public-read",
+				Help:     "Owner gets FULL_CONTROL. The AllUsers group gets READ access. This acl is available on IBM Cloud (Infra), IBM Cloud (Storage), On-Premise IBM COS",
+				Provider: "IBMCOS",
+			}, {
+				Value:    "public-read-write",
+				Help:     "Owner gets FULL_CONTROL. The AllUsers group gets READ and WRITE access. This acl is available on IBM Cloud (Infra), On-Premise IBM COS",
+				Provider: "IBMCOS",
+			}, {
+				Value:    "authenticated-read",
+				Help:     "Owner gets FULL_CONTROL. The AuthenticatedUsers group gets READ access. Not supported on Buckets. This acl is available on IBM Cloud (Infra) and On-Premise IBM COS",
+				Provider: "IBMCOS",
 			}},
 		}, {
-			Name: "server_side_encryption",
-			Help: "The server-side encryption algorithm used when storing this object in S3.",
+			Name:     "server_side_encryption",
+			Help:     "The server-side encryption algorithm used when storing this object in S3.",
+			Provider: "AWS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "None",
@@ -203,8 +499,9 @@ func init() {
 				Help:  "AES256",
 			}},
 		}, {
-			Name: "storage_class",
-			Help: "The storage class to use when storing objects in S3.",
+			Name:     "storage_class",
+			Help:     "The storage class to use when storing objects in S3.",
+			Provider: "AWS",
 			Examples: []fs.OptionExample{{
 				Value: "",
 				Help:  "Default",
@@ -217,9 +514,14 @@ func init() {
 			}, {
 				Value: "STANDARD_IA",
 				Help:  "Standard Infrequent Access storage class",
+			}, {
+				Value: "ONEZONE_IA",
+				Help:  "One Zone Infrequent Access storage class",
 			}},
-		}},
+		},
+		},
 	})
+	flags.VarP(&s3ChunkSize, "s3-chunk-size", "", "Chunk size to use for uploading")
 }
 
 // Constants
@@ -235,8 +537,11 @@ const (
 // Globals
 var (
 	// Flags
-	s3ACL          = flags.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
-	s3StorageClass = flags.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA)")
+	s3ACL               = flags.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
+	s3StorageClass      = flags.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA)")
+	s3ChunkSize         = fs.SizeSuffix(s3manager.MinUploadPartSize)
+	s3DisableChecksum   = flags.BoolP("s3-disable-checksum", "", false, "Don't store MD5 checksum with object metadata")
+	s3UploadConcurrency = flags.IntP("s3-upload-concurrency", "", 2, "Concurrency for multipart uploads")
 )
 
 // Fs represents a remote s3 server
@@ -336,6 +641,11 @@ func s3Connection(name string) (*s3.S3, *session.Session, error) {
 		// * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY
 		&credentials.EnvProvider{},
 
+		// A SharedCredentialsProvider retrieves credentials
+		// from the current user's home directory.  It checks
+		// AWS_SHARED_CREDENTIALS_FILE and AWS_PROFILE too.
+		&credentials.SharedCredentialsProvider{},
+
 		// Pick up IAM role if we're in an ECS task
 		defaults.RemoteCredProvider(*def.Config, def.Handlers),
 
@@ -427,6 +737,9 @@ func NewFs(name, root string) (fs.Fs, error) {
 	if *s3StorageClass != "" {
 		f.storageClass = *s3StorageClass
 	}
+	if s3ChunkSize < fs.SizeSuffix(s3manager.MinUploadPartSize) {
+		return nil, errors.Errorf("s3 chunk size must be >= %v", fs.SizeSuffix(s3manager.MinUploadPartSize))
+	}
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the object exists
@@ -549,6 +862,18 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 				continue
 			}
 			remote := key[rootLength:]
+			// is this a directory marker?
+			if (strings.HasSuffix(remote, "/") || remote == "") && *object.Size == 0 {
+				if recurse && remote != "" {
+					// add a directory in if --fast-list since will have no prefixes
+					remote = remote[:len(remote)-1]
+					err = fn(remote, &s3.Object{Key: &remote}, true)
+					if err != nil {
+						return err
+					}
+				}
+				continue // skip directory marker
+			}
 			err = fn(remote, object, false)
 			if err != nil {
 				return err
@@ -928,6 +1253,9 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
+	if fs.Config.UseServerModTime {
+		return o.lastModified
+	}
 	err := o.readMetaData()
 	if err != nil {
 		fs.Logf(o, "Failed to read metadata: %v", err)
@@ -1025,10 +1353,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	size := src.Size()
 
 	uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
-		u.Concurrency = 2
+		u.Concurrency = *s3UploadConcurrency
 		u.LeavePartsOnError = false
 		u.S3 = o.fs.c
-		u.PartSize = s3manager.MinUploadPartSize
+		u.PartSize = int64(s3ChunkSize)
 
 		if size == -1 {
 			// Make parts as small as possible while still being able to upload to the
@@ -1048,7 +1376,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		metaMtime: aws.String(swift.TimeToFloatString(modTime)),
 	}
 
-	if size > uploader.PartSize {
+	if !*s3DisableChecksum && size > uploader.PartSize {
 		hash, err := src.Hash(hash.MD5)
 
 		if err == nil && matchMd5.MatchString(hash) {

backend/s3/s3_test.go

@@ -1,75 +1,17 @@
 // Test S3 filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package s3_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/s3"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*s3.Object)(nil))
-	fstests.RemoteName = "TestS3:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestS3:",
+		NilObject:  (*s3.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/sftp/sftp.go

@@ -1,10 +1,12 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9
+// +build !plan9,go1.9
 
 package sftp
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -27,7 +29,6 @@ import (
 	"github.com/pkg/sftp"
 	"github.com/xanzy/ssh-agent"
 	"golang.org/x/crypto/ssh"
-	"golang.org/x/net/context"
 	"golang.org/x/time/rate"
 )
 
@@ -40,6 +41,7 @@ var (
 
 	// Flags
 	sftpAskPassword = flags.BoolP("sftp-ask-password", "", false, "Allow asking for SFTP password when needed.")
+	sshPathOverride = flags.StringP("ssh-path-override", "", "", "Override path used by SSH connection.")
 )
 
 func init() {
@@ -74,7 +76,7 @@ func init() {
 			Optional: true,
 		}, {
 			Name:     "use_insecure_cipher",
-			Help:     "Enable the user of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker..",
+			Help:     "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
 			Optional: true,
 			Examples: []fs.OptionExample{
 				{
@@ -87,7 +89,7 @@ func init() {
 			},
 		}, {
 			Name:     "disable_hashcheck",
-			Help:     "Disable the exectution of SSH commands to determine if remote file hashing is available, leave blank unless you know what you are doing.",
+			Help:     "Disable the execution of SSH commands to determine if remote file hashing is available. Leave blank or set to false to enable hashing (recommended), set to true to disable hashing.",
 			Optional: true,
 		}},
 	}
@@ -123,12 +125,6 @@ type Object struct {
 	sha1sum *string     // Cached SHA1 checksum
 }
 
-// ObjectReader holds the sftp.File interface to a remote SFTP file opened for reading
-type ObjectReader struct {
-	object   *Object
-	sftpFile *sftp.File
-}
-
 // readCurrentUser finds the current user name or "" if not found
 func readCurrentUser() (userName string) {
 	usr, err := user.Current()
@@ -338,7 +334,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 
 	// Ask for password if none was defined and we're allowed to
 	if pass == "" && *sftpAskPassword {
-		fmt.Fprint(os.Stderr, "Enter SFTP password: ")
+		_, _ = fmt.Fprint(os.Stderr, "Enter SFTP password: ")
 		clearpass := config.ReadPassword()
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
@@ -733,40 +729,47 @@ func (o *Object) Remote() string {
 // Hash returns the selected checksum of the file
 // If no checksum is available it returns ""
 func (o *Object) Hash(r hash.Type) (string, error) {
-	if r == hash.MD5 && o.md5sum != nil {
-		return *o.md5sum, nil
-	} else if r == hash.SHA1 && o.sha1sum != nil {
-		return *o.sha1sum, nil
+	var hashCmd string
+	if r == hash.MD5 {
+		if o.md5sum != nil {
+			return *o.md5sum, nil
+		}
+		hashCmd = "md5sum"
+	} else if r == hash.SHA1 {
+		if o.sha1sum != nil {
+			return *o.sha1sum, nil
+		}
+		hashCmd = "sha1sum"
+	} else {
+		return "", hash.ErrUnsupported
 	}
 
 	c, err := o.fs.getSftpConnection()
 	if err != nil {
-		return "", errors.Wrap(err, "Hash")
+		return "", errors.Wrap(err, "Hash get SFTP connection")
 	}
 	session, err := c.sshClient.NewSession()
 	o.fs.putSftpConnection(&c, err)
 	if err != nil {
-		o.fs.cachedHashes = nil // Something has changed on the remote system
-		return "", hash.ErrUnsupported
+		return "", errors.Wrap(err, "Hash put SFTP connection")
 	}
 
-	err = hash.ErrUnsupported
-	var outputBytes []byte
+	var stdout, stderr bytes.Buffer
+	session.Stdout = &stdout
+	session.Stderr = &stderr
 	escapedPath := shellEscape(o.path())
-	if r == hash.MD5 {
-		outputBytes, err = session.Output("md5sum " + escapedPath)
-	} else if r == hash.SHA1 {
-		outputBytes, err = session.Output("sha1sum " + escapedPath)
+	if *sshPathOverride != "" {
+		escapedPath = shellEscape(path.Join(*sshPathOverride, o.remote))
 	}
-
+	err = session.Run(hashCmd + " " + escapedPath)
 	if err != nil {
-		o.fs.cachedHashes = nil // Something has changed on the remote system
 		_ = session.Close()
-		return "", hash.ErrUnsupported
+		fs.Debugf(o, "Failed to calculate %v hash: %v (%s)", r, err, bytes.TrimSpace(stderr.Bytes()))
+		return "", nil
 	}
 
 	_ = session.Close()
-	str := parseHash(outputBytes)
+	str := parseHash(stdout.Bytes())
 	if r == hash.MD5 {
 		o.md5sum = &str
 	} else if r == hash.SHA1 {
@@ -868,15 +871,49 @@ func (o *Object) Storable() bool {
 	return o.mode.IsRegular()
 }
 
+// objectReader represents a file open for reading on the SFTP server
+type objectReader struct {
+	sftpFile   *sftp.File
+	pipeReader *io.PipeReader
+	done       chan struct{}
+}
+
+func newObjectReader(sftpFile *sftp.File) *objectReader {
+	pipeReader, pipeWriter := io.Pipe()
+	file := &objectReader{
+		sftpFile:   sftpFile,
+		pipeReader: pipeReader,
+		done:       make(chan struct{}),
+	}
+
+	go func() {
+		// Use sftpFile.WriteTo to pump data so that it gets a
+		// chance to build the window up.
+		_, err := sftpFile.WriteTo(pipeWriter)
+		// Close the pipeWriter so the pipeReader fails with
+		// the same error or EOF if err == nil
+		_ = pipeWriter.CloseWithError(err)
+		// signal that we've finished
+		close(file.done)
+	}()
+
+	return file
+}
+
 // Read from a remote sftp file object reader
-func (file *ObjectReader) Read(p []byte) (n int, err error) {
-	n, err = file.sftpFile.Read(p)
+func (file *objectReader) Read(p []byte) (n int, err error) {
+	n, err = file.pipeReader.Read(p)
 	return n, err
 }
 
 // Close a reader of a remote sftp file
-func (file *ObjectReader) Close() (err error) {
+func (file *objectReader) Close() (err error) {
+	// Close the sftpFile - this will likely cause the WriteTo to error
 	err = file.sftpFile.Close()
+	// Close the pipeReader so writes to the pipeWriter fail
+	_ = file.pipeReader.Close()
+	// Wait for the background process to finish
+	<-file.done
 	return err
 }
 
@@ -905,15 +942,12 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		return nil, errors.Wrap(err, "Open failed")
 	}
 	if offset > 0 {
-		off, err := sftpFile.Seek(offset, 0)
+		off, err := sftpFile.Seek(offset, io.SeekStart)
 		if err != nil || off != offset {
 			return nil, errors.Wrap(err, "Open Seek failed")
 		}
 	}
-	in = readers.NewLimitedReadCloser(&ObjectReader{
-		object:   o,
-		sftpFile: sftpFile,
-	}, limit)
+	in = readers.NewLimitedReadCloser(newObjectReader(sftpFile), limit)
 	return in, nil
 }
 

backend/sftp/sftp_internal_test.go

@@ -1,3 +1,5 @@
+// +build !plan9,go1.9
+
 package sftp
 
 import (

backend/sftp/sftp_test.go

@@ -1,75 +1,20 @@
 // Test Sftp filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
+
+// +build !plan9,go1.9
+
 package sftp_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/sftp"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*sftp.Object)(nil))
-	fstests.RemoteName = "TestSftp:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSftp:",
+		NilObject:  (*sftp.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9
+// +build plan9 !go1.9
 
 package sftp

backend/sftp/stringlock.go

@@ -1,3 +1,5 @@
+// +build !plan9,go1.9
+
 package sftp
 
 import "sync"

backend/sftp/stringlock_test.go

@@ -1,3 +1,5 @@
+// +build !plan9,go1.9
+
 package sftp
 
 import (

backend/swift/auth.go

@@ -44,7 +44,7 @@ func (a *auth) Response(resp *http.Response) error {
 
 // The public storage URL - set Internal to true to read
 // internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string {
+func (a *auth) StorageUrl(Internal bool) string { // nolint
 	if a.storageURL != "" {
 		return a.storageURL
 	}
@@ -66,7 +66,7 @@ func (a *auth) Token() string {
 }
 
 // The CDN url if available
-func (a *auth) CdnUrl() string {
+func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
 		return ""
 	}

backend/swift/swift.go

@@ -221,7 +221,7 @@ func swiftConnection(name string) (*swift.Connection, error) {
 			return nil, errors.Wrap(err, "failed to read environment variables")
 		}
 	}
-	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken
+	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
 	if !c.Authenticated() {
 		if c.UserName == "" && c.UserId == "" {
 			return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
@@ -391,7 +391,8 @@ func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error {
 			err = fn(d)
 		} else {
 			// newObjectWithInfo does a full metadata read on 0 size objects which might be dynamic large objects
-			o, err := f.newObjectWithInfo(remote, object)
+			var o fs.Object
+			o, err = f.newObjectWithInfo(remote, object)
 			if err != nil {
 				return err
 			}
@@ -498,6 +499,24 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
 	return list.Flush()
 }
 
+// About gets quota information
+func (f *Fs) About() (*fs.Usage, error) {
+	containers, err := f.c.ContainersAll(nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "container listing failed")
+	}
+	var total, objects int64
+	for _, c := range containers {
+		total += c.Bytes
+		objects += c.Count
+	}
+	usage := &fs.Usage{
+		Used:    fs.NewUsageValue(total),   // bytes in use
+		Objects: fs.NewUsageValue(objects), // objects in use
+	}
+	return usage, nil
+}
+
 // Put the object into the container
 //
 // Copy the reader in to the new object which is returned
@@ -719,6 +738,9 @@ func (o *Object) readMetaData() (err error) {
 // It attempts to read the objects mtime and if that isn't present the
 // LastModified returned in the http headers
 func (o *Object) ModTime() time.Time {
+	if fs.Config.UseServerModTime {
+		return o.info.LastModified
+	}
 	err := o.readMetaData()
 	if err != nil {
 		fs.Debugf(o, "Failed to read metadata: %s", err)
@@ -826,7 +848,7 @@ func urlEncode(str string) string {
 // container.  It returns a string which prefixes current segments.
 func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64, contentType string) (string, error) {
 	// Create the segmentsContainer if it doesn't exist
-	var err error = swift.ContainerNotFound
+	var err error
 	_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
 	if err == swift.ContainerNotFound {
 		err = o.fs.c.ContainerCreate(o.fs.segmentsContainer, nil)

backend/swift/swift_test.go

@@ -1,75 +1,17 @@
 // Test Swift filesystem interface
-//
-// Automatically generated - DO NOT EDIT
-// Regenerate with: make gen_tests
 package swift_test
 
 import (
 	"testing"
 
 	"github.com/ncw/rclone/backend/swift"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fstest/fstests"
 )
 
-func TestSetup(t *testing.T) {
-	fstests.NilObject = fs.Object((*swift.Object)(nil))
-	fstests.RemoteName = "TestSwift:"
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestSwift:",
+		NilObject:  (*swift.Object)(nil),
+	})
 }
-
-// Generic tests for the Fs
-func TestInit(t *testing.T)                { fstests.TestInit(t) }
-func TestFsString(t *testing.T)            { fstests.TestFsString(t) }
-func TestFsName(t *testing.T)              { fstests.TestFsName(t) }
-func TestFsRoot(t *testing.T)              { fstests.TestFsRoot(t) }
-func TestFsRmdirEmpty(t *testing.T)        { fstests.TestFsRmdirEmpty(t) }
-func TestFsRmdirNotFound(t *testing.T)     { fstests.TestFsRmdirNotFound(t) }
-func TestFsMkdir(t *testing.T)             { fstests.TestFsMkdir(t) }
-func TestFsMkdirRmdirSubdir(t *testing.T)  { fstests.TestFsMkdirRmdirSubdir(t) }
-func TestFsListEmpty(t *testing.T)         { fstests.TestFsListEmpty(t) }
-func TestFsListDirEmpty(t *testing.T)      { fstests.TestFsListDirEmpty(t) }
-func TestFsListRDirEmpty(t *testing.T)     { fstests.TestFsListRDirEmpty(t) }
-func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) }
-func TestFsPutFile1(t *testing.T)          { fstests.TestFsPutFile1(t) }
-func TestFsPutError(t *testing.T)          { fstests.TestFsPutError(t) }
-func TestFsPutFile2(t *testing.T)          { fstests.TestFsPutFile2(t) }
-func TestFsUpdateFile1(t *testing.T)       { fstests.TestFsUpdateFile1(t) }
-func TestFsListDirFile2(t *testing.T)      { fstests.TestFsListDirFile2(t) }
-func TestFsListRDirFile2(t *testing.T)     { fstests.TestFsListRDirFile2(t) }
-func TestFsListDirRoot(t *testing.T)       { fstests.TestFsListDirRoot(t) }
-func TestFsListRDirRoot(t *testing.T)      { fstests.TestFsListRDirRoot(t) }
-func TestFsListSubdir(t *testing.T)        { fstests.TestFsListSubdir(t) }
-func TestFsListRSubdir(t *testing.T)       { fstests.TestFsListRSubdir(t) }
-func TestFsListLevel2(t *testing.T)        { fstests.TestFsListLevel2(t) }
-func TestFsListRLevel2(t *testing.T)       { fstests.TestFsListRLevel2(t) }
-func TestFsListFile1(t *testing.T)         { fstests.TestFsListFile1(t) }
-func TestFsNewObject(t *testing.T)         { fstests.TestFsNewObject(t) }
-func TestFsListFile1and2(t *testing.T)     { fstests.TestFsListFile1and2(t) }
-func TestFsNewObjectDir(t *testing.T)      { fstests.TestFsNewObjectDir(t) }
-func TestFsCopy(t *testing.T)              { fstests.TestFsCopy(t) }
-func TestFsMove(t *testing.T)              { fstests.TestFsMove(t) }
-func TestFsDirMove(t *testing.T)           { fstests.TestFsDirMove(t) }
-func TestFsRmdirFull(t *testing.T)         { fstests.TestFsRmdirFull(t) }
-func TestFsPrecision(t *testing.T)         { fstests.TestFsPrecision(t) }
-func TestFsChangeNotify(t *testing.T)      { fstests.TestFsChangeNotify(t) }
-func TestObjectString(t *testing.T)        { fstests.TestObjectString(t) }
-func TestObjectFs(t *testing.T)            { fstests.TestObjectFs(t) }
-func TestObjectRemote(t *testing.T)        { fstests.TestObjectRemote(t) }
-func TestObjectHashes(t *testing.T)        { fstests.TestObjectHashes(t) }
-func TestObjectModTime(t *testing.T)       { fstests.TestObjectModTime(t) }
-func TestObjectMimeType(t *testing.T)      { fstests.TestObjectMimeType(t) }
-func TestObjectSetModTime(t *testing.T)    { fstests.TestObjectSetModTime(t) }
-func TestObjectSize(t *testing.T)          { fstests.TestObjectSize(t) }
-func TestObjectOpen(t *testing.T)          { fstests.TestObjectOpen(t) }
-func TestObjectOpenSeek(t *testing.T)      { fstests.TestObjectOpenSeek(t) }
-func TestObjectOpenRange(t *testing.T)     { fstests.TestObjectOpenRange(t) }
-func TestObjectPartialRead(t *testing.T)   { fstests.TestObjectPartialRead(t) }
-func TestObjectUpdate(t *testing.T)        { fstests.TestObjectUpdate(t) }
-func TestObjectStorable(t *testing.T)      { fstests.TestObjectStorable(t) }
-func TestFsIsFile(t *testing.T)            { fstests.TestFsIsFile(t) }
-func TestFsIsFileNotFound(t *testing.T)    { fstests.TestFsIsFileNotFound(t) }
-func TestObjectRemove(t *testing.T)        { fstests.TestObjectRemove(t) }
-func TestFsPutStream(t *testing.T)         { fstests.TestFsPutStream(t) }
-func TestObjectPurge(t *testing.T)         { fstests.TestObjectPurge(t) }
-func TestInternal(t *testing.T)            { fstests.TestInternal(t) }
-func TestFinalise(t *testing.T)            { fstests.TestFinalise(t) }

backend/webdav/api/types.go

@@ -5,6 +5,7 @@ import (
 	"encoding/xml"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -65,8 +66,8 @@ type Prop struct {
 	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
 }
 
-// Parse a status of the form "HTTP/1.1 200 OK",
-var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)\s+(.*)$`)
+// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
+var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)`)
 
 // StatusOK examines the Status and returns an OK flag
 func (p *Prop) StatusOK() bool {
@@ -75,7 +76,7 @@ func (p *Prop) StatusOK() bool {
 		return true
 	}
 	match := parseStatus.FindStringSubmatch(p.Status[0])
-	if len(match) < 3 {
+	if len(match) < 2 {
 		return false
 	}
 	code, err := strconv.Atoi(match[1])
@@ -109,16 +110,20 @@ type Error struct {
 
 // Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
+	var out []string
 	if e.Message != "" {
-		return e.Message
+		out = append(out, e.Message)
 	}
 	if e.Exception != "" {
-		return e.Exception
+		out = append(out, e.Exception)
 	}
 	if e.Status != "" {
-		return e.Status
+		out = append(out, e.Status)
 	}
-	return "Webdav Error"
+	if len(out) == 0 {
+		return "Webdav Error"
+	}
+	return strings.Join(out, ": ")
 }
 
 // Time represents represents date and time information for the

backend/webdav/odrvcookie/fetch.go

@@ -0,0 +1,186 @@
+// Package odrvcookie can fetch authentication cookies for a sharepoint webdav endpoint
+package odrvcookie
+
+import (
+	"bytes"
+	"encoding/xml"
+	"html/template"
+	"net/http"
+	"net/http/cookiejar"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/fshttp"
+	"github.com/pkg/errors"
+	"golang.org/x/net/publicsuffix"
+)
+
+// CookieAuth hold the authentication information
+// These are username and password as well as the authentication endpoint
+type CookieAuth struct {
+	user     string
+	pass     string
+	endpoint string
+}
+
+// CookieResponse contains the requested cookies
+type CookieResponse struct {
+	RtFa    http.Cookie
+	FedAuth http.Cookie
+}
+
+// SuccessResponse hold a response from the sharepoint webdav
+type SuccessResponse struct {
+	XMLName xml.Name            `xml:"Envelope"`
+	Succ    SuccessResponseBody `xml:"Body"`
+}
+
+// SuccessResponseBody is the body of a success response, it holds the token
+type SuccessResponseBody struct {
+	XMLName xml.Name
+	Type    string    `xml:"RequestSecurityTokenResponse>TokenType"`
+	Created time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Created"`
+	Expires time.Time `xml:"RequestSecurityTokenResponse>Lifetime>Expires"`
+	Token   string    `xml:"RequestSecurityTokenResponse>RequestedSecurityToken>BinarySecurityToken"`
+}
+
+// reqString is a template that gets populated with the user data in order to retrieve a "BinarySecurityToken"
+const reqString = `<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope"
+xmlns:a="http://www.w3.org/2005/08/addressing"
+xmlns:u="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-utility-1.0.xsd">
+<s:Header>
+<a:Action s:mustUnderstand="1">http://schemas.xmlsoap.org/ws/2005/02/trust/RST/Issue</a:Action>
+<a:ReplyTo>
+<a:Address>http://www.w3.org/2005/08/addressing/anonymous</a:Address>
+</a:ReplyTo>
+<a:To s:mustUnderstand="1">https://login.microsoftonline.com/extSTS.srf</a:To>
+<o:Security s:mustUnderstand="1"
+ xmlns:o="http://docs.oasis-open.org/wss/2004/01/oasis-200401-wss-wssecurity-secext-1.0.xsd">
+<o:UsernameToken>
+  <o:Username>{{ .Username }}</o:Username>
+  <o:Password>{{ .Password }}</o:Password>
+</o:UsernameToken>
+</o:Security>
+</s:Header>
+<s:Body>
+<t:RequestSecurityToken xmlns:t="http://schemas.xmlsoap.org/ws/2005/02/trust">
+<wsp:AppliesTo xmlns:wsp="http://schemas.xmlsoap.org/ws/2004/09/policy">
+  <a:EndpointReference>
+    <a:Address>{{ .Address }}</a:Address>
+  </a:EndpointReference>
+</wsp:AppliesTo>
+<t:KeyType>http://schemas.xmlsoap.org/ws/2005/05/identity/NoProofKey</t:KeyType>
+<t:RequestType>http://schemas.xmlsoap.org/ws/2005/02/trust/Issue</t:RequestType>
+<t:TokenType>urn:oasis:names:tc:SAML:1.0:assertion</t:TokenType>
+</t:RequestSecurityToken>
+</s:Body>
+</s:Envelope>`
+
+// New creates a new CookieAuth struct
+func New(pUser, pPass, pEndpoint string) CookieAuth {
+	retStruct := CookieAuth{
+		user:     pUser,
+		pass:     pPass,
+		endpoint: pEndpoint,
+	}
+
+	return retStruct
+}
+
+// Cookies creates a CookieResponse. It fetches the auth token and then
+// retrieves the Cookies
+func (ca *CookieAuth) Cookies() (*CookieResponse, error) {
+	tokenResp, err := ca.getSPToken()
+	if err != nil {
+		return nil, err
+	}
+	return ca.getSPCookie(tokenResp)
+}
+
+func (ca *CookieAuth) getSPCookie(conf *SuccessResponse) (*CookieResponse, error) {
+	spRoot, err := url.Parse(ca.endpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while contructing endpoint URL")
+	}
+
+	u, err := url.Parse("https://" + spRoot.Host + "/_forms/default.aspx?wa=wsignin1.0")
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while constructing login URL")
+	}
+
+	// To authenticate with davfs or anything else we need two cookies (rtFa and FedAuth)
+	// In order to get them we use the token we got earlier and a cookieJar
+	jar, err := cookiejar.New(&cookiejar.Options{PublicSuffixList: publicsuffix.List})
+	if err != nil {
+		return nil, err
+	}
+
+	client := &http.Client{
+		Jar: jar,
+	}
+
+	// Send the previously aquired Token as a Post parameter
+	if _, err = client.Post(u.String(), "text/xml", strings.NewReader(conf.Succ.Token)); err != nil {
+		return nil, errors.Wrap(err, "Error while grabbing cookies from endpoint: %v")
+	}
+
+	cookieResponse := CookieResponse{}
+	for _, cookie := range jar.Cookies(u) {
+		if (cookie.Name == "rtFa") || (cookie.Name == "FedAuth") {
+			switch cookie.Name {
+			case "rtFa":
+				cookieResponse.RtFa = *cookie
+			case "FedAuth":
+				cookieResponse.FedAuth = *cookie
+			}
+		}
+	}
+	return &cookieResponse, nil
+}
+
+func (ca *CookieAuth) getSPToken() (conf *SuccessResponse, err error) {
+	reqData := map[string]interface{}{
+		"Username": ca.user,
+		"Password": ca.pass,
+		"Address":  ca.endpoint,
+	}
+
+	t := template.Must(template.New("authXML").Parse(reqString))
+
+	buf := &bytes.Buffer{}
+	if err := t.Execute(buf, reqData); err != nil {
+		return nil, errors.Wrap(err, "Error while filling auth token template")
+	}
+
+	// Create and execute the first request which returns an auth token for the sharepoint service
+	// With this token we can authenticate on the login page and save the returned cookies
+	req, err := http.NewRequest("POST", "https://login.microsoftonline.com/extSTS.srf", buf)
+	if err != nil {
+		return nil, err
+	}
+
+	client := fshttp.NewClient(fs.Config)
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error while logging in to endpoint")
+	}
+	defer fs.CheckClose(resp.Body, &err)
+
+	respBuf := bytes.Buffer{}
+	_, err = respBuf.ReadFrom(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	s := respBuf.Bytes()
+
+	conf = &SuccessResponse{}
+	err = xml.Unmarshal(s, conf)
+	if err != nil {
+		// FIXME: Try to parse with FailedResponse struct (check for server error code)
+		return nil, errors.Wrap(err, "Error while reading endpoint response")
+	}
+
+	return
+}

backend/webdav/webdav.go

@@ -19,16 +19,17 @@ package webdav
 // For example the ownCloud WebDAV server does it that way.
 
 import (
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
 	"github.com/ncw/rclone/backend/webdav/api"
+	"github.com/ncw/rclone/backend/webdav/odrvcookie"
 	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/obscure"
@@ -70,6 +71,9 @@ func init() {
 			}, {
 				Value: "owncloud",
 				Help:  "Owncloud",
+			}, {
+				Value: "sharepoint",
+				Help:  "Sharepoint",
 			}, {
 				Value: "other",
 				Help:  "Other site/service or software",
@@ -83,6 +87,10 @@ func init() {
 			Help:       "Password.",
 			Optional:   true,
 			IsPassword: true,
+		}, {
+			Name:     "bearer_token",
+			Help:     "Bearer token instead of user/pass (eg a Macaroon)",
+			Optional: true,
 		}},
 	})
 }
@@ -113,7 +121,6 @@ type Object struct {
 	hasMetaData bool      // whether info below has been set
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
-	id          string    // ID of the object
 	sha1        string    // SHA-1 of the object content
 }
 
@@ -139,15 +146,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a webdav path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
-// parsePath parses an webdav 'url'
-func parsePath(path string) (root string) {
-	root = strings.Trim(path, "/")
-	return
-}
-
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	429, // Too Many Requests.
@@ -185,6 +183,10 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Prop, err error) {
 	opts := rest.Opts{
 		Method: "PROPFIND",
 		Path:   f.filePath(path),
+		ExtraHeaders: map[string]string{
+			"Depth": "1",
+		},
+		NoRedirect: true,
 	}
 	var result api.Multistatus
 	var resp *http.Response
@@ -194,7 +196,13 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Prop, err error) {
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// does not exist
-		if apiErr.StatusCode == http.StatusNotFound {
+		switch apiErr.StatusCode {
+		case http.StatusNotFound:
+			return nil, fs.ErrorObjectNotFound
+		case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther:
+			// Some sort of redirect - go doesn't deal with these properly (it resets
+			// the method to GET).  However we can assume that if it was redirected the
+			// object was not found.
 			return nil, fs.ErrorObjectNotFound
 		}
 	}
@@ -216,11 +224,16 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Prop, err error) {
 
 // errorHandler parses a non 2xx error response into an error
 func errorHandler(resp *http.Response) error {
+	body, err := rest.ReadBody(resp)
+	if err != nil {
+		return errors.Wrap(err, "error when trying to read error from body")
+	}
 	// Decode error response
 	errResponse := new(api.Error)
-	err := rest.DecodeXML(resp, &errResponse)
+	err = xml.Unmarshal(body, &errResponse)
 	if err != nil {
-		fs.Debugf(nil, "Couldn't decode error response: %v", err)
+		// set the Message to be the body if can't parse the XML
+		errResponse.Message = strings.TrimSpace(string(body))
 	}
 	errResponse.Status = resp.Status
 	errResponse.StatusCode = resp.StatusCode
@@ -256,9 +269,12 @@ func NewFs(name, root string) (fs.Fs, error) {
 	if !strings.HasSuffix(endpoint, "/") {
 		endpoint += "/"
 	}
+	rootIsDir := strings.HasSuffix(root, "/")
+	root = strings.Trim(root, "/")
 
 	user := config.FileGet(name, "user")
 	pass := config.FileGet(name, "pass")
+	bearerToken := config.FileGet(name, "bearer_token")
 	if pass != "" {
 		var err error
 		pass, err = obscure.Reveal(pass)
@@ -279,7 +295,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 		root:        root,
 		endpoint:    u,
 		endpointURL: u.String(),
-		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()).SetUserPass(user, pass),
+		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
 		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		user:        user,
 		pass:        pass,
@@ -288,10 +304,18 @@ func NewFs(name, root string) (fs.Fs, error) {
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(f)
+	if user != "" || pass != "" {
+		f.srv.SetUserPass(user, pass)
+	} else if bearerToken != "" {
+		f.srv.SetHeader("Authorization", "BEARER "+bearerToken)
+	}
 	f.srv.SetErrorHandler(errorHandler)
-	f.setQuirks(vendor)
+	err = f.setQuirks(vendor)
+	if err != nil {
+		return nil, err
+	}
 
-	if root != "" {
+	if root != "" && !rootIsDir {
 		// Check to see if the root actually an existing file
 		remote := path.Base(root)
 		f.root = path.Dir(root)
@@ -314,7 +338,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 }
 
 // setQuirks adjusts the Fs for the vendor passed in
-func (f *Fs) setQuirks(vendor string) {
+func (f *Fs) setQuirks(vendor string) error {
 	if vendor == "" {
 		vendor = "other"
 	}
@@ -327,6 +351,16 @@ func (f *Fs) setQuirks(vendor string) {
 	case "nextcloud":
 		f.precision = time.Second
 		f.useOCMtime = true
+	case "sharepoint":
+		// To mount sharepoint, two Cookies are required
+		// They have to be set instead of BasicAuth
+		f.srv.RemoveHeader("Authorization") // We don't need this Header if using cookies
+		spCk := odrvcookie.New(f.user, f.pass, f.endpointURL)
+		spCookies, err := spCk.Cookies()
+		if err != nil {
+			return err
+		}
+		f.srv.SetCookie(&spCookies.FedAuth, &spCookies.RtFa)
 	case "other":
 	default:
 		fs.Debugf(f, "Unknown vendor %q", vendor)
@@ -336,6 +370,7 @@ func (f *Fs) setQuirks(vendor string) {
 	if !f.canStream {
 		f.features.PutStream = nil
 	}
+	return nil
 }
 
 // Return an Object from a path
@@ -525,6 +560,11 @@ func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption
 // mkParentDir makes the parent of the native path dirPath if
 // necessary and any directories above that
 func (f *Fs) mkParentDir(dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
+	// chop off trailing / if it exists
+	if strings.HasSuffix(dirPath, "/") {
+		dirPath = dirPath[:len(dirPath)-1]
+	}
 	parent := path.Dir(dirPath)
 	if parent == "." {
 		parent = ""
@@ -534,10 +574,15 @@ func (f *Fs) mkParentDir(dirPath string) error {
 
 // mkdir makes the directory and parents using native paths
 func (f *Fs) mkdir(dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
 	// We assume the root is already ceated
 	if dirPath == "" {
 		return nil
 	}
+	// Collections must end with /
+	if !strings.HasSuffix(dirPath, "/") {
+		dirPath += "/"
+	}
 	opts := rest.Opts{
 		Method:     "MKCOL",
 		Path:       dirPath,
@@ -549,7 +594,7 @@ func (f *Fs) mkdir(dirPath string) error {
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// already exists
-		if apiErr.StatusCode == http.StatusMethodNotAllowed {
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
 			return nil
 		}
 		// parent does not exists