mount: use the equivalent of kernel_cache by default #FIXME WIP

Before this copyto would parse windows paths incorrectly.

This change moves the parsing code into fspath and makes sure
fspath.Split calls fspath.Parse which does the parsing correctly for

This also renames fspath.RemoteParse to fspath.Parse for consistency

.gometalinter.json

@@ -0,0 +1,14 @@
+{
+    "Enable": [
+        "deadcode",
+        "errcheck",
+        "goimports",
+        "golint",
+        "ineffassign",
+        "structcheck",
+        "varcheck",
+        "vet"
+    ],
+    "EnableGC": true,
+    "Vendor": true
+}

.travis.yml

@@ -47,4 +47,4 @@ deploy:
   on:
     all_branches: true
     go: "1.10.1"
-    condition: $TRAVIS_OS_NAME == linux && $TRAVIS_PULL_REQUEST == false
+    condition: $TRAVIS_PULL_REQUEST == false

CONTRIBUTING.md

@@ -235,15 +235,15 @@ in the `vendor` directory for perfectly reproducable builds.
 
 The `vendor` directory is entirely managed by the `dep` tool.
 
-To add a new dependency
+To add a new dependency, run `dep ensure` and `dep` will pull in the
+new dependency to the `vendor` directory and update the `Gopkg.lock`
+file.
 
-    dep ensure -add github.com/pkg/errors
+You can add constraints on that package in the `Gopkg.toml` file (see
+the `dep` documentation), but don't unless you really need to.
 
-You can add constraints on that package (see the `dep` documentation),
-but don't unless you really need to.
-
-Please check in the changes generated by dep including the `vendor`
-directory and `Godep.toml` and `Godep.locl` in a single commit
+Please check in the changes generated by `dep` including the `vendor`
+directory and `Godep.toml` and `Godep.lock` in a single commit
 separate from any other code changes.  Watch out for new files in
 `vendor`.
 

Gopkg.lock

@@ -9,13 +9,13 @@
     "fs",
     "fuseutil"
   ]
-  revision = "371fbbdaa8987b715bdd21d6adc4c9b20155f748"
+  revision = "65cc252bf6691cb3c7014bcb2c8dc29de91e3a7e"
 
 [[projects]]
   name = "cloud.google.com/go"
   packages = ["compute/metadata"]
-  revision = "20d4028b8a750c2aca76bf9fefa8ed2d0109b573"
-  version = "v0.19.0"
+  revision = "0fd7230b2a7505833d5f69b75cbd6c9582401479"
+  version = "v0.23.0"
 
 [[projects]]
   name = "github.com/Azure/azure-sdk-for-go"
@@ -23,8 +23,8 @@
     "storage",
     "version"
   ]
-  revision = "e67cd39e942c417ae5e9ae1165f778d9fe8996e0"
-  version = "v14.5.0"
+  revision = "fbe7db0e3f9793ba3e5704efbab84f51436c136e"
+  version = "v18.0.0"
 
 [[projects]]
   name = "github.com/Azure/go-autorest"
@@ -34,8 +34,8 @@
     "autorest/azure",
     "autorest/date"
   ]
-  revision = "0ae36a9e544696de46fdadb7b0d5fb38af48c063"
-  version = "v10.2.0"
+  revision = "1f7cd6cfe0adea687ad44a512dfe76140f804318"
+  version = "v10.12.0"
 
 [[projects]]
   branch = "master"
@@ -44,16 +44,16 @@
   revision = "ef1e4c783f8f0478bd8bff0edb3dd0bade552599"
 
 [[projects]]
-  branch = "master"
   name = "github.com/VividCortex/ewma"
   packages = ["."]
-  revision = "43880d236f695d39c62cf7aa4ebd4508c258e6c0"
+  revision = "b24eb346a94c3ba12c1da1e564dbac1b498a77ce"
+  version = "v1.1.1"
 
 [[projects]]
   branch = "master"
   name = "github.com/a8m/tree"
   packages = ["."]
-  revision = "cf42b1e486f0b025942a768a9ad59c9939d6ca40"
+  revision = "3cf936ce15d6100c49d9c75f79c220ae7e579599"
 
 [[projects]]
   name = "github.com/abbot/go-http-auth"
@@ -62,7 +62,6 @@
   version = "v0.4.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/aws/aws-sdk-go"
   packages = [
     "aws",
@@ -75,6 +74,7 @@
     "aws/credentials/ec2rolecreds",
     "aws/credentials/endpointcreds",
     "aws/credentials/stscreds",
+    "aws/csm",
     "aws/defaults",
     "aws/ec2metadata",
     "aws/endpoints",
@@ -85,6 +85,8 @@
     "internal/sdkrand",
     "internal/shareddefaults",
     "private/protocol",
+    "private/protocol/eventstream",
+    "private/protocol/eventstream/eventstreamapi",
     "private/protocol/query",
     "private/protocol/query/queryutil",
     "private/protocol/rest",
@@ -95,13 +97,14 @@
     "service/s3/s3manager",
     "service/sts"
   ]
-  revision = "12fe7d35f8ad5f7f2715d414624b0723737de1f7"
+  revision = "bfc1a07cf158c30c41a3eefba8aae043d0bb5bff"
+  version = "v1.14.8"
 
 [[projects]]
   name = "github.com/billziss-gh/cgofuse"
   packages = ["fuse"]
-  revision = "487e2baa5611bab252a906d7f9b869f944607305"
-  version = "v1.0.4"
+  revision = "ea66f9809c71af94522d494d3d617545662ea59d"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
@@ -134,7 +137,6 @@
   version = "v1.0.1"
 
 [[projects]]
-  branch = "master"
   name = "github.com/dropbox/dropbox-sdk-go-unofficial"
   packages = [
     "dropbox",
@@ -142,25 +144,27 @@
     "dropbox/common",
     "dropbox/file_properties",
     "dropbox/files",
+    "dropbox/seen_state",
     "dropbox/sharing",
     "dropbox/team_common",
     "dropbox/team_policies",
     "dropbox/users",
     "dropbox/users_common"
   ]
-  revision = "f0b3f3ded6d415a94e83e9a514fb8025e4e6be31"
+  revision = "7afa861bfde5a348d765522b303b6fbd9d250155"
+  version = "v4.1.0"
 
 [[projects]]
   name = "github.com/go-ini/ini"
   packages = ["."]
-  revision = "6333e38ac20b8949a8dd68baa3650f4dee8f39f0"
-  version = "v1.33.0"
+  revision = "06f5f3d67269ccec1fe5fe4134ba6e982984f7f5"
+  version = "v1.37.0"
 
 [[projects]]
   name = "github.com/golang/protobuf"
   packages = ["proto"]
-  revision = "925541529c1fa6821df4e44ce2723319eb2be768"
-  version = "v1.0.0"
+  revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
+  version = "v1.1.0"
 
 [[projects]]
   branch = "master"
@@ -178,7 +182,7 @@
   branch = "master"
   name = "github.com/jlaffaye/ftp"
   packages = ["."]
-  revision = "427467931c6fbc25acba4537c9d3cbc40cfa569b"
+  revision = "2403248fa8cc9f7909862627aa7337f13f8e0bf1"
 
 [[projects]]
   name = "github.com/jmespath/go-jmespath"
@@ -192,10 +196,10 @@
   revision = "ae77be60afb1dcacde03767a8c37337fad28ac14"
 
 [[projects]]
-  branch = "master"
   name = "github.com/kr/fs"
   packages = ["."]
-  revision = "2788f0dbd16903de03cb8186e5c7d97b69ad387b"
+  revision = "1455def202f6e05b95cc7bfc7e8ae67ae5141eba"
+  version = "v0.1.0"
 
 [[projects]]
   name = "github.com/marstr/guid"
@@ -216,16 +220,16 @@
   revision = "887eb06ab6a255fbf5744b5812788e884078620a"
 
 [[projects]]
-  branch = "master"
   name = "github.com/ncw/swift"
   packages = ["."]
-  revision = "ae9f0ea1605b9aa6434ed5c731ca35d83ba67c55"
+  revision = "b2a7479cf26fa841ff90dd932d0221cb5c50782d"
+  version = "v1.0.39"
 
 [[projects]]
   branch = "master"
   name = "github.com/nsf/termbox-go"
   packages = ["."]
-  revision = "e2050e41c8847748ec5288741c0b19a8cb26d084"
+  revision = "5c94acc5e6eb520f1bcd183974e01171cc4c23b3"
 
 [[projects]]
   branch = "master"
@@ -248,20 +252,20 @@
     "log",
     "reopen"
   ]
-  revision = "b98065a377794d577e2a0e32869378b9ce4b8952"
-  version = "v0.1.1"
+  revision = "807ee759d82c84982a89fb3dc875ef884942f1e5"
+  version = "v0.2.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/pkg/errors"
   packages = ["."]
-  revision = "816c9085562cd7ee03e7f8188a1cfd942858cded"
+  revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
+  version = "v0.8.0"
 
 [[projects]]
   name = "github.com/pkg/sftp"
   packages = ["."]
-  revision = "43ec6c679d353f6e077d3965dc74f6d996eb4a09"
-  version = "1.5.1"
+  revision = "57673e38ea946592a59c26592b7e6fbda646975b"
+  version = "1.8.0"
 
 [[projects]]
   name = "github.com/pmezard/go-difflib"
@@ -270,10 +274,10 @@
   version = "v1.0.0"
 
 [[projects]]
-  branch = "master"
   name = "github.com/rfjakob/eme"
   packages = ["."]
-  revision = "2222dbd4ba467ab3fc7e8af41562fcfe69c0d770"
+  revision = "01668ae55fe0b79a483095689043cce3e80260db"
+  version = "v1.1"
 
 [[projects]]
   name = "github.com/russross/blackfriday"
@@ -291,7 +295,7 @@
   branch = "master"
   name = "github.com/sevlyar/go-daemon"
   packages = ["."]
-  revision = "32749a731f76154d29bc6a547e6585f320eb235e"
+  revision = "f9261e73885de99b1647d68bedadf2b9a99ad11f"
 
 [[projects]]
   branch = "master"
@@ -300,34 +304,34 @@
   revision = "75fb7ed4208cf72d323d7d02fd1a5964a7a9073c"
 
 [[projects]]
-  branch = "master"
   name = "github.com/spf13/cobra"
   packages = [
     ".",
     "doc"
   ]
-  revision = "c439c4fa093711d42e1b01acb1235b52004753c1"
+  revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385"
+  version = "v0.0.3"
 
 [[projects]]
-  branch = "master"
   name = "github.com/spf13/pflag"
   packages = ["."]
-  revision = "ee5fd03fd6acfd43e44aea0b4135958546ed8e73"
+  revision = "583c0c0531f06d5278b7d917446061adc344b5cd"
+  version = "v1.0.1"
 
 [[projects]]
-  branch = "master"
   name = "github.com/stretchr/testify"
   packages = [
     "assert",
     "require"
   ]
-  revision = "380174f817a09abe5982a82f94ad50938a8df65d"
+  revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
+  version = "v1.2.2"
 
 [[projects]]
   branch = "master"
   name = "github.com/t3rm1n4l/go-mega"
   packages = ["."]
-  revision = "4e68b16e97ffc3b77abacbf727817a4d48fb0b66"
+  revision = "57978a63bd3f91fa7e188b751a7e7e6dd4e33813"
 
 [[projects]]
   branch = "master"
@@ -336,7 +340,6 @@
   revision = "ba9c9e33906f58169366275e3450db66139a31a9"
 
 [[projects]]
-  branch = "master"
   name = "github.com/yunify/qingstor-sdk-go"
   packages = [
     ".",
@@ -351,7 +354,8 @@
     "service",
     "utils"
   ]
-  revision = "a3cbaaf92247eaf55751a7ff37c126c511757492"
+  revision = "4f9ac88c5fec7350e960aabd0de1f1ede0ad2895"
+  version = "v2.2.14"
 
 [[projects]]
   branch = "master"
@@ -363,6 +367,7 @@
     "ed25519",
     "ed25519/internal/edwards25519",
     "internal/chacha20",
+    "internal/subtle",
     "nacl/secretbox",
     "pbkdf2",
     "poly1305",
@@ -372,7 +377,7 @@
     "ssh/agent",
     "ssh/terminal"
   ]
-  revision = "c3a3ad6d03f7a915c0f7e194b7152974bb73d287"
+  revision = "027cca12c2d63e3d62b670d901e8a2c95854feec"
 
 [[projects]]
   branch = "master"
@@ -382,15 +387,16 @@
     "context/ctxhttp",
     "html",
     "html/atom",
+    "http/httpguts",
     "http2",
     "http2/hpack",
     "idna",
-    "lex/httplex",
+    "publicsuffix",
     "webdav",
     "webdav/internal/xml",
     "websocket"
   ]
-  revision = "92b859f39abd2d91a854c9f9c4621b2f5054a92d"
+  revision = "db08ff08e8622530d9ed3a0e8ac279f6d4c02196"
 
 [[projects]]
   branch = "master"
@@ -402,7 +408,7 @@
     "jws",
     "jwt"
   ]
-  revision = "fdc9e635145ae97e6c2cb777c48305600cf515cb"
+  revision = "1e0a3fa8ba9a5c9eb35c271780101fdaf1b205d7"
 
 [[projects]]
   branch = "master"
@@ -411,18 +417,15 @@
     "unix",
     "windows"
   ]
-  revision = "d8e400bc7db4870d786864138af681469693d18c"
+  revision = "6c888cc515d3ed83fc103cf1d84468aad274b0a7"
 
 [[projects]]
-  branch = "master"
   name = "golang.org/x/text"
   packages = [
     "collate",
     "collate/build",
     "internal/colltab",
     "internal/gen",
-    "internal/language",
-    "internal/language/compact",
     "internal/tag",
     "internal/triegen",
     "internal/ucd",
@@ -434,13 +437,14 @@
     "unicode/norm",
     "unicode/rangetable"
   ]
-  revision = "8c34f848e18c4bd34d02db7f19a0ed1a0a8f5852"
+  revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
+  version = "v0.3.0"
 
 [[projects]]
   branch = "master"
   name = "golang.org/x/time"
   packages = ["rate"]
-  revision = "26559e0f760e39c24d730d3224364aef164ee23f"
+  revision = "fbb02b2291d28baffd63558aa44b4b56f178d650"
 
 [[projects]]
   branch = "master"
@@ -452,7 +456,7 @@
     "googleapi/internal/uritemplates",
     "storage/v1"
   ]
-  revision = "55e9fb4044f4757138d4273ace23060d022d18f9"
+  revision = "2eea9ba0a3d94f6ab46508083e299a00bbbc65f6"
 
 [[projects]]
   name = "google.golang.org/appengine"
@@ -469,18 +473,18 @@
     "log",
     "urlfetch"
   ]
-  revision = "150dc57a1b433e64154302bdc40b6bb8aefa313a"
-  version = "v1.0.0"
+  revision = "b1f26356af11148e710935ed1ac8a7f5702c7612"
+  version = "v1.1.0"
 
 [[projects]]
   name = "gopkg.in/yaml.v2"
   packages = ["."]
-  revision = "7f97868eec74b32b0982dd158a51a446d1da7eb5"
-  version = "v2.1.1"
+  revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183"
+  version = "v2.2.1"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "09939c0d5f32998497c8304c84dd5c397c88816d235441d87f5306ae5db43b8a"
+  inputs-digest = "c1378c5fc821e27711155958ff64b3c74b56818ba4733dbfe0c86d518c32880e"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -1,154 +1,11 @@
-
-## Gopkg.toml example (these lines may be deleted)
-
-## "required" lists a set of packages (not projects) that must be included in
-## Gopkg.lock. This list is merged with the set of packages imported by the current
-## project. Use it when your project needs a package it doesn't explicitly import -
-## including "main" packages.
-# required = ["github.com/user/thing/cmd/thing"]
-
-## "ignored" lists a set of packages (not projects) that are ignored when
-## dep statically analyzes source code. Ignored packages can be in this project,
-## or in a dependency.
-# ignored = ["github.com/user/project/badpkg"]
-
-## Dependencies define constraints on dependent projects. They are respected by
-## dep whether coming from the Gopkg.toml of the current project or a dependency.
-# [[constraint]]
-## Required: the root import path of the project being constrained.
-# name = "github.com/user/project"
-#
-## Recommended: the version constraint to enforce for the project.
-## Only one of "branch", "version" or "revision" can be specified.
-# version = "1.0.0"
-# branch = "master"
-# revision = "abc123"
-#
-## Optional: an alternate location (URL or import path) for the project's source.
-# source = "https://github.com/myfork/package.git"
-
-## Overrides have the same structure as [[constraint]], but supercede all
-## [[constraint]] declarations from all projects. Only the current project's
-## [[override]] are applied.
-##
-## Overrides are a sledgehammer. Use them only as a last resort.
-# [[override]]
-## Required: the root import path of the project being constrained.
-# name = "github.com/user/project"
-#
-## Optional: specifying a version constraint override will cause all other
-## constraints on this project to be ignored; only the overriden constraint
-## need be satisfied.
-## Again, only one of "branch", "version" or "revision" can be specified.
-# version = "1.0.0"
-# branch = "master"
-# revision = "abc123"
-#
-## Optional: specifying an alternate source location as an override will
-## enforce that the alternate location is used for that project, regardless of
-## what source location any dependent projects specify.
-# source = "https://github.com/myfork/package.git"
-
-[[constraint]]
-  branch = "master"
-  name = "bazil.org/fuse"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/Unknwon/goconfig"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/VividCortex/ewma"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/aws/aws-sdk-go"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ncw/go-acd"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/ncw/swift"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/pkg/errors"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/pkg/sftp"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/rfjakob/eme"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/skratchdot/open-golang"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/cobra"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/spf13/pflag"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/stretchr/testify"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/crypto"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/net"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/oauth2"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/sys"
-
-[[constraint]]
-  branch = "master"
-  name = "golang.org/x/text"
-
-[[constraint]]
-  branch = "master"
-  name = "google.golang.org/api"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/dropbox/dropbox-sdk-go-unofficial"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/yunify/qingstor-sdk-go"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/coreos/bbolt"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/patrickmn/go-cache"
-
-[[constraint]]
-  branch = "master"
-  name = "github.com/okzk/sdnotify"
-
-[[constraint]]
+# pin this to master to pull in the macOS changes
+# can likely remove for 1.43
+[[override]]
   branch = "master"
   name = "github.com/sevlyar/go-daemon"
 
-[[constraint]]
+# pin this to master to pull in the fix for linux/mips
+# can likely remove for 1.43
+[[override]]
   branch = "master"
-  name = "github.com/t3rm1n4l/go-mega"
+  name = "github.com/coreos/bbolt"

MANUAL.html

@@ -12,7 +12,7 @@
 <div id="header">
 <h1 class="title">rclone(1) User Manual</h1>
 <h2 class="author">Nick Craig-Wood</h2>
-<h3 class="date">Apr 28, 2018</h3>
+<h3 class="date">Jun 16, 2018</h3>
 </div>
 <h1 id="rclone">Rclone</h1>
 <p><a href="https://rclone.org/"><img src="https://rclone.org/img/rclone-120x120.png" alt="Logo" /></a></p>
@@ -37,8 +37,9 @@
 <li>Microsoft Azure Blob Storage</li>
 <li>Microsoft OneDrive</li>
 <li>Minio</li>
-<li>Nextloud</li>
+<li>Nextcloud</li>
 <li>OVH</li>
+<li>OpenDrive</li>
 <li>Openstack Swift</li>
 <li>Oracle Cloud Storage</li>
 <li>ownCloud</li>
@@ -156,6 +157,7 @@ sudo mv rclone /usr/local/bin/</code></pre>
 <li><a href="https://rclone.org/azureblob/">Microsoft Azure Blob Storage</a></li>
 <li><a href="https://rclone.org/onedrive/">Microsoft OneDrive</a></li>
 <li><a href="https://rclone.org/swift/">Openstack Swift / Rackspace Cloudfiles / Memset Memstore</a></li>
+<li><a href="https://rclone.org/opendrive/">OpenDrive</a></li>
 <li><a href="https://rclone.org/pcloud/">Pcloud</a></li>
 <li><a href="https://rclone.org/qingstor/">QingStor</a></li>
 <li><a href="https://rclone.org/sftp/">SFTP</a></li>
@@ -266,10 +268,12 @@ rclone --dry-run --min-size 100M delete remote:path</code></pre>
 <p>Checks the files in the source and destination match. It compares sizes and hashes (MD5 or SHA1) and logs a report of files which don't match. It doesn't alter the source or destination.</p>
 <p>If you supply the --size-only flag, it will only compare the sizes not the hashes as well. Use this for a quick check.</p>
 <p>If you supply the --download flag, it will download the data from both remotes and check them against each other on the fly. This can be useful for remotes that don't support hashes or if you really want to check all the data.</p>
+<p>If you supply the --one-way flag, it will only check that files in source match the files in destination, not the other way around. Meaning extra files in destination that are not in the source will not trigger an error.</p>
 <pre><code>rclone check source:path dest:path [flags]</code></pre>
 <h3 id="options-8">Options</h3>
 <pre><code>      --download   Check by downloading rather than with hash.
-  -h, --help       help for check</code></pre>
+  -h, --help       help for check
+      --one-way    Check one way only, source files must exist on remote</code></pre>
 <h2 id="rclone-ls">rclone ls</h2>
 <p>List the objects in the path with size and path.</p>
 <h3 id="synopsis-9">Synopsis</h3>
@@ -625,9 +629,11 @@ if src is directory
 <p>You can use it like this also, but that will involve downloading all the files in remote:path.</p>
 <pre><code>rclone cryptcheck remote:path encryptedremote:path</code></pre>
 <p>After it has run it will log the status of the encryptedremote:.</p>
+<p>If you supply the --one-way flag, it will only check that files in source match the files in destination, not the other way around. Meaning extra files in destination that are not in the source will not trigger an error.</p>
 <pre><code>rclone cryptcheck remote:path cryptedremote:path [flags]</code></pre>
 <h3 id="options-32">Options</h3>
-<pre><code>  -h, --help   help for cryptcheck</code></pre>
+<pre><code>  -h, --help      help for cryptcheck
+      --one-way   Check one way only, source files must exist on destination</code></pre>
 <h2 id="rclone-cryptdecode">rclone cryptdecode</h2>
 <p>Cryptdecode returns unencrypted file names.</p>
 <h3 id="synopsis-33">Synopsis</h3>
@@ -648,15 +654,22 @@ rclone cryptdecode --reverse encryptedremote: filename1 filename2</code></pre>
 <pre><code>rclone dbhashsum remote:path [flags]</code></pre>
 <h3 id="options-34">Options</h3>
 <pre><code>  -h, --help   help for dbhashsum</code></pre>
+<h2 id="rclone-deletefile">rclone deletefile</h2>
+<p>Remove a single file path from remote.</p>
+<h3 id="synopsis-35">Synopsis</h3>
+<p>Remove a single file path from remote. Unlike <code>delete</code> it cannot be used to remove a directory and it doesn't obey include/exclude filters - if the specified file exists, it will always be removed.</p>
+<pre><code>rclone deletefile remote:path [flags]</code></pre>
+<h3 id="options-35">Options</h3>
+<pre><code>  -h, --help   help for deletefile</code></pre>
 <h2 id="rclone-genautocomplete">rclone genautocomplete</h2>
 <p>Output completion script for a given shell.</p>
-<h3 id="synopsis-35">Synopsis</h3>
+<h3 id="synopsis-36">Synopsis</h3>
 <p>Generates a shell completion script for rclone. Run with --help to list the supported shells.</p>
-<h3 id="options-35">Options</h3>
+<h3 id="options-36">Options</h3>
 <pre><code>  -h, --help   help for genautocomplete</code></pre>
 <h2 id="rclone-genautocomplete-bash">rclone genautocomplete bash</h2>
 <p>Output bash completion script for rclone.</p>
-<h3 id="synopsis-36">Synopsis</h3>
+<h3 id="synopsis-37">Synopsis</h3>
 <p>Generates a bash shell autocompletion script for rclone.</p>
 <p>This writes to /etc/bash_completion.d/rclone by default so will probably need to be run with sudo or as root, eg</p>
 <pre><code>sudo rclone genautocomplete bash</code></pre>
@@ -664,11 +677,11 @@ rclone cryptdecode --reverse encryptedremote: filename1 filename2</code></pre>
 <pre><code>. /etc/bash_completion</code></pre>
 <p>If you supply a command line argument the script will be written there.</p>
 <pre><code>rclone genautocomplete bash [output_file] [flags]</code></pre>
-<h3 id="options-36">Options</h3>
+<h3 id="options-37">Options</h3>
 <pre><code>  -h, --help   help for bash</code></pre>
 <h2 id="rclone-genautocomplete-zsh">rclone genautocomplete zsh</h2>
 <p>Output zsh completion script for rclone.</p>
-<h3 id="synopsis-37">Synopsis</h3>
+<h3 id="synopsis-38">Synopsis</h3>
 <p>Generates a zsh autocompletion script for rclone.</p>
 <p>This writes to /usr/share/zsh/vendor-completions/_rclone by default so will probably need to be run with sudo or as root, eg</p>
 <pre><code>sudo rclone genautocomplete zsh</code></pre>
@@ -676,18 +689,18 @@ rclone cryptdecode --reverse encryptedremote: filename1 filename2</code></pre>
 <pre><code>autoload -U compinit &amp;&amp; compinit</code></pre>
 <p>If you supply a command line argument the script will be written there.</p>
 <pre><code>rclone genautocomplete zsh [output_file] [flags]</code></pre>
-<h3 id="options-37">Options</h3>
+<h3 id="options-38">Options</h3>
 <pre><code>  -h, --help   help for zsh</code></pre>
 <h2 id="rclone-gendocs">rclone gendocs</h2>
 <p>Output markdown docs for rclone to the directory supplied.</p>
-<h3 id="synopsis-38">Synopsis</h3>
+<h3 id="synopsis-39">Synopsis</h3>
 <p>This produces markdown docs for the rclone commands to the directory supplied. These are in a format suitable for hugo to render into the rclone.org website.</p>
 <pre><code>rclone gendocs output_directory [flags]</code></pre>
-<h3 id="options-38">Options</h3>
+<h3 id="options-39">Options</h3>
 <pre><code>  -h, --help   help for gendocs</code></pre>
 <h2 id="rclone-hashsum">rclone hashsum</h2>
 <p>Produces an hashsum file for all the objects in the path.</p>
-<h3 id="synopsis-39">Synopsis</h3>
+<h3 id="synopsis-40">Synopsis</h3>
 <p>Produces a hash file for all the objects in the path using the hash named. The output is in the same format as the standard md5sum/sha1sum tool.</p>
 <p>Run without a hash to see the list of supported hashes, eg</p>
 <pre><code>$ rclone hashsum
@@ -699,30 +712,30 @@ Supported hashes are:
 <p>Then</p>
 <pre><code>$ rclone hashsum MD5 remote:path</code></pre>
 <pre><code>rclone hashsum &lt;hash&gt; remote:path [flags]</code></pre>
-<h3 id="options-39">Options</h3>
+<h3 id="options-40">Options</h3>
 <pre><code>  -h, --help   help for hashsum</code></pre>
 <h2 id="rclone-link">rclone link</h2>
 <p>Generate public link to file/folder.</p>
-<h3 id="synopsis-40">Synopsis</h3>
+<h3 id="synopsis-41">Synopsis</h3>
 <p>rclone link will create or retrieve a public link to the given file or folder.</p>
 <pre><code>rclone link remote:path/to/file
 rclone link remote:path/to/folder/</code></pre>
 <p>If successful, the last line of the output will contain the link. Exact capabilities depend on the remote, but the link will always be created with the least constraints – e.g. no expiry, no password protection, accessible without account.</p>
 <pre><code>rclone link remote:path [flags]</code></pre>
-<h3 id="options-40">Options</h3>
+<h3 id="options-41">Options</h3>
 <pre><code>  -h, --help   help for link</code></pre>
 <h2 id="rclone-listremotes">rclone listremotes</h2>
 <p>List all the remotes in the config file.</p>
-<h3 id="synopsis-41">Synopsis</h3>
+<h3 id="synopsis-42">Synopsis</h3>
 <p>rclone listremotes lists all the available remotes from the config file.</p>
 <p>When uses with the -l flag it lists the types too.</p>
 <pre><code>rclone listremotes [flags]</code></pre>
-<h3 id="options-41">Options</h3>
+<h3 id="options-42">Options</h3>
 <pre><code>  -h, --help   help for listremotes
   -l, --long   Show the type as well as names.</code></pre>
 <h2 id="rclone-lsf">rclone lsf</h2>
 <p>List directories and objects in remote:path formatted for parsing</p>
-<h3 id="synopsis-42">Synopsis</h3>
+<h3 id="synopsis-43">Synopsis</h3>
 <p>List the contents of the source path (directories and objects) to standard output in a form which is easy to parse by scripts. By default this will just be the names of the objects and directories, one per line. The directories will have a / suffix.</p>
 <p>Eg</p>
 <pre><code>$ rclone lsf swift:bucket
@@ -735,7 +748,9 @@ fubuwic</code></pre>
 <pre><code>p - path
 s - size
 t - modification time
-h - hash</code></pre>
+h - hash
+i - ID of object if known
+m - MimeType of object if known</code></pre>
 <p>So if you wanted the path, size and modification time, you would use --format &quot;pst&quot;, or maybe --format &quot;tsp&quot; to put the path last.</p>
 <p>Eg</p>
 <pre><code>$ rclone lsf  --format &quot;tsp&quot; swift:bucket
@@ -763,6 +778,16 @@ cd65ac234e6fea5925974a51cdd865cc  canole
 2016-06-25 18:55:43,94467,03b5341b4f234b9d984d03ad076bae91,diwogej7
 2018-04-26 08:52:53,0,,ferejej3gux/
 2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic</code></pre>
+<p>You can output in CSV standard format. This will escape things in &quot; if they contain ,</p>
+<p>Eg</p>
+<pre><code>$ rclone lsf --csv --files-only --format ps remote:path
+test.log,22355
+test.sh,449
+&quot;this file contains a comma, in the file name.txt&quot;,6</code></pre>
+<p>Note that the --absolute parameter is useful for making lists of files to pass to an rclone copy with the --files-from flag.</p>
+<p>For example to find all the files modified within one day and copy those only (without traversing the whole directory structure):</p>
+<pre><code>rclone lsf --absolute --files-only --max-age 1d /path/to/local &gt; new_files
+rclone copy --files-from new_files /path/to/local remote:path</code></pre>
 <p>Any of the filtering options can be applied to this commmand.</p>
 <p>There are several related list commands</p>
 <ul>
@@ -777,8 +802,10 @@ cd65ac234e6fea5925974a51cdd865cc  canole
 <p>The other list commands <code>lsd</code>,<code>lsf</code>,<code>lsjson</code> do not recurse by default - use &quot;-R&quot; to make them recurse.</p>
 <p>Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).</p>
 <pre><code>rclone lsf remote:path [flags]</code></pre>
-<h3 id="options-42">Options</h3>
-<pre><code>  -d, --dir-slash          Append a slash to directory names. (default true)
+<h3 id="options-43">Options</h3>
+<pre><code>      --absolute           Put a leading / in front of path names.
+      --csv                Output in CSV format.
+  -d, --dir-slash          Append a slash to directory names. (default true)
       --dirs-only          Only list directories.
       --files-only         Only list files.
   -F, --format string      Output format - see  help for details (default &quot;p&quot;)
@@ -788,10 +815,10 @@ cd65ac234e6fea5925974a51cdd865cc  canole
   -s, --separator string   Separator for the items in the format. (default &quot;;&quot;)</code></pre>
 <h2 id="rclone-lsjson">rclone lsjson</h2>
 <p>List directories and objects in the path in JSON format.</p>
-<h3 id="synopsis-43">Synopsis</h3>
+<h3 id="synopsis-44">Synopsis</h3>
 <p>List directories and objects in the path in JSON format.</p>
 <p>The output is an array of Items, where each Item looks like this</p>
-<p>{ &quot;Hashes&quot; : { &quot;SHA-1&quot; : &quot;f572d396fae9206628714fb2ce00f72e94f2258f&quot;, &quot;MD5&quot; : &quot;b1946ac92492d2347c6235b4d2611184&quot;, &quot;DropboxHash&quot; : &quot;ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc&quot; }, &quot;IsDir&quot; : false, &quot;ModTime&quot; : &quot;2017-05-31T16:15:57.034468261+01:00&quot;, &quot;Name&quot; : &quot;file.txt&quot;, &quot;Encrypted&quot; : &quot;v0qpsdq8anpci8n929v3uu9338&quot;, &quot;Path&quot; : &quot;full/path/goes/here/file.txt&quot;, &quot;Size&quot; : 6 }</p>
+<p>{ &quot;Hashes&quot; : { &quot;SHA-1&quot; : &quot;f572d396fae9206628714fb2ce00f72e94f2258f&quot;, &quot;MD5&quot; : &quot;b1946ac92492d2347c6235b4d2611184&quot;, &quot;DropboxHash&quot; : &quot;ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc&quot; }, &quot;ID&quot;: &quot;y2djkhiujf83u33&quot;, &quot;IsDir&quot; : false, &quot;MimeType&quot; : &quot;application/octet-stream&quot;, &quot;ModTime&quot; : &quot;2017-05-31T16:15:57.034468261+01:00&quot;, &quot;Name&quot; : &quot;file.txt&quot;, &quot;Encrypted&quot; : &quot;v0qpsdq8anpci8n929v3uu9338&quot;, &quot;Path&quot; : &quot;full/path/goes/here/file.txt&quot;, &quot;Size&quot; : 6 }</p>
 <p>If --hash is not specified the Hashes property won't be emitted.</p>
 <p>If --no-modtime is specified then ModTime will be blank.</p>
 <p>If --encrypted is not specified the Encrypted won't be emitted.</p>
@@ -812,7 +839,7 @@ cd65ac234e6fea5925974a51cdd865cc  canole
 <p>The other list commands <code>lsd</code>,<code>lsf</code>,<code>lsjson</code> do not recurse by default - use &quot;-R&quot; to make them recurse.</p>
 <p>Listing a non existent directory will produce an error except for remotes which can't have empty directories (eg s3, swift, gcs, etc - the bucket based remotes).</p>
 <pre><code>rclone lsjson remote:path [flags]</code></pre>
-<h3 id="options-43">Options</h3>
+<h3 id="options-44">Options</h3>
 <pre><code>  -M, --encrypted    Show the encrypted names.
       --hash         Include hashes in the output (may take longer).
   -h, --help         help for lsjson
@@ -820,7 +847,7 @@ cd65ac234e6fea5925974a51cdd865cc  canole
   -R, --recursive    Recurse into the listing.</code></pre>
 <h2 id="rclone-mount">rclone mount</h2>
 <p>Mount the remote as a mountpoint. <strong>EXPERIMENTAL</strong></p>
-<h3 id="synopsis-44">Synopsis</h3>
+<h3 id="synopsis-45">Synopsis</h3>
 <p>rclone mount allows Linux, FreeBSD, macOS and Windows to mount any of Rclone's cloud storage systems as a file system with FUSE.</p>
 <p>This is <strong>EXPERIMENTAL</strong> - use with care.</p>
 <p>First set up your remote using <code>rclone config</code>. Check it works with <code>rclone ls</code> etc.</p>
@@ -858,6 +885,11 @@ umount /path/to/local/mount</code></pre>
 <p>Note that all the rclone filters can be used to select a subset of the files to be visible in the mount.</p>
 <h3 id="systemd">systemd</h3>
 <p>When running rclone mount as a systemd service, it is possible to use Type=notify. In this case the service will enter the started state after the mountpoint has been successfully set up. Units having the rclone mount service specified as a requirement will see all files and folders immediately in this mode.</p>
+<h3 id="chunked-reading">chunked reading</h3>
+<p>--vfs-read-chunk-size will enable reading the source objects in parts. This can reduce the used download quota for some remotes by requesting only chunks from the remote that are actually read at the cost of an increased number of requests.</p>
+<p>When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size, the chunk size for each open file will get doubled for each chunk read, until the specified value is reached. A value of -1 will disable the limit and the chunk size will grow indefinitely.</p>
+<p>With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on. When --vfs-read-chunk-size-limit 500M is specified, the result would be 0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.</p>
+<p>Chunked reading will only work with --vfs-cache-mode &lt; full, as the file will always be copied to the vfs cache before opening with --vfs-cache-mode full.</p>
 <h3 id="directory-cache">Directory Cache</h3>
 <p>Using the <code>--dir-cache-time</code> flag, you can set how long a directory should be considered up to date and not refreshed from the backend. Changes made locally in the mount may appear immediately or invalidate the cache. However, changes done on the remote will only be picked up once the cache expires.</p>
 <p>Alternatively, you can send a <code>SIGHUP</code> signal to rclone for it to flush all directory caches, regardless of how old they are. Assuming only one rclone instance is running, you can reset the cache like this:</p>
@@ -910,7 +942,7 @@ umount /path/to/local/mount</code></pre>
 <p>This mode should support all normal file system operations.</p>
 <p>If an upload or download fails it will be retried up to --low-level-retries times.</p>
 <pre><code>rclone mount remote:path /path/to/mountpoint [flags]</code></pre>
-<h3 id="options-44">Options</h3>
+<h3 id="options-45">Options</h3>
 <pre><code>      --allow-non-empty                    Allow mounting over a non-empty directory.
       --allow-other                        Allow access to other users.
       --allow-root                         Allow access to root user.
@@ -934,10 +966,13 @@ umount /path/to/local/mount</code></pre>
       --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
       --vfs-cache-mode string              Cache mode off|minimal|writes|full (default &quot;off&quot;)
       --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks.
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
+      --volname string                     Set the volume name (not supported by all OSes).
       --write-back-cache                   Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.</code></pre>
 <h2 id="rclone-moveto">rclone moveto</h2>
 <p>Move file or directory from source to dest.</p>
-<h3 id="synopsis-45">Synopsis</h3>
+<h3 id="synopsis-46">Synopsis</h3>
 <p>If source:path is a file or directory then it moves it to a file or directory named dest:path.</p>
 <p>This can be used to rename files or upload single files to other than their existing name. If the source is a directory then it acts exacty like the move command.</p>
 <p>So</p>
@@ -952,11 +987,11 @@ if src is directory
 <p>This doesn't transfer unchanged files, testing by size and modification time or MD5SUM. src will be deleted on successful transfer.</p>
 <p><strong>Important</strong>: Since this can cause data loss, test first with the --dry-run flag.</p>
 <pre><code>rclone moveto source:path dest:path [flags]</code></pre>
-<h3 id="options-45">Options</h3>
+<h3 id="options-46">Options</h3>
 <pre><code>  -h, --help   help for moveto</code></pre>
 <h2 id="rclone-ncdu">rclone ncdu</h2>
 <p>Explore a remote with a text based user interface.</p>
-<h3 id="synopsis-46">Synopsis</h3>
+<h3 id="synopsis-47">Synopsis</h3>
 <p>This displays a text based user interface allowing the navigation of a remote. It is most useful for answering the question - &quot;What is using all my disk space?&quot;.</p>
 <script src="https://asciinema.org/a/157793.js" id="asciicast-157793" async></script>
 <p>To make the user interface it first scans the entire remote given and builds an in memory representation. rclone ncdu can be used during this scanning phase and you will see it building up the directory structure as it goes along.</p>
@@ -972,61 +1007,61 @@ if src is directory
  q/ESC/c-C to quit</code></pre>
 <p>This an homage to the <a href="https://dev.yorhel.nl/ncdu">ncdu tool</a> but for rclone remotes. It is missing lots of features at the moment, most importantly deleting files, but is useful as it stands.</p>
 <pre><code>rclone ncdu remote:path [flags]</code></pre>
-<h3 id="options-46">Options</h3>
+<h3 id="options-47">Options</h3>
 <pre><code>  -h, --help   help for ncdu</code></pre>
 <h2 id="rclone-obscure">rclone obscure</h2>
 <p>Obscure password for use in the rclone.conf</p>
-<h3 id="synopsis-47">Synopsis</h3>
+<h3 id="synopsis-48">Synopsis</h3>
 <p>Obscure password for use in the rclone.conf</p>
 <pre><code>rclone obscure password [flags]</code></pre>
-<h3 id="options-47">Options</h3>
+<h3 id="options-48">Options</h3>
 <pre><code>  -h, --help   help for obscure</code></pre>
 <h2 id="rclone-rc">rclone rc</h2>
 <p>Run a command against a running rclone.</p>
-<h3 id="synopsis-48">Synopsis</h3>
+<h3 id="synopsis-49">Synopsis</h3>
 <p>This runs a command against a running rclone. By default it will use that specified in the --rc-addr command.</p>
 <p>Arguments should be passed in as parameter=value.</p>
 <p>The result will be returned as a JSON object by default.</p>
-<p>Use &quot;rclone rc list&quot; to see a list of all possible commands.</p>
+<p>Use &quot;rclone rc&quot; to see a list of all possible commands.</p>
 <pre><code>rclone rc commands parameter [flags]</code></pre>
-<h3 id="options-48">Options</h3>
+<h3 id="options-49">Options</h3>
 <pre><code>  -h, --help         help for rc
       --no-output    If set don&#39;t output the JSON result.
       --url string   URL to connect to rclone remote control. (default &quot;http://localhost:5572/&quot;)</code></pre>
 <h2 id="rclone-rcat">rclone rcat</h2>
 <p>Copies standard input to file on remote.</p>
-<h3 id="synopsis-49">Synopsis</h3>
+<h3 id="synopsis-50">Synopsis</h3>
 <p>rclone rcat reads from standard input (stdin) and copies it to a single remote file.</p>
 <pre><code>echo &quot;hello world&quot; | rclone rcat remote:path/to/file
-ffmpeg - | rclone rcat --checksum remote:path/to/file</code></pre>
+ffmpeg - | rclone rcat remote:path/to/file</code></pre>
 <p>If the remote file already exists, it will be overwritten.</p>
 <p>rcat will try to upload small files in a single request, which is usually more efficient than the streaming/chunked upload endpoints, which use multiple requests. Exact behaviour depends on the remote. What is considered a small file may be set through <code>--streaming-upload-cutoff</code>. Uploading only starts after the cutoff is reached or if the file ends before that. The data must fit into RAM. The cutoff needs to be small enough to adhere the limits of your remote, please see there. Generally speaking, setting this cutoff too high will decrease your performance.</p>
 <p>Note that the upload can also not be retried because the data is not kept around until the upload succeeds. If you need to transfer a lot of data, you're better off caching locally and then <code>rclone move</code> it to the destination.</p>
 <pre><code>rclone rcat remote:path [flags]</code></pre>
-<h3 id="options-49">Options</h3>
+<h3 id="options-50">Options</h3>
 <pre><code>  -h, --help   help for rcat</code></pre>
 <h2 id="rclone-rmdirs">rclone rmdirs</h2>
 <p>Remove empty directories under the path.</p>
-<h3 id="synopsis-50">Synopsis</h3>
+<h3 id="synopsis-51">Synopsis</h3>
 <p>This removes any empty directories (or directories that only contain empty directories) under the path that it finds, including the path if it has nothing in.</p>
 <p>If you supply the --leave-root flag, it will not remove the root directory.</p>
 <p>This is useful for tidying up remotes that rclone has left a lot of empty directories in.</p>
 <pre><code>rclone rmdirs remote:path [flags]</code></pre>
-<h3 id="options-50">Options</h3>
+<h3 id="options-51">Options</h3>
 <pre><code>  -h, --help         help for rmdirs
       --leave-root   Do not remove root directory if empty</code></pre>
 <h2 id="rclone-serve">rclone serve</h2>
 <p>Serve a remote over a protocol.</p>
-<h3 id="synopsis-51">Synopsis</h3>
+<h3 id="synopsis-52">Synopsis</h3>
 <p>rclone serve is used to serve a remote over a given protocol. This command requires the use of a subcommand to specify the protocol, eg</p>
 <pre><code>rclone serve http remote:</code></pre>
 <p>Each subcommand has its own options which you can see in their help.</p>
 <pre><code>rclone serve &lt;protocol&gt; [opts] &lt;remote&gt; [flags]</code></pre>
-<h3 id="options-51">Options</h3>
+<h3 id="options-52">Options</h3>
 <pre><code>  -h, --help   help for serve</code></pre>
 <h2 id="rclone-serve-http">rclone serve http</h2>
 <p>Serve the remote over HTTP.</p>
-<h3 id="synopsis-52">Synopsis</h3>
+<h3 id="synopsis-53">Synopsis</h3>
 <p>rclone serve http implements a basic web server to serve the remote over HTTP. This can be viewed in a web browser or you can make a remote of type http read from it.</p>
 <p>You can use the filter flags (eg --include, --exclude) to control what is served.</p>
 <p>The server will log errors. Use -v to see access logs.</p>
@@ -1101,7 +1136,7 @@ htpasswd -B htpasswd anotherUser</code></pre>
 <p>This mode should support all normal file system operations.</p>
 <p>If an upload or download fails it will be retried up to --low-level-retries times.</p>
 <pre><code>rclone serve http remote:path [flags]</code></pre>
-<h3 id="options-52">Options</h3>
+<h3 id="options-53">Options</h3>
 <pre><code>      --addr string                        IPaddress:Port or :Port to bind server to. (default &quot;localhost:8080&quot;)
       --cert string                        SSL PEM key (concatenation of certificate and CA certificate)
       --client-ca string                   Client certificate authority to verify clients with
@@ -1125,10 +1160,12 @@ htpasswd -B htpasswd anotherUser</code></pre>
       --user string                        User name for authentication.
       --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
       --vfs-cache-mode string              Cache mode off|minimal|writes|full (default &quot;off&quot;)
-      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)</code></pre>
+      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks.
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.</code></pre>
 <h2 id="rclone-serve-restic">rclone serve restic</h2>
 <p>Serve the remote for restic's REST API.</p>
-<h3 id="synopsis-53">Synopsis</h3>
+<h3 id="synopsis-54">Synopsis</h3>
 <p>rclone serve restic implements restic's REST backend API over HTTP. This allows restic to use rclone as a data storage mechanism for cloud providers that restic does not support directly.</p>
 <p><a href="https://restic.net/">Restic</a> is a command line program for doing backups.</p>
 <p>The server will log errors. Use -v to see access logs.</p>
@@ -1185,7 +1222,7 @@ htpasswd -B htpasswd anotherUser</code></pre>
 <p>By default this will serve over http. If you want you can serve over https. You will need to supply the --cert and --key flags. If you wish to do client side certificate validation then you will need to supply --client-ca also.</p>
 <p>--cert should be a either a PEM encoded certificate or a concatenation of that with the CA certificate. --key should be the PEM encoded private key and --client-ca should be the PEM encoded client certificate authority certificate.</p>
 <pre><code>rclone serve restic remote:path [flags]</code></pre>
-<h3 id="options-53">Options</h3>
+<h3 id="options-54">Options</h3>
 <pre><code>      --addr string                     IPaddress:Port or :Port to bind server to. (default &quot;localhost:8080&quot;)
       --append-only                     disallow deletion of repository data
       --cert string                     SSL PEM key (concatenation of certificate and CA certificate)
@@ -1202,7 +1239,7 @@ htpasswd -B htpasswd anotherUser</code></pre>
       --user string                     User name for authentication.</code></pre>
 <h2 id="rclone-serve-webdav">rclone serve webdav</h2>
 <p>Serve remote:path over webdav.</p>
-<h3 id="synopsis-54">Synopsis</h3>
+<h3 id="synopsis-55">Synopsis</h3>
 <p>rclone serve webdav implements a basic webdav server to serve the remote over HTTP via the webdav protocol. This can be viewed with a webdav client or you can make a remote of type webdav to read and write it.</p>
 <p>NB at the moment each directory listing reads the start of each file which is undesirable: see https://github.com/golang/go/issues/22577</p>
 <h3 id="server-options-2">Server options</h3>
@@ -1275,7 +1312,7 @@ htpasswd -B htpasswd anotherUser</code></pre>
 <p>This mode should support all normal file system operations.</p>
 <p>If an upload or download fails it will be retried up to --low-level-retries times.</p>
 <pre><code>rclone serve webdav remote:path [flags]</code></pre>
-<h3 id="options-54">Options</h3>
+<h3 id="options-55">Options</h3>
 <pre><code>      --addr string                        IPaddress:Port or :Port to bind server to. (default &quot;localhost:8080&quot;)
       --cert string                        SSL PEM key (concatenation of certificate and CA certificate)
       --client-ca string                   Client certificate authority to verify clients with
@@ -1299,19 +1336,21 @@ htpasswd -B htpasswd anotherUser</code></pre>
       --user string                        User name for authentication.
       --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
       --vfs-cache-mode string              Cache mode off|minimal|writes|full (default &quot;off&quot;)
-      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)</code></pre>
+      --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks.
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.</code></pre>
 <h2 id="rclone-touch">rclone touch</h2>
 <p>Create new file or change file modification time.</p>
-<h3 id="synopsis-55">Synopsis</h3>
+<h3 id="synopsis-56">Synopsis</h3>
 <p>Create new file or change file modification time.</p>
 <pre><code>rclone touch remote:path [flags]</code></pre>
-<h3 id="options-55">Options</h3>
+<h3 id="options-56">Options</h3>
 <pre><code>  -h, --help               help for touch
   -C, --no-create          Do not create the file if it does not exist.
   -t, --timestamp string   Change the modification times to the specified time instead of the current time of day. The argument is of the form &#39;YYMMDD&#39; (ex. 17.10.30) or &#39;YYYY-MM-DDTHH:MM:SS&#39; (ex. 2006-01-02T15:04:05)</code></pre>
 <h2 id="rclone-tree">rclone tree</h2>
 <p>List the contents of the remote in a tree like fashion.</p>
-<h3 id="synopsis-56">Synopsis</h3>
+<h3 id="synopsis-57">Synopsis</h3>
 <p>rclone tree lists the contents of a remote in a similar way to the unix tree command.</p>
 <p>For example</p>
 <pre><code>$ rclone tree remote:path
@@ -1327,7 +1366,7 @@ htpasswd -B htpasswd anotherUser</code></pre>
 <p>You can use any of the filtering options with the tree command (eg --include and --exclude). You can also use --fast-list.</p>
 <p>The tree command has many options for controlling the listing which are compatible with the tree command. Note that not all of them have short options as they conflict with rclone's short options.</p>
 <pre><code>rclone tree remote:path [flags]</code></pre>
-<h3 id="options-56">Options</h3>
+<h3 id="options-57">Options</h3>
 <pre><code>  -a, --all             All files are listed (list . files too).
   -C, --color           Turn colorization on always.
   -d, --dirs-only       List directories only.
@@ -1391,7 +1430,7 @@ htpasswd -B htpasswd anotherUser</code></pre>
 <p>This can be used when scripting to make aged backups efficiently, eg</p>
 <pre><code>rclone sync remote:current-backup remote:previous-backup
 rclone sync /path/to/files remote:current-backup</code></pre>
-<h2 id="options-57">Options</h2>
+<h2 id="options-58">Options</h2>
 <p>Rclone has a number of options to control its behaviour.</p>
 <p>Options which use TIME use the go time parser. A duration string is a possibly signed sequence of decimal numbers, each with optional fraction and a unit suffix, such as &quot;300ms&quot;, &quot;-1.5h&quot; or &quot;2h45m&quot;. Valid time units are &quot;ns&quot;, &quot;us&quot; (or &quot;µs&quot;), &quot;ms&quot;, &quot;s&quot;, &quot;m&quot;, &quot;h&quot;.</p>
 <p>Options which use SIZE use kByte by default. However, a suffix of <code>b</code> for bytes, <code>k</code> for kBytes, <code>M</code> for MBytes, <code>G</code> for GBytes, <code>T</code> for TBytes and <code>P</code> for PBytes may be used. These are the binary units, eg 1, 2**10, 2**20, 2**30 respectively.</p>
@@ -1493,6 +1532,10 @@ rclone sync /path/to/files remote:current-backup</code></pre>
 <p>For historical reasons the <code>lsd</code> command defaults to using a <code>--max-depth</code> of 1 - you can override this with the command line flag.</p>
 <p>You can use this command to disable recursion (with <code>--max-depth 1</code>).</p>
 <p>Note that if you use this with <code>sync</code> and <code>--delete-excluded</code> the files not recursed through are considered excluded and will be deleted on the destination. Test first with <code>--dry-run</code> if you are not sure what will happen.</p>
+<h3 id="max-transfersize">--max-transfer=SIZE</h3>
+<p>Rclone will stop transferring when it has reached the size specified. Defaults to off.</p>
+<p>When the limit is reached all transfers will stop immediately.</p>
+<p>Rclone will exit with exit code 8 if the transfer limit is reached.</p>
 <h3 id="modify-windowtime">--modify-window=TIME</h3>
 <p>When checking whether a file has been modified, this is the maximum allowed time difference that a file can have and still be considered equivalent.</p>
 <p>The default is <code>1ns</code> unless this is overridden by a remote. For example OS X only stores modification times to the nearest second so if you are reading and writing to an OS X filing system this will be <code>1s</code> by default.</p>
@@ -1509,6 +1552,9 @@ rclone sync /path/to/files remote:current-backup</code></pre>
 <p>Retry the entire sync if it fails this many times it fails (default 3).</p>
 <p>Some remotes can be unreliable and a few retries help pick up the files which didn't get transferred because of errors.</p>
 <p>Disable retries with <code>--retries 1</code>.</p>
+<h3 id="retries-sleeptime">--retries-sleep=TIME</h3>
+<p>This sets the interval between each retry specified by <code>--retries</code></p>
+<p>The default is 0. Use 0 to disable.</p>
 <h3 id="size-only">--size-only</h3>
 <p>Normally rclone will look at modification time and size of files to see if they are equal. If you set this flag then rclone will check only the size.</p>
 <p>This can be useful transferring files from Dropbox which have been modified by the desktop sync client which doesn't set checksums of modification times in the same way as rclone.</p>
@@ -1518,6 +1564,7 @@ rclone sync /path/to/files remote:current-backup</code></pre>
 <p>The default is <code>1m</code>. Use 0 to disable.</p>
 <p>If you set the stats interval then all commands can show stats. This can be useful when running other commands, <code>check</code> or <code>mount</code> for example.</p>
 <p>Stats are logged at <code>INFO</code> level by default which means they won't show at default log level <code>NOTICE</code>. Use <code>--stats-log-level NOTICE</code> or <code>-v</code> to make them show. See the <a href="#logging">Logging section</a> for more info on log levels.</p>
+<p>Note that on macOS you can send a SIGINFO (which is normally ctrl-T in the terminal) to make the stats print immediately.</p>
 <h3 id="stats-file-name-length-integer">--stats-file-name-length integer</h3>
 <p>By default, the <code>--stats</code> output will truncate file names and paths longer than 40 characters. This is equivalent to providing <code>--stats-file-name-length 40</code>. Use <code>--stats-file-name-length 0</code> to disable any truncation of file names printed by stats.</p>
 <h3 id="stats-log-level-string">--stats-log-level string</h3>
@@ -1713,10 +1760,11 @@ export RCLONE_CONFIG_PASS</code></pre>
 <li><code>5</code> - Temporary error (one that more retries might fix) (Retry errors)</li>
 <li><code>6</code> - Less serious errors (like 461 errors from dropbox) (NoRetry errors)</li>
 <li><code>7</code> - Fatal error (one that more retries won't fix, like account suspended) (Fatal errors)</li>
+<li><code>8</code> - Transfer exceeded - limit set by --max-transfer reached</li>
 </ul>
 <h2 id="environment-variables">Environment Variables</h2>
 <p>Rclone can be configured entirely using environment variables. These can be used to set defaults for options or config file entries.</p>
-<h3 id="options-58">Options</h3>
+<h3 id="options-59">Options</h3>
 <p>Every option in rclone can have its default set by environment variable.</p>
 <p>To find the name of the environment variable, first, take the long option name, strip the leading <code>--</code>, change <code>-</code> to <code>_</code>, make upper case and prepend <code>RCLONE_</code>.</p>
 <p>For example, to always set <code>--stats 5s</code>, set the environment variable <code>RCLONE_STATS=5s</code>. If you set stats on the command line this will override the environment variable setting.</p>
@@ -2077,6 +2125,8 @@ rclone rc cache/expire remote=/ withData=true</code></pre>
 <pre><code>rclone rc core/bwlimit rate=1M
 rclone rc core/bwlimit rate=off</code></pre>
 <p>The format of the parameter is exactly the same as passed to --bwlimit except only one bandwidth may be specified.</p>
+<h3 id="coregc-runs-a-garbage-collection.">core/gc: Runs a garbage collection.</h3>
+<p>This tells the go runtime to do a garbage collection run. It isn't necessary to call this normally, but it can be useful for debugging memory problems.</p>
 <h3 id="corememstats-returns-the-memory-statistics">core/memstats: Returns the memory statistics</h3>
 <p>This returns the memory statistics of the running program. What the values mean are explained in the go docs: https://golang.org/pkg/runtime/#MemStats</p>
 <p>The most interesting values for most people are:</p>
@@ -2159,6 +2209,36 @@ $ echo $?
     &quot;rutabaga&quot;: &quot;3&quot;,
     &quot;sausage&quot;: 1
 }</code></pre>
+<h2 id="debugging-rclone-with-pprof">Debugging rclone with pprof</h2>
+<p>If you use the <code>--rc</code> flag this will also enable the use of the go profiling tools on the same port.</p>
+<p>To use these, first <a href="https://golang.org/doc/install">install go</a>.</p>
+<p>Then (for example) to profile rclone's memory use you can run:</p>
+<pre><code>go tool pprof -web http://localhost:5572/debug/pprof/heap</code></pre>
+<p>This should open a page in your browser showing what is using what memory.</p>
+<p>You can also use the <code>-text</code> flag to produce a textual summary</p>
+<pre><code>$ go tool pprof -text http://localhost:5572/debug/pprof/heap
+Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
+      flat  flat%   sum%        cum   cum%
+ 1024.03kB 66.62% 66.62%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode
+     513kB 33.38%   100%      513kB 33.38%  net/http.newBufioWriterSize
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/all.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve/restic.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0
+         0     0%   100%  1024.03kB 66.62%  main.init
+         0     0%   100%      513kB 33.38%  net/http.(*conn).readRequest
+         0     0%   100%      513kB 33.38%  net/http.(*conn).serve
+         0     0%   100%  1024.03kB 66.62%  runtime.main</code></pre>
+<p>Possible profiles to look at:</p>
+<ul>
+<li>Memory: <code>go tool pprof http://localhost:5572/debug/pprof/heap</code></li>
+<li>30-second CPU profile: <code>go tool pprof http://localhost:5572/debug/pprof/profile</code></li>
+<li>5-second execution trace: <code>wget http://localhost:5572/debug/pprof/trace?seconds=5</code></li>
+</ul>
+<p>See the <a href="https://golang.org/pkg/net/http/pprof/">net/http/pprof docs</a> for more info on how to use the profiling and for a general overview see <a href="https://blog.golang.org/profiling-go-programs">the Go team's blog post on profiling go programs</a>.</p>
+<p>The profiling hook is <a href="https://stackoverflow.com/q/26545159/164234">zero overhead unless it is used</a>.</p>
 <h1 id="overview-of-cloud-storage-systems">Overview of cloud storage systems</h1>
 <p>Each cloud storage system is slightly different. Rclone attempts to provide a unified interface to them, but some underlying differences show through.</p>
 <h2 id="features">Features</h2>
@@ -2280,6 +2360,14 @@ $ echo $?
 <td align="center">R</td>
 </tr>
 <tr class="even">
+<td>OpenDrive</td>
+<td align="center">MD5</td>
+<td align="center">Yes</td>
+<td align="center">Yes</td>
+<td align="center">No</td>
+<td align="center">-</td>
+</tr>
+<tr class="odd">
 <td>Openstack Swift</td>
 <td align="center">MD5</td>
 <td align="center">Yes</td>
@@ -2287,7 +2375,7 @@ $ echo $?
 <td align="center">No</td>
 <td align="center">R/W</td>
 </tr>
-<tr class="odd">
+<tr class="even">
 <td>pCloud</td>
 <td align="center">MD5, SHA1</td>
 <td align="center">Yes</td>
@@ -2295,7 +2383,7 @@ $ echo $?
 <td align="center">No</td>
 <td align="center">W</td>
 </tr>
-<tr class="even">
+<tr class="odd">
 <td>QingStor</td>
 <td align="center">MD5</td>
 <td align="center">No</td>
@@ -2303,7 +2391,7 @@ $ echo $?
 <td align="center">No</td>
 <td align="center">R/W</td>
 </tr>
-<tr class="odd">
+<tr class="even">
 <td>SFTP</td>
 <td align="center">MD5, SHA1 ‡</td>
 <td align="center">Yes</td>
@@ -2311,7 +2399,7 @@ $ echo $?
 <td align="center">No</td>
 <td align="center">-</td>
 </tr>
-<tr class="even">
+<tr class="odd">
 <td>WebDAV</td>
 <td align="center">-</td>
 <td align="center">Yes ††</td>
@@ -2319,7 +2407,7 @@ $ echo $?
 <td align="center">No</td>
 <td align="center">-</td>
 </tr>
-<tr class="odd">
+<tr class="even">
 <td>Yandex Disk</td>
 <td align="center">MD5</td>
 <td align="center">Yes</td>
@@ -2327,7 +2415,7 @@ $ echo $?
 <td align="center">No</td>
 <td align="center">R/W</td>
 </tr>
-<tr class="even">
+<tr class="odd">
 <td>The local filesystem</td>
 <td align="center">All</td>
 <td align="center">Yes</td>
@@ -2540,6 +2628,18 @@ $ echo $?
 <td align="center">Yes</td>
 </tr>
 <tr class="even">
+<td>OpenDrive</td>
+<td align="center">Yes</td>
+<td align="center">Yes</td>
+<td align="center">Yes</td>
+<td align="center">Yes</td>
+<td align="center">No</td>
+<td align="center">No</td>
+<td align="center">No</td>
+<td align="center">No</td>
+<td align="center">No</td>
+</tr>
+<tr class="odd">
 <td>Openstack Swift</td>
 <td align="center">Yes †</td>
 <td align="center">Yes</td>
@@ -2551,7 +2651,7 @@ $ echo $?
 <td align="center">No <a href="https://github.com/ncw/rclone/issues/2178">#2178</a></td>
 <td align="center">Yes</td>
 </tr>
-<tr class="odd">
+<tr class="even">
 <td>pCloud</td>
 <td align="center">Yes</td>
 <td align="center">Yes</td>
@@ -2563,7 +2663,7 @@ $ echo $?
 <td align="center">No <a href="https://github.com/ncw/rclone/issues/2178">#2178</a></td>
 <td align="center">Yes</td>
 </tr>
-<tr class="even">
+<tr class="odd">
 <td>QingStor</td>
 <td align="center">No</td>
 <td align="center">Yes</td>
@@ -2575,7 +2675,7 @@ $ echo $?
 <td align="center">No <a href="https://github.com/ncw/rclone/issues/2178">#2178</a></td>
 <td align="center">No</td>
 </tr>
-<tr class="odd">
+<tr class="even">
 <td>SFTP</td>
 <td align="center">No</td>
 <td align="center">No</td>
@@ -2587,7 +2687,7 @@ $ echo $?
 <td align="center">No <a href="https://github.com/ncw/rclone/issues/2178">#2178</a></td>
 <td align="center">No</td>
 </tr>
-<tr class="even">
+<tr class="odd">
 <td>WebDAV</td>
 <td align="center">Yes</td>
 <td align="center">Yes</td>
@@ -2599,7 +2699,7 @@ $ echo $?
 <td align="center">No <a href="https://github.com/ncw/rclone/issues/2178">#2178</a></td>
 <td align="center">No</td>
 </tr>
-<tr class="odd">
+<tr class="even">
 <td>Yandex Disk</td>
 <td align="center">Yes</td>
 <td align="center">No</td>
@@ -2611,7 +2711,7 @@ $ echo $?
 <td align="center">No <a href="https://github.com/ncw/rclone/issues/2178">#2178</a></td>
 <td align="center">No</td>
 </tr>
-<tr class="even">
+<tr class="odd">
 <td>The local filesystem</td>
 <td align="center">Yes</td>
 <td align="center">No</td>
@@ -3172,6 +3272,9 @@ y/e/d&gt; </code></pre>
 <p>Any files larger than this will be uploaded in chunks of this size. The default is 5MB. The minimum is 5MB.</p>
 <p>Note that 2 chunks of this size are buffered in memory per transfer.</p>
 <p>If you are transferring large files over high speed links and you have enough memory, then increasing this will speed up the transfers.</p>
+<h4 id="s3-upload-concurrency">--s3-upload-concurrency</h4>
+<p>Number of chunks of the same file that are uploaded concurrently. Default is 2.</p>
+<p>If you are uploading small amount of large file over high speed link and these uploads do not fully utilize your bandwidth, then increasing this may help to speed up the transfers.</p>
 <h3 id="anonymous-access-to-public-buckets">Anonymous access to public buckets</h3>
 <p>If you want to use rclone to access a public bucket, configure with a blank <code>access_key_id</code> and <code>secret_access_key</code>. Your config should end up looking like this:</p>
 <pre><code>[anons3]
@@ -4394,6 +4497,12 @@ y/e/d&gt; y</code></pre>
 <pre><code>rclone ls remote:</code></pre>
 <p>To copy a local directory to a dropbox directory called backup</p>
 <pre><code>rclone copy /home/source remote:backup</code></pre>
+<h3 id="dropbox-for-business">Dropbox for business</h3>
+<p>Rclone supports Dropbox for business and Team Folders.</p>
+<p>When using Dropbox for business <code>remote:</code> and <code>remote:path/to/file</code> will refer to your personal folder.</p>
+<p>If you wish to see Team Folders you must use a leading <code>/</code> in the path, so <code>rclone lsd remote:/</code> will refer to the root and show you all Team Folders and your User Folder.</p>
+<p>You can then use team folders like this <code>remote:/TeamFolder</code> and <code>remote:/TeamFolder/path/to/file</code>.</p>
+<p>A leading <code>/</code> for a Dropbox personal account will do nothing, but it will take an extra HTTP transaction so it should be avoided.</p>
 <h3 id="modified-time-and-hashes-2">Modified time and Hashes</h3>
 <p>Dropbox supports modified times, but the only way to set a modification time is to re-upload the file.</p>
 <p>This means that if you uploaded your data with an older version of rclone which didn't support the v2 API and modified times, rclone will decide to upload all your old data to fix the modification times. If you don't want this to happen use <code>--size-only</code> or <code>--checksum</code> flag to stop it.</p>
@@ -4866,6 +4975,8 @@ y/e/d&gt; y</code></pre>
 <p>To view your current quota you can use the <code>rclone about remote:</code> command which will display your usage limit (quota), the usage in Google Drive, the size of all files in the Trash and the space used by other Google services such as Gmail. This command does not take any path arguments.</p>
 <h3 id="specific-options-7">Specific options</h3>
 <p>Here are the command line options specific to this cloud storage system.</p>
+<h4 id="drive-acknowledge-abuse">--drive-acknowledge-abuse</h4>
+<p>If downloading a file returns the error <code>This file has been identified as malware or spam and cannot be downloaded</code> with the error code <code>cannotDownloadAbusiveFile</code> then supply this flag to rclone to indicate you acknowledge the risks of downloading the file and rclone will download it anyway.</p>
 <h4 id="drive-auth-owner-only">--drive-auth-owner-only</h4>
 <p>Only consider files owned by the authenticated user.</p>
 <h4 id="drive-chunk-sizesize">--drive-chunk-size=SIZE</h4>
@@ -4995,6 +5106,9 @@ y/e/d&gt; y</code></pre>
 </tr>
 </tbody>
 </table>
+<h4 id="drive-alternate-export">--drive-alternate-export</h4>
+<p>If this option is set this instructs rclone to use an alternate set of export URLs for drive documents. Users have reported that the official export URLs can't export large documents, whereas these unofficial ones can.</p>
+<p>See rclone issue <a href="https://github.com/ncw/rclone/issues/2243">#2243</a> for background, <a href="https://issuetracker.google.com/issues/36761333">this google drive issue</a> and <a href="https://www.labnol.org/internet/direct-links-for-google-drive/28356/">this helpful post</a>.</p>
 <h4 id="drive-impersonate-user">--drive-impersonate user</h4>
 <p>When using a service account, this instructs rclone to impersonate the user passed in.</p>
 <h4 id="drive-list-chunk-int">--drive-list-chunk int</h4>
@@ -5465,7 +5579,7 @@ y/e/d&gt; y</code></pre>
  * Say p for a personal OneDrive account
 b) Business
 p) Personal
-b/p&gt; </code></pre>
+b/p&gt;</code></pre>
 <p>After that rclone requires an authentication of your account. The application will first authenticate your account, then query the OneDrive resource URL and do a second (silent) authentication for this resource URL.</p>
 <h3 id="modified-time-and-hashes-4">Modified time and hashes</h3>
 <p>OneDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.</p>
@@ -5482,8 +5596,7 @@ b/p&gt; </code></pre>
 <p>There are quite a few characters that can't be in OneDrive file names. These can't occur on Windows platforms, but on non-Windows platforms they are common. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a <code>?</code> in it will be mapped to <code>？</code> instead.</p>
 <p>The largest allowed file size is 10GiB (10,737,418,240 bytes).</p>
 <h3 id="versioning-issue">Versioning issue</h3>
-<p>Every change in OneDrive causes the service to create a new version. This counts against a users quota.<br />
-For example changing the modification time of a file creates a second version, so the file is using twice the space.</p>
+<p>Every change in OneDrive causes the service to create a new version. This counts against a users quota. For example changing the modification time of a file creates a second version, so the file is using twice the space.</p>
 <p>The <code>copy</code> is the only rclone command affected by this as we copy the file and then afterwards set the modification time to match the source file.</p>
 <p>User <a href="https://github.com/Weropol">Weropol</a> has found a method to disable versioning on OneDrive</p>
 <ol style="list-style-type: decimal">
@@ -5492,12 +5605,91 @@ For example changing the modification time of a file creates a second version, s
 <li>Once on the Site settings page, navigate to Site Administration &gt; Site libraries and lists.</li>
 <li>Click Customize &quot;Documents&quot;.</li>
 <li>Click General Settings &gt; Versioning Settings.</li>
-<li>Under Document Version History select the option No versioning.<br />
-Note: This will disable the creation of new file versions, but will not remove any previous versions. Your documents are safe.</li>
+<li>Under Document Version History select the option No versioning. Note: This will disable the creation of new file versions, but will not remove any previous versions. Your documents are safe.</li>
 <li>Apply the changes by clicking OK.</li>
 <li>Use rclone to upload or modify files. (I also use the --no-update-modtime flag)</li>
 <li>Restore the versioning settings after using rclone. (Optional)</li>
 </ol>
+<h3 id="troubleshooting">Troubleshooting</h3>
+<pre><code>Error: access_denied
+Code: AADSTS65005
+Description: Using application &#39;rclone&#39; is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned.</code></pre>
+<p>This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins.</p>
+<p>However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint</p>
+<h2 id="opendrive">OpenDrive</h2>
+<p>Paths are specified as <code>remote:path</code></p>
+<p>Paths may be as deep as required, eg <code>remote:directory/subdirectory</code>.</p>
+<p>Here is an example of how to make a remote called <code>remote</code>. First run:</p>
+<pre><code> rclone config</code></pre>
+<p>This will guide you through an interactive setup process:</p>
+<pre><code>n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q&gt; n
+name&gt; remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ &quot;amazon cloud drive&quot;
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ &quot;s3&quot;
+ 3 / Backblaze B2
+   \ &quot;b2&quot;
+ 4 / Dropbox
+   \ &quot;dropbox&quot;
+ 5 / Encrypt/Decrypt a remote
+   \ &quot;crypt&quot;
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ &quot;google cloud storage&quot;
+ 7 / Google Drive
+   \ &quot;drive&quot;
+ 8 / Hubic
+   \ &quot;hubic&quot;
+ 9 / Local Disk
+   \ &quot;local&quot;
+10 / OpenDrive
+   \ &quot;opendrive&quot;
+11 / Microsoft OneDrive
+   \ &quot;onedrive&quot;
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ &quot;swift&quot;
+13 / SSH/SFTP Connection
+   \ &quot;sftp&quot;
+14 / Yandex Disk
+   \ &quot;yandex&quot;
+Storage&gt; 10
+Username
+username&gt;
+Password
+y) Yes type in my own password
+g) Generate random password
+y/g&gt; y
+Enter the password:
+password:
+Confirm the password:
+password:
+--------------------
+[remote]
+username =
+password = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d&gt; y</code></pre>
+<p>List directories in top level of your OpenDrive</p>
+<pre><code>rclone lsd remote:</code></pre>
+<p>List all the files in your OpenDrive</p>
+<pre><code>rclone ls remote:</code></pre>
+<p>To copy a local directory to an OpenDrive directory called backup</p>
+<pre><code>rclone copy /home/source remote:backup</code></pre>
+<h3 id="modified-time-and-md5sums-1">Modified time and MD5SUMs</h3>
+<p>OpenDrive allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not.</p>
+<h3 id="deleting-files-4">Deleting files</h3>
+<p>Any files you delete with rclone will end up in the trash. Amazon don't provide an API to permanently delete files, nor to empty the trash, so you will have to do that with one of Amazon's apps or via the OpenDrive website. As of November 17, 2016, files are automatically deleted by Amazon from the trash after 30 days.</p>
+<h3 id="limitations-10">Limitations</h3>
+<p>Note that OpenDrive is case insensitive so you can't have a file called &quot;Hello.doc&quot; and one called &quot;hello.doc&quot;.</p>
+<p>There are quite a few characters that can't be in OpenDrive file names. These can't occur on Windows platforms, but on non-Windows platforms they are common. Rclone will map these names to and from an identical looking unicode equivalent. For example if a file has a <code>?</code> in it will be mapped to <code>？</code> instead.</p>
 <h2 id="qingstor">QingStor</h2>
 <p>Paths are specified as <code>remote:bucket</code> (or <code>remote:</code> for the <code>lsd</code> command.) You may put subdirectories in too, eg <code>remote:bucket/path/to/dir</code>.</p>
 <p>Here is an example of making an QingStor configuration. First run</p>
@@ -5803,9 +5995,9 @@ rclone lsd myremote:</code></pre>
 <h3 id="modified-time-8">Modified time</h3>
 <p>The modified time is stored as metadata on the object as <code>X-Object-Meta-Mtime</code> as floating point since the epoch accurate to 1 ns.</p>
 <p>This is a defacto standard (used in the official python-swiftclient amongst others) for storing the modification time for an object.</p>
-<h3 id="limitations-10">Limitations</h3>
+<h3 id="limitations-11">Limitations</h3>
 <p>The Swift API doesn't return a correct MD5SUM for segmented files (Dynamic or Static Large Objects) so rclone won't check or use the MD5SUM for these.</p>
-<h3 id="troubleshooting">Troubleshooting</h3>
+<h3 id="troubleshooting-1">Troubleshooting</h3>
 <h4 id="rclone-gives-failed-to-create-file-system-for-remote-bad-request">Rclone gives Failed to create file system for &quot;remote:&quot;: Bad Request</h4>
 <p>Due to an oddity of the underlying swift library, it gives a &quot;Bad Request&quot; error rather than a more sensible error when the authentication fails for Swift.</p>
 <p>So this most likely means your username / password is wrong. You can investigate further with the <code>--dump-bodies</code> flag.</p>
@@ -5903,7 +6095,7 @@ y/e/d&gt; y</code></pre>
 <h3 id="modified-time-and-hashes-5">Modified time and hashes</h3>
 <p>pCloud allows modification times to be set on objects accurate to 1 second. These will be used to detect whether objects need syncing or not. In order to set a Modification time pCloud requires the object be re-uploaded.</p>
 <p>pCloud supports MD5 and SHA1 type hashes, so you can use the <code>--checksum</code> flag.</p>
-<h3 id="deleting-files-4">Deleting files</h3>
+<h3 id="deleting-files-5">Deleting files</h3>
 <p>Deleted files will be moved to the trash. Your subscription level will determine how long items stay in the trash. <code>rclone cleanup</code> can be used to empty the trash.</p>
 <h2 id="sftp">SFTP</h2>
 <p>SFTP is the <a href="https://en.wikipedia.org/wiki/SSH_File_Transfer_Protocol">Secure (or SSH) File Transfer Protocol</a>.</p>
@@ -6010,11 +6202,17 @@ y/e/d&gt; y</code></pre>
 <p>Here are the command line options specific to this remote.</p>
 <h4 id="sftp-ask-password">--sftp-ask-password</h4>
 <p>Ask for the SFTP password if needed when no password has been configured.</p>
+<h4 id="ssh-path-override">--ssh-path-override</h4>
+<p>Override path used by SSH connection. Allows checksum calculation when SFTP and SSH paths are different. This issue affects among others Synology NAS boxes.</p>
+<p>Shared folders can be found in directories representing volumes</p>
+<pre><code>rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory</code></pre>
+<p>Home directory can be found in a shared folder called <code>homes</code></p>
+<pre><code>rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory</code></pre>
 <h3 id="modified-time-9">Modified time</h3>
 <p>Modified times are stored on the server to 1 second precision.</p>
 <p>Modified times are used in syncing and are fully supported.</p>
 <p>Some SFTP servers disable setting/modifying the file modification time after upload (for example, certain configurations of ProFTPd with mod_sftp). If you are using one of these servers, you can set the option <code>set_modtime = false</code> in your RClone backend configuration to disable this behaviour.</p>
-<h3 id="limitations-11">Limitations</h3>
+<h3 id="limitations-12">Limitations</h3>
 <p>SFTP supports checksums if the same login has shell access and <code>md5sum</code> or <code>sha1sum</code> as well as <code>echo</code> are in the remote's PATH. This remote checksumming (file hashing) is recommended and enabled by default. Disabling the checksumming may be required if you are connecting to SFTP servers which are not under your control, and to which the execution of remote commands is prohibited. Set the configuration option <code>disable_hashcheck</code> to <code>true</code> to disable checksumming.</p>
 <p>Note that some SFTP servers (eg Synology) the paths are different for SSH and SFTP so the hashes can't be calculated properly. For them using <code>disable_hashcheck</code> is a good idea.</p>
 <p>The only ssh agent supported under Windows is Putty's pageant.</p>
@@ -6326,6 +6524,103 @@ nounc = true</code></pre>
 <p>This flag disables warning messages on skipped symlinks or junction points, as you explicitly acknowledge that they should be skipped.</p>
 <h2 id="changelog">Changelog</h2>
 <ul>
+<li>v1.42 - 2018-06-16
+<ul>
+<li>New backends</li>
+<li>OpenDrive (Oliver Heyme, Jakub Karlicek, ncw)</li>
+<li>New commands</li>
+<li>deletefile command (Filip Bartodziej)</li>
+<li>New Features</li>
+<li>copy, move: Copy single files directly, don't use <code>--files-from</code> work-around
+<ul>
+<li>this makes them much more efficient</li>
+</ul></li>
+<li>Implement <code>--max-transfer</code> flag to quit transferring at a limit
+<ul>
+<li>make exit code 8 for <code>--max-transfer</code> exceeded</li>
+</ul></li>
+<li>copy: copy empty source directories to destination (Ishuah Kariuki)</li>
+<li>check: Add <code>--one-way</code> flag (Kasper Byrdal Nielsen)</li>
+<li>Add siginfo handler for macOS for ctrl-T stats (kubatasiemski)</li>
+<li>rc
+<ul>
+<li>add core/gc to run a garbage collection on demand</li>
+<li>enable go profiling by default on the <code>--rc</code> port</li>
+<li>return error from remote on failure</li>
+</ul></li>
+<li>lsf
+<ul>
+<li>Add <code>--absolute</code> flag to add a leading / onto path names</li>
+<li>Add <code>--csv</code> flag for compliant CSV output</li>
+<li>Add 'm' format specifier to show the MimeType</li>
+<li>Implement 'i' format for showing object ID</li>
+</ul></li>
+<li>lsjson
+<ul>
+<li>Add MimeType to the output</li>
+<li>Add ID field to output to show Object ID</li>
+</ul></li>
+<li>Add <code>--retries-sleep</code> flag (Benjamin Joseph Dag)</li>
+<li>Oauth tidy up web page and error handling (Henning Surmeier)</li>
+<li>Bug Fixes</li>
+<li>Password prompt output with <code>--log-file</code> fixed for unix (Filip Bartodziej)</li>
+<li>Calculate ModifyWindow each time on the fly to fix various problems (Stefan Breunig)</li>
+<li>Mount</li>
+<li>Only print &quot;File.rename error&quot; if there actually is an error (Stefan Breunig)</li>
+<li>Delay rename if file has open writers instead of failing outright (Stefan Breunig)</li>
+<li>Ensure atexit gets run on interrupt</li>
+<li>macOS enhancements
+<ul>
+<li>Make <code>--noappledouble</code> <code>--noapplexattr</code></li>
+<li>Add <code>--volname</code> flag and remove special chars from it</li>
+<li>Make Get/List/Set/Remove xattr return ENOSYS for efficiency</li>
+<li>Make <code>--daemon</code> work for macOS without CGO</li>
+</ul></li>
+<li>VFS
+<ul>
+<li>Add <code>--vfs-read-chunk-size</code> and <code>--vfs-read-chunk-size-limit</code> (Fabian Möller)</li>
+<li>Fix ChangeNotify for new or changed folders (Fabian Möller)</li>
+</ul></li>
+<li>Local</li>
+<li>Fix symlink/junction point directory handling under Windows
+<ul>
+<li><strong>NB</strong> you will need to add <code>-L</code> to your command line to copy files with reparse points</li>
+</ul></li>
+<li>Cache</li>
+<li>Add non cached dirs on notifications (Remus Bunduc)</li>
+<li>Allow root to be expired from rc (Remus Bunduc)</li>
+<li>Clean remaining empty folders from temp upload path (Remus Bunduc)</li>
+<li>Cache lists using batch writes (Remus Bunduc)</li>
+<li>Use secure websockets for HTTPS Plex addresses (John Clayton)</li>
+<li>Reconnect plex websocket on failures (Remus Bunduc)</li>
+<li>Fix panic when running without plex configs (Remus Bunduc)</li>
+<li>Fix root folder caching (Remus Bunduc)</li>
+<li>Crypt</li>
+<li>Check the crypted hash of files when uploading for extra data security</li>
+<li>Dropbox</li>
+<li>Make Dropbox for business folders accessible using an initial <code>/</code> in the path</li>
+<li>Google Cloud Storage</li>
+<li>Low level retry all operations if necessary</li>
+<li>Google Drive</li>
+<li>Add <code>--drive-acknowledge-abuse</code> to download flagged files</li>
+<li>Add <code>--drive-alternate-export</code> to fix large doc export</li>
+<li>Don't attempt to choose Team Drives when using rclone config create</li>
+<li>Fix change list polling with team drives</li>
+<li>Fix ChangeNotify for folders (Fabian Möller)</li>
+<li>Fix about (and df on a mount) for team drives</li>
+<li>Onedrive</li>
+<li>Errorhandler for onedrive for business requests (Henning Surmeier)</li>
+<li>S3</li>
+<li>Adjust upload concurrency with <code>--s3-upload-concurrency</code> (themylogin)</li>
+<li>Fix <code>--s3-chunk-size</code> which was always using the minimum</li>
+<li>SFTP</li>
+<li>Add <code>--ssh-path-override</code> flag (Piotr Oleszczyk)</li>
+<li>Fix slow downloads for long latency connections</li>
+<li>Webdav</li>
+<li>Add workarounds for biz.mail.ru</li>
+<li>Ignore Reason-Phrase in status line to fix 4shared (Rodrigo)</li>
+<li>Better error message generation</li>
+</ul></li>
 <li>v1.41 - 2018-04-28
 <ul>
 <li>New backends</li>
@@ -7881,6 +8176,14 @@ THE SOFTWARE.</code></pre>
 <li>hensur <a href="mailto:me@hensur.de">me@hensur.de</a></li>
 <li>Oliver Heyme <a href="mailto:de8olihe@lego.com">de8olihe@lego.com</a></li>
 <li>Richard Yang <a href="mailto:richard@yenforyang.com">richard@yenforyang.com</a></li>
+<li>Piotr Oleszczyk <a href="mailto:piotr.oleszczyk@gmail.com">piotr.oleszczyk@gmail.com</a></li>
+<li>Rodrigo <a href="mailto:rodarima@gmail.com">rodarima@gmail.com</a></li>
+<li>NoLooseEnds <a href="mailto:NoLooseEnds@users.noreply.github.com">NoLooseEnds@users.noreply.github.com</a></li>
+<li>Jakub Karlicek <a href="mailto:jakub@karlicek.me">jakub@karlicek.me</a></li>
+<li>John Clayton <a href="mailto:john@codemonkeylabs.com">john@codemonkeylabs.com</a></li>
+<li>Kasper Byrdal Nielsen <a href="mailto:byrdal76@gmail.com">byrdal76@gmail.com</a></li>
+<li>Benjamin Joseph Dag <a href="mailto:bjdag1234@users.noreply.github.com">bjdag1234@users.noreply.github.com</a></li>
+<li>themylogin <a href="mailto:themylogin@gmail.com">themylogin@gmail.com</a></li>
 </ul>
 <h1 id="contact-the-rclone-project">Contact the rclone project</h1>
 <h2 id="forum">Forum</h2>

MANUAL.md

@@ -1,6 +1,6 @@
 % rclone(1) User Manual
 % Nick Craig-Wood
-% Apr 28, 2018
+% Jun 16, 2018
 
 Rclone
 ======
@@ -28,8 +28,9 @@ Rclone is a command line program to sync files and directories to and from:
 * Microsoft Azure Blob Storage
 * Microsoft OneDrive
 * Minio
-* Nextloud
+* Nextcloud
 * OVH
+* OpenDrive
 * Openstack Swift
 * Oracle Cloud Storage
 * ownCloud
@@ -203,6 +204,7 @@ See the following for detailed instructions for
   * [Microsoft Azure Blob Storage](https://rclone.org/azureblob/)
   * [Microsoft OneDrive](https://rclone.org/onedrive/)
   * [Openstack Swift / Rackspace Cloudfiles / Memset Memstore](https://rclone.org/swift/)
+  * [OpenDrive](https://rclone.org/opendrive/)
   * [Pcloud](https://rclone.org/pcloud/)
   * [QingStor](https://rclone.org/qingstor/)
   * [SFTP](https://rclone.org/sftp/)
@@ -498,6 +500,10 @@ both remotes and check them against each other on the fly.  This can
 be useful for remotes that don't support hashes or if you really want
 to check all the data.
 
+If you supply the --one-way flag, it will only check that files in source
+match the files in destination, not the other way around. Meaning extra files in
+destination that are not in the source will not trigger an error.
+
 
 ```
 rclone check source:path dest:path [flags]
@@ -508,6 +514,7 @@ rclone check source:path dest:path [flags]
 ```
       --download   Check by downloading rather than with hash.
   -h, --help       help for check
+      --one-way    Check one way only, source files must exist on remote
 ```
 
 ## rclone ls
@@ -1282,6 +1289,10 @@ the files in remote:path.
 
 After it has run it will log the status of the encryptedremote:.
 
+If you supply the --one-way flag, it will only check that files in source
+match the files in destination, not the other way around. Meaning extra files in
+destination that are not in the source will not trigger an error.
+
 
 ```
 rclone cryptcheck remote:path cryptedremote:path [flags]
@@ -1290,7 +1301,8 @@ rclone cryptcheck remote:path cryptedremote:path [flags]
 ### Options
 
 ```
-  -h, --help   help for cryptcheck
+  -h, --help      help for cryptcheck
+      --one-way   Check one way only, source files must exist on destination
 ```
 
 ## rclone cryptdecode
@@ -1346,6 +1358,28 @@ rclone dbhashsum remote:path [flags]
   -h, --help   help for dbhashsum
 ```
 
+## rclone deletefile
+
+Remove a single file path from remote.
+
+### Synopsis
+
+
+Remove a single file path from remote.  Unlike `delete` it cannot be used to
+remove a directory and it doesn't obey include/exclude filters - if the specified file exists,
+it will always be removed.
+
+
+```
+rclone deletefile remote:path [flags]
+```
+
+### Options
+
+```
+  -h, --help   help for deletefile
+```
+
 ## rclone genautocomplete
 
 Output completion script for a given shell.
@@ -1565,6 +1599,8 @@ output:
     s - size
     t - modification time
     h - hash
+    i - ID of object if known
+    m - MimeType of object if known
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -1613,6 +1649,25 @@ Eg
     2018-04-26 08:52:53,0,,ferejej3gux/
     2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic
 
+You can output in CSV standard format.  This will escape things in "
+if they contain ,
+
+Eg
+
+    $ rclone lsf --csv --files-only --format ps remote:path
+    test.log,22355
+    test.sh,449
+    "this file contains a comma, in the file name.txt",6
+
+Note that the --absolute parameter is useful for making lists of files
+to pass to an rclone copy with the --files-from flag.
+
+For example to find all the files modified within one day and copy
+those only (without traversing the whole directory structure):
+
+    rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files
+    rclone copy --files-from new_files /path/to/local remote:path
+
 
 Any of the filtering options can be applied to this commmand.
 
@@ -1644,6 +1699,8 @@ rclone lsf remote:path [flags]
 ### Options
 
 ```
+      --absolute           Put a leading / in front of path names.
+      --csv                Output in CSV format.
   -d, --dir-slash          Append a slash to directory names. (default true)
       --dirs-only          Only list directories.
       --files-only         Only list files.
@@ -1670,7 +1727,9 @@ The output is an array of Items, where each Item looks like this
          "MD5" : "b1946ac92492d2347c6235b4d2611184",
          "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc"
       },
+      "ID": "y2djkhiujf83u33",
       "IsDir" : false,
+      "MimeType" : "application/octet-stream",
       "ModTime" : "2017-05-31T16:15:57.034468261+01:00",
       "Name" : "file.txt",
       "Encrypted" : "v0qpsdq8anpci8n929v3uu9338",
@@ -1866,6 +1925,25 @@ after the mountpoint has been successfully set up.
 Units having the rclone mount service specified as a requirement
 will see all files and folders immediately in this mode.
 
+### chunked reading ###
+
+--vfs-read-chunk-size will enable reading the source objects in parts.
+This can reduce the used download quota for some remotes by requesting only chunks
+from the remote that are actually read at the cost of an increased number of requests.
+
+When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size,
+the chunk size for each open file will get doubled for each chunk read, until the
+specified value is reached. A value of -1 will disable the limit and the chunk size will
+grow indefinitely.
+
+With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
+parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
+When --vfs-read-chunk-size-limit 500M is specified, the result would be
+0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.
+
+Chunked reading will only work with --vfs-cache-mode < full, as the file will always
+be copied to the vfs cache before opening with --vfs-cache-mode full.
+
 ### Directory Cache
 
 Using the `--dir-cache-time` flag, you can set how long a
@@ -2010,6 +2088,9 @@ rclone mount remote:path /path/to/mountpoint [flags]
       --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
       --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
       --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks.
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
+      --volname string                     Set the volume name (not supported by all OSes).
       --write-back-cache                   Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.
 ```
 
@@ -2137,7 +2218,7 @@ Arguments should be passed in as parameter=value.
 
 The result will be returned as a JSON object by default.
 
-Use "rclone rc list" to see a list of all possible commands.
+Use "rclone rc" to see a list of all possible commands.
 
 ```
 rclone rc commands parameter [flags]
@@ -2162,7 +2243,7 @@ rclone rcat reads from standard input (stdin) and copies it to a
 single remote file.
 
     echo "hello world" | rclone rcat remote:path/to/file
-    ffmpeg - | rclone rcat --checksum remote:path/to/file
+    ffmpeg - | rclone rcat remote:path/to/file
 
 If the remote file already exists, it will be overwritten.
 
@@ -2456,6 +2537,8 @@ rclone serve http remote:path [flags]
       --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
       --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
       --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks.
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
 ```
 
 ## rclone serve restic
@@ -2822,6 +2905,8 @@ rclone serve webdav remote:path [flags]
       --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
       --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
       --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+      --vfs-read-chunk-size int            Read the source objects in chunks.
+      --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
 ```
 
 ## rclone touch
@@ -3339,6 +3424,15 @@ files not recursed through are considered excluded and will be deleted
 on the destination.  Test first with `--dry-run` if you are not sure
 what will happen.
 
+### --max-transfer=SIZE ###
+
+Rclone will stop transferring when it has reached the size specified.
+Defaults to off.
+
+When the limit is reached all transfers will stop immediately.
+
+Rclone will exit with exit code 8 if the transfer limit is reached.
+
 ### --modify-window=TIME ###
 
 When checking whether a file has been modified, this is the maximum
@@ -3384,6 +3478,12 @@ files which didn't get transferred because of errors.
 
 Disable retries with `--retries 1`.
 
+### --retries-sleep=TIME ###
+
+This sets the interval between each retry specified by `--retries` 
+
+The default is 0. Use 0 to disable.
+
 ### --size-only ###
 
 Normally rclone will look at modification time and size of files to
@@ -3413,6 +3513,9 @@ show at default log level `NOTICE`.  Use `--stats-log-level NOTICE` or
 `-v` to make them show.  See the [Logging section](#logging) for more
 info on log levels.
 
+Note that on macOS you can send a SIGINFO (which is normally ctrl-T in
+the terminal) to make the stats print immediately.
+
 ### --stats-file-name-length integer ###
 By default, the `--stats` output will truncate file names and paths longer 
 than 40 characters.  This is equivalent to providing 
@@ -3893,6 +3996,7 @@ it will log a high priority message if the retry was successful.
   * `5` - Temporary error (one that more retries might fix) (Retry errors)
   * `6` - Less serious errors (like 461 errors from dropbox) (NoRetry errors)
   * `7` - Fatal error (one that more retries won't fix, like account suspended) (Fatal errors)
+  * `8` - Transfer exceeded - limit set by --max-transfer reached
 
 Environment Variables
 ---------------------
@@ -4591,6 +4695,12 @@ Eg
 The format of the parameter is exactly the same as passed to --bwlimit
 except only one bandwidth may be specified.
 
+### core/gc: Runs a garbage collection.
+
+This tells the go runtime to do a garbage collection run.  It isn't
+necessary to call this normally, but it can be useful for debugging
+memory problems.
+
 ### core/memstats: Returns the memory statistics
 
 This returns the memory statistics of the running program.  What the values mean
@@ -4766,6 +4876,52 @@ curl -H "Content-Type: application/json" -X POST -d '{"potato":2,"sausage":1}' '
 }
 ```
 
+## Debugging rclone with pprof ##
+
+If you use the `--rc` flag this will also enable the use of the go
+profiling tools on the same port.
+
+To use these, first [install go](https://golang.org/doc/install).
+
+Then (for example) to profile rclone's memory use you can run:
+
+    go tool pprof -web http://localhost:5572/debug/pprof/heap
+
+This should open a page in your browser showing what is using what
+memory.
+
+You can also use the `-text` flag to produce a textual summary
+
+```
+$ go tool pprof -text http://localhost:5572/debug/pprof/heap
+Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
+      flat  flat%   sum%        cum   cum%
+ 1024.03kB 66.62% 66.62%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode
+     513kB 33.38%   100%      513kB 33.38%  net/http.newBufioWriterSize
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/all.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve/restic.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init
+         0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0
+         0     0%   100%  1024.03kB 66.62%  main.init
+         0     0%   100%      513kB 33.38%  net/http.(*conn).readRequest
+         0     0%   100%      513kB 33.38%  net/http.(*conn).serve
+         0     0%   100%  1024.03kB 66.62%  runtime.main
+```
+
+Possible profiles to look at:
+
+  * Memory: `go tool pprof http://localhost:5572/debug/pprof/heap`
+  * 30-second CPU profile: `go tool pprof http://localhost:5572/debug/pprof/profile`
+  * 5-second execution trace: `wget http://localhost:5572/debug/pprof/trace?seconds=5`
+
+See the [net/http/pprof docs](https://golang.org/pkg/net/http/pprof/)
+for more info on how to use the profiling and for a general overview
+see [the Go team's blog post on profiling go programs](https://blog.golang.org/profiling-go-programs).
+
+The profiling hook is [zero overhead unless it is used](https://stackoverflow.com/q/26545159/164234).
+
 # Overview of cloud storage systems #
 
 Each cloud storage system is slightly different.  Rclone attempts to
@@ -4791,6 +4947,7 @@ Here is an overview of the major features of each cloud storage system.
 | Mega                         | -           | No      | No               | Yes             | -         |
 | Microsoft Azure Blob Storage | MD5         | Yes     | No               | No              | R/W       |
 | Microsoft OneDrive           | SHA1 ‡‡     | Yes     | Yes              | No              | R         |
+| OpenDrive                    | MD5         | Yes     | Yes              | No              | -         |
 | Openstack Swift              | MD5         | Yes     | No               | No              | R/W       |
 | pCloud                       | MD5, SHA1   | Yes     | No               | No              | W         |
 | QingStor                     | MD5         | No      | No               | No              | R/W       |
@@ -4900,6 +5057,7 @@ operations more efficient.
 | Mega                         | Yes   | No   | Yes  | Yes     | No      | No    | No           | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
 | Microsoft Azure Blob Storage | Yes   | Yes  | No   | No      | No      | Yes   | No           | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
 | Microsoft OneDrive           | Yes   | Yes  | Yes  | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
+| OpenDrive                    | Yes   | Yes  | Yes  | Yes     | No      | No    | No           | No                                                    | No  |
 | Openstack Swift              | Yes † | Yes  | No   | No      | No      | Yes   | Yes          | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
 | pCloud                       | Yes   | Yes  | Yes  | Yes     | Yes     | No    | No           | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
 | QingStor                     | No    | Yes  | No   | No      | No      | Yes   | No           | No [#2178](https://github.com/ncw/rclone/issues/2178) | No  |
@@ -5717,6 +5875,15 @@ Note that 2 chunks of this size are buffered in memory per transfer.
 If you are transferring large files over high speed links and you have
 enough memory, then increasing this will speed up the transfers.
 
+#### --s3-upload-concurrency ####
+
+Number of chunks of the same file that are uploaded concurrently.
+Default is 2.
+
+If you are uploading small amount of large file over high speed link
+and these uploads do not fully utilize your bandwidth, then increasing
+this may help to speed up the transfers.
+
 ### Anonymous access to public buckets ###
 
 If you want to use rclone to access a public bucket, configure with a
@@ -7693,6 +7860,23 @@ To copy a local directory to a dropbox directory called backup
 
     rclone copy /home/source remote:backup
 
+### Dropbox for business ###
+
+Rclone supports Dropbox for business and Team Folders.
+
+When using Dropbox for business `remote:` and `remote:path/to/file`
+will refer to your personal folder.
+
+If you wish to see Team Folders you must use a leading `/` in the
+path, so `rclone lsd remote:/` will refer to the root and show you all
+Team Folders and your User Folder.
+
+You can then use team folders like this `remote:/TeamFolder` and
+`remote:/TeamFolder/path/to/file`.
+
+A leading `/` for a Dropbox personal account will do nothing, but it
+will take an extra HTTP transaction so it should be avoided.
+
 ### Modified time and Hashes ###
 
 Dropbox supports modified times, but the only way to set a
@@ -8440,6 +8624,14 @@ arguments.
 Here are the command line options specific to this cloud storage
 system.
 
+#### --drive-acknowledge-abuse ####
+
+If downloading a file returns the error `This file has been identified
+as malware or spam and cannot be downloaded` with the error code
+`cannotDownloadAbusiveFile` then supply this flag to rclone to
+indicate you acknowledge the risks of downloading the file and rclone
+will download it anyway.
+
 #### --drive-auth-owner-only ####
 
 Only consider files owned by the authenticated user.
@@ -8500,6 +8692,17 @@ Here are the possible extensions with their corresponding mime types.
 | xlsx | application/vnd.openxmlformats-officedocument.spreadsheetml.sheet | Microsoft Office Spreadsheet |
 | zip  | application/zip | A ZIP file of HTML, Images CSS |
 
+#### --drive-alternate-export ####
+
+If this option is set this instructs rclone to use an alternate set of
+export URLs for drive documents.  Users have reported that the
+official export URLs can't export large documents, whereas these
+unofficial ones can.
+
+See rclone issue [#2243](https://github.com/ncw/rclone/issues/2243) for background,
+[this google drive issue](https://issuetracker.google.com/issues/36761333) and
+[this helpful post](https://www.labnol.org/internet/direct-links-for-google-drive/28356/).
+
 #### --drive-impersonate user ####
 
 When using a service account, this instructs rclone to impersonate the user passed in.
@@ -9284,10 +9487,10 @@ Choose OneDrive account type?
  * Say p for a personal OneDrive account
 b) Business
 p) Personal
-b/p> 
+b/p>
 ```
 After that rclone requires an authentication of your account. The application will first authenticate your account, then query the OneDrive resource URL
-and do a second (silent) authentication for this resource URL. 
+and do a second (silent) authentication for this resource URL.
 
 ### Modified time and hashes ###
 
@@ -9334,7 +9537,7 @@ The largest allowed file size is 10GiB (10,737,418,240 bytes).
 ### Versioning issue ###
 
 Every change in OneDrive causes the service to create a new version.
-This counts against a users quota.  
+This counts against a users quota.
 For example changing the modification time of a file creates a second
 version, so the file is using twice the space.
 
@@ -9350,12 +9553,132 @@ versioning on OneDrive
 3. Once on the Site settings page, navigate to Site Administration > Site libraries and lists.
 4. Click Customize "Documents".
 5. Click General Settings > Versioning Settings.
-6. Under Document Version History select the option No versioning.  
+6. Under Document Version History select the option No versioning.
 Note: This will disable the creation of new file versions, but will not remove any previous versions. Your documents are safe.
 7. Apply the changes by clicking OK.
 8. Use rclone to upload or modify files. (I also use the --no-update-modtime flag)
 9. Restore the versioning settings after using rclone. (Optional)
 
+### Troubleshooting ###
+
+```
+Error: access_denied
+Code: AADSTS65005
+Description: Using application 'rclone' is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned.
+```
+
+This means that rclone can't use the OneDrive for Business API with your account. You can't do much about it, maybe write an email to your admins.
+
+However, there are other ways to interact with your OneDrive account. Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint
+
+OpenDrive
+------------------------------------
+
+Paths are specified as `remote:path`
+
+Paths may be as deep as required, eg `remote:directory/subdirectory`.
+
+Here is an example of how to make a remote called `remote`.  First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+```
+n) New remote
+d) Delete remote
+q) Quit config
+e/n/d/q> n
+name> remote
+Type of storage to configure.
+Choose a number from below, or type in your own value
+ 1 / Amazon Drive
+   \ "amazon cloud drive"
+ 2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+   \ "s3"
+ 3 / Backblaze B2
+   \ "b2"
+ 4 / Dropbox
+   \ "dropbox"
+ 5 / Encrypt/Decrypt a remote
+   \ "crypt"
+ 6 / Google Cloud Storage (this is not Google Drive)
+   \ "google cloud storage"
+ 7 / Google Drive
+   \ "drive"
+ 8 / Hubic
+   \ "hubic"
+ 9 / Local Disk
+   \ "local"
+10 / OpenDrive
+   \ "opendrive"
+11 / Microsoft OneDrive
+   \ "onedrive"
+12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+   \ "swift"
+13 / SSH/SFTP Connection
+   \ "sftp"
+14 / Yandex Disk
+   \ "yandex"
+Storage> 10
+Username
+username>
+Password
+y) Yes type in my own password
+g) Generate random password
+y/g> y
+Enter the password:
+password:
+Confirm the password:
+password:
+--------------------
+[remote]
+username =
+password = *** ENCRYPTED ***
+--------------------
+y) Yes this is OK
+e) Edit this remote
+d) Delete this remote
+y/e/d> y
+```
+
+List directories in top level of your OpenDrive
+
+    rclone lsd remote:
+
+List all the files in your OpenDrive
+
+    rclone ls remote:
+
+To copy a local directory to an OpenDrive directory called backup
+
+    rclone copy /home/source remote:backup
+
+### Modified time and MD5SUMs ###
+
+OpenDrive allows modification times to be set on objects accurate to 1
+second. These will be used to detect whether objects need syncing or
+not.
+
+### Deleting files ###
+
+Any files you delete with rclone will end up in the trash. Amazon
+don't provide an API to permanently delete files, nor to empty the
+trash, so you will have to do that with one of Amazon's apps or via
+the OpenDrive website. As of November 17, 2016, files are 
+automatically deleted by Amazon from the trash after 30 days.
+
+### Limitations ###
+
+Note that OpenDrive is case insensitive so you can't have a
+file called "Hello.doc" and one called "hello.doc".
+
+There are quite a few characters that can't be in OpenDrive file
+names.  These can't occur on Windows platforms, but on non-Windows
+platforms they are common.  Rclone will map these names to and from an
+identical looking unicode equivalent.  For example if a file has a `?`
+in it will be mapped to `？` instead.
+
 QingStor
 ---------------------------------------
 
@@ -10086,6 +10409,20 @@ Here are the command line options specific to this remote.
 
 Ask for the SFTP password if needed when no password has been configured.
 
+#### --ssh-path-override ####
+
+Override path used by SSH connection. Allows checksum calculation when
+SFTP and SSH paths are different. This issue affects among others Synology
+NAS boxes.
+
+Shared folders can be found in directories representing volumes
+
+    rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
+
+Home directory can be found in a shared folder called `homes`
+
+    rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory
+
 ### Modified time ###
 
 Modified times are stored on the server to 1 second precision.
@@ -10646,6 +10983,85 @@ points, as you explicitly acknowledge that they should be skipped.
 Changelog
 ---------
 
+  * v1.42 - 2018-06-16
+    * New backends
+      * OpenDrive (Oliver Heyme, Jakub Karlicek, ncw)
+    * New commands
+      * deletefile command (Filip Bartodziej)
+    * New Features
+      * copy, move: Copy single files directly, don't use `--files-from` work-around
+         * this makes them much more efficient
+      * Implement `--max-transfer` flag to quit transferring at a limit
+         * make exit code 8 for `--max-transfer` exceeded
+      * copy: copy empty source directories to destination (Ishuah Kariuki)
+      * check: Add `--one-way` flag (Kasper Byrdal Nielsen)
+      * Add siginfo handler for macOS for ctrl-T stats (kubatasiemski)
+      * rc
+         * add core/gc to run a garbage collection on demand
+         * enable go profiling by default on the `--rc` port
+         * return error from remote on failure
+      * lsf
+         * Add `--absolute` flag to add a leading / onto path names
+         * Add `--csv` flag for compliant CSV output
+         * Add 'm' format specifier to show the MimeType
+         * Implement 'i' format for showing object ID
+      * lsjson
+         * Add MimeType to the output
+         * Add ID field to output to show Object ID
+      * Add `--retries-sleep` flag (Benjamin Joseph Dag)
+      * Oauth tidy up web page and error handling (Henning Surmeier)
+    * Bug Fixes
+      * Password prompt output with `--log-file` fixed for unix (Filip Bartodziej)
+      * Calculate ModifyWindow each time on the fly to fix various problems (Stefan Breunig)
+    * Mount
+      * Only print "File.rename error" if there actually is an error (Stefan Breunig)
+      * Delay rename if file has open writers instead of failing outright (Stefan Breunig)
+      * Ensure atexit gets run on interrupt
+      * macOS enhancements
+         * Make `--noappledouble` `--noapplexattr`
+         * Add `--volname` flag and remove special chars from it
+         * Make Get/List/Set/Remove xattr return ENOSYS for efficiency
+         * Make `--daemon` work for macOS without CGO
+    * VFS
+        * Add `--vfs-read-chunk-size` and `--vfs-read-chunk-size-limit` (Fabian Möller)
+        * Fix ChangeNotify for new or changed folders (Fabian Möller)
+    * Local
+      * Fix symlink/junction point directory handling under Windows
+         * **NB** you will need to add `-L` to your command line to copy files with reparse points
+    * Cache
+      * Add non cached dirs on notifications (Remus Bunduc)
+      * Allow root to be expired from rc (Remus Bunduc)
+      * Clean remaining empty folders from temp upload path (Remus Bunduc)
+      * Cache lists using batch writes (Remus Bunduc)
+      * Use secure websockets for HTTPS Plex addresses (John Clayton)
+      * Reconnect plex websocket on failures (Remus Bunduc)
+      * Fix panic when running without plex configs (Remus Bunduc)
+      * Fix root folder caching (Remus Bunduc)
+    * Crypt
+      * Check the crypted hash of files when uploading for extra data security
+    * Dropbox
+      * Make Dropbox for business folders accessible using an initial `/` in the path
+    * Google Cloud Storage
+      * Low level retry all operations if necessary
+    * Google Drive
+      * Add `--drive-acknowledge-abuse` to download flagged files
+      * Add `--drive-alternate-export` to fix large doc export
+      * Don't attempt to choose Team Drives when using rclone config create
+      * Fix change list polling with team drives
+      * Fix ChangeNotify for folders (Fabian Möller)
+      * Fix about (and df on a mount) for team drives
+    * Onedrive
+      * Errorhandler for onedrive for business requests (Henning Surmeier)
+    * S3
+      * Adjust upload concurrency with `--s3-upload-concurrency` (themylogin)
+      * Fix `--s3-chunk-size` which was always using the minimum
+    * SFTP
+      * Add `--ssh-path-override` flag (Piotr Oleszczyk)
+      * Fix slow downloads for long latency connections
+    * Webdav
+      * Add workarounds for biz.mail.ru
+      * Ignore Reason-Phrase in status line to fix 4shared (Rodrigo)
+      * Better error message generation
   * v1.41 - 2018-04-28
     * New backends
       * Mega support added
@@ -12106,6 +12522,14 @@ Contributors
   * hensur <me@hensur.de>
   * Oliver Heyme <de8olihe@lego.com>
   * Richard Yang <richard@yenforyang.com>
+  * Piotr Oleszczyk <piotr.oleszczyk@gmail.com>
+  * Rodrigo <rodarima@gmail.com>
+  * NoLooseEnds <NoLooseEnds@users.noreply.github.com>
+  * Jakub Karlicek <jakub@karlicek.me>
+  * John Clayton <john@codemonkeylabs.com>
+  * Kasper Byrdal Nielsen <byrdal76@gmail.com>
+  * Benjamin Joseph Dag <bjdag1234@users.noreply.github.com>
+  * themylogin <themylogin@gmail.com>
 
 # Contact the rclone project #
 

MANUAL.txt

@@ -1,6 +1,6 @@
 rclone(1) User Manual
 Nick Craig-Wood
-Apr 28, 2018
+Jun 16, 2018
 
 
 
@@ -31,8 +31,9 @@ from:
 -   Microsoft Azure Blob Storage
 -   Microsoft OneDrive
 -   Minio
--   Nextloud
+-   Nextcloud
 -   OVH
+-   OpenDrive
 -   Openstack Swift
 -   Oracle Cloud Storage
 -   ownCloud
@@ -214,6 +215,7 @@ See the following for detailed instructions for
 -   Microsoft Azure Blob Storage
 -   Microsoft OneDrive
 -   Openstack Swift / Rackspace Cloudfiles / Memset Memstore
+-   OpenDrive
 -   Pcloud
 -   QingStor
 -   SFTP
@@ -472,12 +474,18 @@ remotes and check them against each other on the fly. This can be useful
 for remotes that don't support hashes or if you really want to check all
 the data.
 
+If you supply the --one-way flag, it will only check that files in
+source match the files in destination, not the other way around. Meaning
+extra files in destination that are not in the source will not trigger
+an error.
+
     rclone check source:path dest:path [flags]
 
 Options
 
           --download   Check by downloading rather than with hash.
       -h, --help       help for check
+          --one-way    Check one way only, source files must exist on remote
 
 
 rclone ls
@@ -1160,11 +1168,17 @@ files in remote:path.
 
 After it has run it will log the status of the encryptedremote:.
 
+If you supply the --one-way flag, it will only check that files in
+source match the files in destination, not the other way around. Meaning
+extra files in destination that are not in the source will not trigger
+an error.
+
     rclone cryptcheck remote:path cryptedremote:path [flags]
 
 Options
 
-      -h, --help   help for cryptcheck
+      -h, --help      help for cryptcheck
+          --one-way   Check one way only, source files must exist on destination
 
 
 rclone cryptdecode
@@ -1209,6 +1223,23 @@ Options
       -h, --help   help for dbhashsum
 
 
+rclone deletefile
+
+Remove a single file path from remote.
+
+Synopsis
+
+Remove a single file path from remote. Unlike delete it cannot be used
+to remove a directory and it doesn't obey include/exclude filters - if
+the specified file exists, it will always be removed.
+
+    rclone deletefile remote:path [flags]
+
+Options
+
+      -h, --help   help for deletefile
+
+
 rclone genautocomplete
 
 Output completion script for a given shell.
@@ -1393,6 +1424,8 @@ just the path, but you can use these parameters to control the output:
     s - size
     t - modification time
     h - hash
+    i - ID of object if known
+    m - MimeType of object if known
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -1440,6 +1473,25 @@ Eg
     2018-04-26 08:52:53,0,,ferejej3gux/
     2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic
 
+You can output in CSV standard format. This will escape things in " if
+they contain ,
+
+Eg
+
+    $ rclone lsf --csv --files-only --format ps remote:path
+    test.log,22355
+    test.sh,449
+    "this file contains a comma, in the file name.txt",6
+
+Note that the --absolute parameter is useful for making lists of files
+to pass to an rclone copy with the --files-from flag.
+
+For example to find all the files modified within one day and copy those
+only (without traversing the whole directory structure):
+
+    rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files
+    rclone copy --files-from new_files /path/to/local remote:path
+
 Any of the filtering options can be applied to this commmand.
 
 There are several related list commands
@@ -1467,6 +1519,8 @@ bucket based remotes).
 
 Options
 
+          --absolute           Put a leading / in front of path names.
+          --csv                Output in CSV format.
       -d, --dir-slash          Append a slash to directory names. (default true)
           --dirs-only          Only list directories.
           --files-only         Only list files.
@@ -1490,9 +1544,11 @@ The output is an array of Items, where each Item looks like this
 { "Hashes" : { "SHA-1" : "f572d396fae9206628714fb2ce00f72e94f2258f",
 "MD5" : "b1946ac92492d2347c6235b4d2611184", "DropboxHash" :
 "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc" },
-"IsDir" : false, "ModTime" : "2017-05-31T16:15:57.034468261+01:00",
-"Name" : "file.txt", "Encrypted" : "v0qpsdq8anpci8n929v3uu9338", "Path"
-: "full/path/goes/here/file.txt", "Size" : 6 }
+"ID": "y2djkhiujf83u33", "IsDir" : false, "MimeType" :
+"application/octet-stream", "ModTime" :
+"2017-05-31T16:15:57.034468261+01:00", "Name" : "file.txt", "Encrypted"
+: "v0qpsdq8anpci8n929v3uu9338", "Path" : "full/path/goes/here/file.txt",
+"Size" : 6 }
 
 If --hash is not specified the Hashes property won't be emitted.
 
@@ -1673,6 +1729,29 @@ the mountpoint has been successfully set up. Units having the rclone
 mount service specified as a requirement will see all files and folders
 immediately in this mode.
 
+chunked reading
+
+--vfs-read-chunk-size will enable reading the source objects in parts.
+This can reduce the used download quota for some remotes by requesting
+only chunks from the remote that are actually read at the cost of an
+increased number of requests.
+
+When --vfs-read-chunk-size-limit is also specified and greater than
+--vfs-read-chunk-size, the chunk size for each open file will get
+doubled for each chunk read, until the specified value is reached. A
+value of -1 will disable the limit and the chunk size will grow
+indefinitely.
+
+With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the
+following parts will be downloaded: 0-100M, 100M-200M, 200M-300M,
+300M-400M and so on. When --vfs-read-chunk-size-limit 500M is specified,
+the result would be 0-100M, 100M-300M, 300M-700M, 700M-1200M,
+1200M-1700M and so on.
+
+Chunked reading will only work with --vfs-cache-mode < full, as the file
+will always be copied to the vfs cache before opening with
+--vfs-cache-mode full.
+
 Directory Cache
 
 Using the --dir-cache-time flag, you can set how long a directory should
@@ -1811,6 +1890,9 @@ Options
           --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
           --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
           --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+          --vfs-read-chunk-size int            Read the source objects in chunks.
+          --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
+          --volname string                     Set the volume name (not supported by all OSes).
           --write-back-cache                   Makes kernel buffer writes before sending them to rclone. Without this, writethrough caching is used.
 
 
@@ -1921,7 +2003,7 @@ Arguments should be passed in as parameter=value.
 
 The result will be returned as a JSON object by default.
 
-Use "rclone rc list" to see a list of all possible commands.
+Use "rclone rc" to see a list of all possible commands.
 
     rclone rc commands parameter [flags]
 
@@ -1942,7 +2024,7 @@ rclone rcat reads from standard input (stdin) and copies it to a single
 remote file.
 
     echo "hello world" | rclone rcat remote:path/to/file
-    ffmpeg - | rclone rcat --checksum remote:path/to/file
+    ffmpeg - | rclone rcat remote:path/to/file
 
 If the remote file already exists, it will be overwritten.
 
@@ -2218,6 +2300,8 @@ Options
           --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
           --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
           --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+          --vfs-read-chunk-size int            Read the source objects in chunks.
+          --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
 
 
 rclone serve restic
@@ -2568,6 +2652,8 @@ Options
           --vfs-cache-max-age duration         Max age of objects in the cache. (default 1h0m0s)
           --vfs-cache-mode string              Cache mode off|minimal|writes|full (default "off")
           --vfs-cache-poll-interval duration   Interval to poll the cache for stale objects. (default 1m0s)
+          --vfs-read-chunk-size int            Read the source objects in chunks.
+          --vfs-read-chunk-size-limit int      If greater than --vfs-read-chunk-size, double the chunk size after each chunk read, until the limit is reached. -1 is unlimited.
 
 
 rclone touch
@@ -3074,6 +3160,15 @@ recursed through are considered excluded and will be deleted on the
 destination. Test first with --dry-run if you are not sure what will
 happen.
 
+--max-transfer=SIZE
+
+Rclone will stop transferring when it has reached the size specified.
+Defaults to off.
+
+When the limit is reached all transfers will stop immediately.
+
+Rclone will exit with exit code 8 if the transfer limit is reached.
+
 --modify-window=TIME
 
 When checking whether a file has been modified, this is the maximum
@@ -3118,6 +3213,12 @@ which didn't get transferred because of errors.
 
 Disable retries with --retries 1.
 
+--retries-sleep=TIME
+
+This sets the interval between each retry specified by --retries
+
+The default is 0. Use 0 to disable.
+
 --size-only
 
 Normally rclone will look at modification time and size of files to see
@@ -3144,6 +3245,9 @@ Stats are logged at INFO level by default which means they won't show at
 default log level NOTICE. Use --stats-log-level NOTICE or -v to make
 them show. See the Logging section for more info on log levels.
 
+Note that on macOS you can send a SIGINFO (which is normally ctrl-T in
+the terminal) to make the stats print immediately.
+
 --stats-file-name-length integer
 
 By default, the --stats output will truncate file names and paths longer
@@ -3617,6 +3721,7 @@ List of exit codes
     errors)
 -   7 - Fatal error (one that more retries won't fix, like account
     suspended) (Fatal errors)
+-   8 - Transfer exceeded - limit set by --max-transfer reached
 
 
 Environment Variables
@@ -4329,6 +4434,12 @@ Eg
 The format of the parameter is exactly the same as passed to --bwlimit
 except only one bandwidth may be specified.
 
+core/gc: Runs a garbage collection.
+
+This tells the go runtime to do a garbage collection run. It isn't
+necessary to call this normally, but it can be useful for debugging
+memory problems.
+
 core/memstats: Returns the memory statistics
 
 This returns the memory statistics of the running program. What the
@@ -4477,6 +4588,53 @@ takes precedence.
     }
 
 
+Debugging rclone with pprof
+
+If you use the --rc flag this will also enable the use of the go
+profiling tools on the same port.
+
+To use these, first install go.
+
+Then (for example) to profile rclone's memory use you can run:
+
+    go tool pprof -web http://localhost:5572/debug/pprof/heap
+
+This should open a page in your browser showing what is using what
+memory.
+
+You can also use the -text flag to produce a textual summary
+
+    $ go tool pprof -text http://localhost:5572/debug/pprof/heap
+    Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
+          flat  flat%   sum%        cum   cum%
+     1024.03kB 66.62% 66.62%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.addDecoderNode
+         513kB 33.38%   100%      513kB 33.38%  net/http.newBufioWriterSize
+             0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/all.init
+             0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve.init
+             0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/cmd/serve/restic.init
+             0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2.init
+             0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init
+             0     0%   100%  1024.03kB 66.62%  github.com/ncw/rclone/vendor/golang.org/x/net/http2/hpack.init.0
+             0     0%   100%  1024.03kB 66.62%  main.init
+             0     0%   100%      513kB 33.38%  net/http.(*conn).readRequest
+             0     0%   100%      513kB 33.38%  net/http.(*conn).serve
+             0     0%   100%  1024.03kB 66.62%  runtime.main
+
+Possible profiles to look at:
+
+-   Memory: go tool pprof http://localhost:5572/debug/pprof/heap
+-   30-second CPU profile:
+    go tool pprof http://localhost:5572/debug/pprof/profile
+-   5-second execution trace:
+    wget http://localhost:5572/debug/pprof/trace?seconds=5
+
+See the net/http/pprof docs for more info on how to use the profiling
+and for a general overview see the Go team's blog post on profiling go
+programs.
+
+The profiling hook is zero overhead unless it is used.
+
+
 
 OVERVIEW OF CLOUD STORAGE SYSTEMS
 
@@ -4505,6 +4663,7 @@ Here is an overview of the major features of each cloud storage system.
   Mega                                 -          No             No                Yes             -
   Microsoft Azure Blob Storage        MD5         Yes            No                No             R/W
   Microsoft OneDrive                SHA1 ‡‡       Yes           Yes                No              R
+  OpenDrive                           MD5         Yes           Yes                No              -
   Openstack Swift                     MD5         Yes            No                No             R/W
   pCloud                           MD5, SHA1      Yes            No                No              W
   QingStor                            MD5         No             No                No             R/W
@@ -4612,6 +4771,7 @@ more efficient.
   Mega                             Yes     No    Yes      Yes       No       No          No         No #2178      Yes
   Microsoft Azure Blob Storage     Yes    Yes     No      No        No       Yes         No         No #2178      No
   Microsoft OneDrive               Yes    Yes    Yes    No #197   No #575    No          No         No #2178      Yes
+  OpenDrive                        Yes    Yes    Yes      Yes       No       No          No            No         No
   Openstack Swift                 Yes †   Yes     No      No        No       Yes        Yes         No #2178      Yes
   pCloud                           Yes    Yes    Yes      Yes       Yes      No          No         No #2178      Yes
   QingStor                         No     Yes     No      No        No       Yes         No         No #2178      No
@@ -5417,6 +5577,15 @@ Note that 2 chunks of this size are buffered in memory per transfer.
 If you are transferring large files over high speed links and you have
 enough memory, then increasing this will speed up the transfers.
 
+--s3-upload-concurrency
+
+Number of chunks of the same file that are uploaded concurrently.
+Default is 2.
+
+If you are uploading small amount of large file over high speed link and
+these uploads do not fully utilize your bandwidth, then increasing this
+may help to speed up the transfers.
+
 Anonymous access to public buckets
 
 If you want to use rclone to access a public bucket, configure with a
@@ -7343,6 +7512,23 @@ To copy a local directory to a dropbox directory called backup
 
     rclone copy /home/source remote:backup
 
+Dropbox for business
+
+Rclone supports Dropbox for business and Team Folders.
+
+When using Dropbox for business remote: and remote:path/to/file will
+refer to your personal folder.
+
+If you wish to see Team Folders you must use a leading / in the path, so
+rclone lsd remote:/ will refer to the root and show you all Team Folders
+and your User Folder.
+
+You can then use team folders like this remote:/TeamFolder and
+remote:/TeamFolder/path/to/file.
+
+A leading / for a Dropbox personal account will do nothing, but it will
+take an extra HTTP transaction so it should be avoided.
+
 Modified time and Hashes
 
 Dropbox supports modified times, but the only way to set a modification
@@ -8066,6 +8252,14 @@ Specific options
 
 Here are the command line options specific to this cloud storage system.
 
+--drive-acknowledge-abuse
+
+If downloading a file returns the error
+This file has been identified as malware or spam and cannot be downloaded
+with the error code cannotDownloadAbusiveFile then supply this flag to
+rclone to indicate you acknowledge the risks of downloading the file and
+rclone will download it anyway.
+
 --drive-auth-owner-only
 
 Only consider files owned by the authenticated user.
@@ -8126,6 +8320,16 @@ Here are the possible extensions with their corresponding mime types.
   xlsx                                                                        application/vnd.openxmlformats-officedocument.spreadsheetml.sheet                          Microsoft Office Spreadsheet
   zip                                                                         application/zip                                                                            A ZIP file of HTML, Images CSS
 
+--drive-alternate-export
+
+If this option is set this instructs rclone to use an alternate set of
+export URLs for drive documents. Users have reported that the official
+export URLs can't export large documents, whereas these unofficial ones
+can.
+
+See rclone issue #2243 for background, this google drive issue and this
+helpful post.
+
 --drive-impersonate user
 
 When using a service account, this instructs rclone to impersonate the
@@ -8891,7 +9095,7 @@ ask
      * Say p for a personal OneDrive account
     b) Business
     p) Personal
-    b/p> 
+    b/p>
 
 After that rclone requires an authentication of your account. The
 application will first authenticate your account, then query the
@@ -8941,9 +9145,9 @@ The largest allowed file size is 10GiB (10,737,418,240 bytes).
 Versioning issue
 
 Every change in OneDrive causes the service to create a new version.
-This counts against a users quota.
-For example changing the modification time of a file creates a second
-version, so the file is using twice the space.
+This counts against a users quota. For example changing the modification
+time of a file creates a second version, so the file is using twice the
+space.
 
 The copy is the only rclone command affected by this as we copy the file
 and then afterwards set the modification time to match the source file.
@@ -8965,6 +9169,125 @@ User Weropol has found a method to disable versioning on OneDrive
     --no-update-modtime flag)
 9.  Restore the versioning settings after using rclone. (Optional)
 
+Troubleshooting
+
+    Error: access_denied
+    Code: AADSTS65005
+    Description: Using application 'rclone' is currently not supported for your organization [YOUR_ORGANIZATION] because it is in an unmanaged state. An administrator needs to claim ownership of the company by DNS validation of [YOUR_ORGANIZATION] before the application rclone can be provisioned.
+
+This means that rclone can't use the OneDrive for Business API with your
+account. You can't do much about it, maybe write an email to your
+admins.
+
+However, there are other ways to interact with your OneDrive account.
+Have a look at the webdav backend: https://rclone.org/webdav/#sharepoint
+
+
+OpenDrive
+
+Paths are specified as remote:path
+
+Paths may be as deep as required, eg remote:directory/subdirectory.
+
+Here is an example of how to make a remote called remote. First run:
+
+     rclone config
+
+This will guide you through an interactive setup process:
+
+    n) New remote
+    d) Delete remote
+    q) Quit config
+    e/n/d/q> n
+    name> remote
+    Type of storage to configure.
+    Choose a number from below, or type in your own value
+     1 / Amazon Drive
+       \ "amazon cloud drive"
+     2 / Amazon S3 (also Dreamhost, Ceph, Minio)
+       \ "s3"
+     3 / Backblaze B2
+       \ "b2"
+     4 / Dropbox
+       \ "dropbox"
+     5 / Encrypt/Decrypt a remote
+       \ "crypt"
+     6 / Google Cloud Storage (this is not Google Drive)
+       \ "google cloud storage"
+     7 / Google Drive
+       \ "drive"
+     8 / Hubic
+       \ "hubic"
+     9 / Local Disk
+       \ "local"
+    10 / OpenDrive
+       \ "opendrive"
+    11 / Microsoft OneDrive
+       \ "onedrive"
+    12 / Openstack Swift (Rackspace Cloud Files, Memset Memstore, OVH)
+       \ "swift"
+    13 / SSH/SFTP Connection
+       \ "sftp"
+    14 / Yandex Disk
+       \ "yandex"
+    Storage> 10
+    Username
+    username>
+    Password
+    y) Yes type in my own password
+    g) Generate random password
+    y/g> y
+    Enter the password:
+    password:
+    Confirm the password:
+    password:
+    --------------------
+    [remote]
+    username =
+    password = *** ENCRYPTED ***
+    --------------------
+    y) Yes this is OK
+    e) Edit this remote
+    d) Delete this remote
+    y/e/d> y
+
+List directories in top level of your OpenDrive
+
+    rclone lsd remote:
+
+List all the files in your OpenDrive
+
+    rclone ls remote:
+
+To copy a local directory to an OpenDrive directory called backup
+
+    rclone copy /home/source remote:backup
+
+Modified time and MD5SUMs
+
+OpenDrive allows modification times to be set on objects accurate to 1
+second. These will be used to detect whether objects need syncing or
+not.
+
+Deleting files
+
+Any files you delete with rclone will end up in the trash. Amazon don't
+provide an API to permanently delete files, nor to empty the trash, so
+you will have to do that with one of Amazon's apps or via the OpenDrive
+website. As of November 17, 2016, files are automatically deleted by
+Amazon from the trash after 30 days.
+
+Limitations
+
+Note that OpenDrive is case insensitive so you can't have a file called
+"Hello.doc" and one called "hello.doc".
+
+There are quite a few characters that can't be in OpenDrive file names.
+These can't occur on Windows platforms, but on non-Windows platforms
+they are common. Rclone will map these names to and from an identical
+looking unicode equivalent. For example if a file has a ? in it will be
+mapped to ？ instead.
+
 
 QingStor
 
@@ -9680,6 +10003,20 @@ Here are the command line options specific to this remote.
 Ask for the SFTP password if needed when no password has been
 configured.
 
+--ssh-path-override
+
+Override path used by SSH connection. Allows checksum calculation when
+SFTP and SSH paths are different. This issue affects among others
+Synology NAS boxes.
+
+Shared folders can be found in directories representing volumes
+
+    rclone sync /home/local/directory remote:/directory --ssh-path-override /volume2/directory
+
+Home directory can be found in a shared folder called homes
+
+    rclone sync /home/local/directory remote:/home/directory --ssh-path-override /volume1/homes/USER/directory
+
 Modified time
 
 Modified times are stored on the server to 1 second precision.
@@ -10211,6 +10548,99 @@ points, as you explicitly acknowledge that they should be skipped.
 
 Changelog
 
+-   v1.42 - 2018-06-16
+    -   New backends
+    -   OpenDrive (Oliver Heyme, Jakub Karlicek, ncw)
+    -   New commands
+    -   deletefile command (Filip Bartodziej)
+    -   New Features
+    -   copy, move: Copy single files directly, don't use --files-from
+        work-around
+        -   this makes them much more efficient
+    -   Implement --max-transfer flag to quit transferring at a limit
+        -   make exit code 8 for --max-transfer exceeded
+    -   copy: copy empty source directories to destination (Ishuah
+        Kariuki)
+    -   check: Add --one-way flag (Kasper Byrdal Nielsen)
+    -   Add siginfo handler for macOS for ctrl-T stats (kubatasiemski)
+    -   rc
+        -   add core/gc to run a garbage collection on demand
+        -   enable go profiling by default on the --rc port
+        -   return error from remote on failure
+    -   lsf
+        -   Add --absolute flag to add a leading / onto path names
+        -   Add --csv flag for compliant CSV output
+        -   Add 'm' format specifier to show the MimeType
+        -   Implement 'i' format for showing object ID
+    -   lsjson
+        -   Add MimeType to the output
+        -   Add ID field to output to show Object ID
+    -   Add --retries-sleep flag (Benjamin Joseph Dag)
+    -   Oauth tidy up web page and error handling (Henning Surmeier)
+    -   Bug Fixes
+    -   Password prompt output with --log-file fixed for unix (Filip
+        Bartodziej)
+    -   Calculate ModifyWindow each time on the fly to fix various
+        problems (Stefan Breunig)
+    -   Mount
+    -   Only print "File.rename error" if there actually is an error
+        (Stefan Breunig)
+    -   Delay rename if file has open writers instead of failing
+        outright (Stefan Breunig)
+    -   Ensure atexit gets run on interrupt
+    -   macOS enhancements
+        -   Make --noappledouble --noapplexattr
+        -   Add --volname flag and remove special chars from it
+        -   Make Get/List/Set/Remove xattr return ENOSYS for efficiency
+        -   Make --daemon work for macOS without CGO
+    -   VFS
+        -   Add --vfs-read-chunk-size and --vfs-read-chunk-size-limit
+            (Fabian Möller)
+        -   Fix ChangeNotify for new or changed folders (Fabian Möller)
+    -   Local
+    -   Fix symlink/junction point directory handling under Windows
+        -   NB you will need to add -L to your command line to copy
+            files with reparse points
+    -   Cache
+    -   Add non cached dirs on notifications (Remus Bunduc)
+    -   Allow root to be expired from rc (Remus Bunduc)
+    -   Clean remaining empty folders from temp upload path (Remus
+        Bunduc)
+    -   Cache lists using batch writes (Remus Bunduc)
+    -   Use secure websockets for HTTPS Plex addresses (John Clayton)
+    -   Reconnect plex websocket on failures (Remus Bunduc)
+    -   Fix panic when running without plex configs (Remus Bunduc)
+    -   Fix root folder caching (Remus Bunduc)
+    -   Crypt
+    -   Check the crypted hash of files when uploading for extra data
+        security
+    -   Dropbox
+    -   Make Dropbox for business folders accessible using an initial /
+        in the path
+    -   Google Cloud Storage
+    -   Low level retry all operations if necessary
+    -   Google Drive
+    -   Add --drive-acknowledge-abuse to download flagged files
+    -   Add --drive-alternate-export to fix large doc export
+    -   Don't attempt to choose Team Drives when using rclone config
+        create
+    -   Fix change list polling with team drives
+    -   Fix ChangeNotify for folders (Fabian Möller)
+    -   Fix about (and df on a mount) for team drives
+    -   Onedrive
+    -   Errorhandler for onedrive for business requests (Henning
+        Surmeier)
+    -   S3
+    -   Adjust upload concurrency with --s3-upload-concurrency
+        (themylogin)
+    -   Fix --s3-chunk-size which was always using the minimum
+    -   SFTP
+    -   Add --ssh-path-override flag (Piotr Oleszczyk)
+    -   Fix slow downloads for long latency connections
+    -   Webdav
+    -   Add workarounds for biz.mail.ru
+    -   Ignore Reason-Phrase in status line to fix 4shared (Rodrigo)
+    -   Better error message generation
 -   v1.41 - 2018-04-28
     -   New backends
     -   Mega support added
@@ -11838,6 +12268,14 @@ Contributors
 -   hensur me@hensur.de
 -   Oliver Heyme de8olihe@lego.com
 -   Richard Yang richard@yenforyang.com
+-   Piotr Oleszczyk piotr.oleszczyk@gmail.com
+-   Rodrigo rodarima@gmail.com
+-   NoLooseEnds NoLooseEnds@users.noreply.github.com
+-   Jakub Karlicek jakub@karlicek.me
+-   John Clayton john@codemonkeylabs.com
+-   Kasper Byrdal Nielsen byrdal76@gmail.com
+-   Benjamin Joseph Dag bjdag1234@users.noreply.github.com
+-   themylogin themylogin@gmail.com
 
 
 

Makefile

@@ -1,12 +1,22 @@
 SHELL = bash
-TAG := $(shell echo $$(git describe --abbrev=8 --tags)-$${APPVEYOR_REPO_BRANCH:-$${TRAVIS_BRANCH:-$$(git rev-parse --abbrev-ref HEAD)}} | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/; s/-\(HEAD\|master\)$$//')
+BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))
+TAG_BRANCH := -$(BRANCH)
+BRANCH_PATH := branch/
+ifeq ($(subst HEAD,,$(subst master,,$(BRANCH))),)
+	TAG_BRANCH :=
+	BRANCH_PATH :=
+endif
+TAG := $(shell echo $$(git describe --abbrev=8 --tags | sed 's/-\([0-9]\)-/-00\1-/; s/-\([0-9][0-9]\)-/-0\1-/'))$(TAG_BRANCH)
 LAST_TAG := $(shell git describe --tags --abbrev=0)
 NEW_TAG := $(shell echo $(LAST_TAG) | perl -lpe 's/v//; $$_ += 0.01; $$_ = sprintf("v%.2f", $$_)')
 GO_VERSION := $(shell go version)
 GO_FILES := $(shell go list ./... | grep -v /vendor/ )
 # Run full tests if go >= go1.9
 FULL_TESTS := $(shell go version | perl -lne 'print "go$$1.$$2" if /go(\d+)\.(\d+)/ && ($$1 > 1 || $$2 >= 9)')
-BETA_URL := https://beta.rclone.org/$(TAG)/
+BETA_PATH := $(BRANCH_PATH)$(TAG)
+BETA_URL := https://beta.rclone.org/$(BETA_PATH)/
+BETA_UPLOAD_ROOT := memstore:beta-rclone-org
+BETA_UPLOAD := $(BETA_UPLOAD_ROOT)/$(BETA_PATH)
 # Pass in GOTAGS=xyz on the make command line to set build tags
 ifdef GOTAGS
 BUILDTAGS=-tags "$(GOTAGS)"
@@ -21,6 +31,7 @@ rclone:
 
 vars:
 	@echo SHELL="'$(SHELL)'"
+	@echo BRANCH="'$(BRANCH)'"
 	@echo TAG="'$(TAG)'"
 	@echo LAST_TAG="'$(LAST_TAG)'"
 	@echo NEW_TAG="'$(NEW_TAG)'"
@@ -56,19 +67,29 @@ else
 	@echo Skipping source quality tests as version of go too old
 endif
 
+gometalinter_install:
+	go get -u github.com/alecthomas/gometalinter
+	gometalinter --install --update
+
+# We aren't using gometalinter as the default linter yet because
+# 1. it doesn't support build tags: https://github.com/alecthomas/gometalinter/issues/275
+# 2. can't get -printfuncs working with the vet linter
+gometalinter:
+	gometalinter ./...
+
 # Get the build dependencies
 build_dep:
 ifdef FULL_TESTS
 	go get -u github.com/kisielk/errcheck
 	go get -u golang.org/x/tools/cmd/goimports
 	go get -u github.com/golang/lint/golint
-	go get -u github.com/inconshreveable/mousetrap
 	go get -u github.com/tools/godep
 endif
 
 # Get the release dependencies
 release_dep:
-	go get -u github.com/goreleaser/nfpm
+	go get -u github.com/goreleaser/nfpm/...
+	go get -u github.com/aktau/github-release
 
 # Update dependencies
 update:
@@ -115,9 +136,9 @@ tarball:
 	git archive -9 --format=tar.gz --prefix=rclone-$(TAG)/ -o build/rclone-$(TAG).tar.gz $(TAG)
 
 sign_upload:
-	cd build && md5sum rclone-* | gpg --clearsign > MD5SUMS
-	cd build && sha1sum rclone-* | gpg --clearsign > SHA1SUMS
-	cd build && sha256sum rclone-* | gpg --clearsign > SHA256SUMS
+	cd build && md5sum rclone-v* | gpg --clearsign > MD5SUMS
+	cd build && sha1sum rclone-v* | gpg --clearsign > SHA1SUMS
+	cd build && sha256sum rclone-v* | gpg --clearsign > SHA256SUMS
 
 check_sign:
 	cd build && gpg --verify MD5SUMS && gpg --decrypt MD5SUMS | md5sum -c
@@ -150,25 +171,32 @@ else
 endif
 
 appveyor_upload:
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ memstore:beta-rclone-org/$(TAG)
-ifeq ($(APPVEYOR_REPO_BRANCH),master)
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ memstore:beta-rclone-org
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
+BUILD_FLAGS := -exclude "^(windows|darwin)/"
+ifeq ($(TRAVIS_OS_NAME),osx)
+	BUILD_FLAGS := -include "^darwin/" -cgo
+endif
+
 travis_beta:
+ifeq ($(TRAVIS_OS_NAME),linux)
 	go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
+endif
 	git log $(LAST_TAG).. > /tmp/git-log.txt
-	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt -exclude "^windows/" -parallel 8 $(BUILDTAGS) $(TAG)β
-	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ memstore:beta-rclone-org/$(TAG)
-ifeq ($(TRAVIS_BRANCH),master)
-	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ memstore:beta-rclone-org
+	go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) -parallel 8 $(BUILDTAGS) $(TAG)β
+	rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
+ifndef BRANCH_PATH
+	rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
 endif
 	@echo Beta release ready at $(BETA_URL)
 
 # Fetch the windows builds from appveyor
 fetch_windows:
-	rclone -v copy --include 'rclone-v*-windows-*.zip' memstore:beta-rclone-org/$(TAG) build/
+	rclone -v copy --include 'rclone-v*-windows-*.zip' $(BETA_UPLOAD) build/
 	-#cp -av build/rclone-v*-windows-386.zip build/rclone-current-windows-386.zip
 	-#cp -av build/rclone-v*-windows-amd64.zip build/rclone-current-windows-amd64.zip
 	md5sum build/rclone-*-windows-*.zip | sort

README.md

@@ -28,6 +28,7 @@ Rclone is a command line program to sync files and directories to and from
   * Mega
   * Microsoft Azure Blob Storage
   * Microsoft OneDrive
+  * OpenDrive
   * Openstack Swift / Rackspace cloud files / Memset Memstore / OVH / Oracle Cloud Storage
   * pCloud
   * QingStor

RELEASE.md

@@ -31,6 +31,7 @@ Making a release
   * # announce with forum post, twitter post, G+ post
 
 Early in the next release cycle update the vendored dependencies
+  * Review any pinned packages in Gopkg.toml and remove if possible
   * make update
   * git status
   * git add new files

backend/alias/alias_internal_test.go

@@ -15,8 +15,6 @@ import (
 
 var (
 	remoteName = "TestAlias"
-	testPath   = "test"
-	filesPath  = filepath.Join(testPath, "files")
 )
 
 func prepare(t *testing.T, root string) {

backend/all/all.go

@@ -18,6 +18,7 @@ import (
 	_ "github.com/ncw/rclone/backend/local"
 	_ "github.com/ncw/rclone/backend/mega"
 	_ "github.com/ncw/rclone/backend/onedrive"
+	_ "github.com/ncw/rclone/backend/opendrive"
 	_ "github.com/ncw/rclone/backend/pcloud"
 	_ "github.com/ncw/rclone/backend/qingstor"
 	_ "github.com/ncw/rclone/backend/s3"

backend/amazonclouddrive/amazonclouddrive.go

@@ -18,7 +18,6 @@ import (
 	"log"
 	"net/http"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -40,7 +39,6 @@ import (
 const (
 	folderKind      = "FOLDER"
 	fileKind        = "FILE"
-	assetKind       = "ASSET"
 	statusAvailable = "AVAILABLE"
 	timeFormat      = time.RFC3339 // 2014-03-07T22:31:12.173Z
 	minSleep        = 20 * time.Millisecond
@@ -138,9 +136,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a acd path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an acd 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -1320,6 +1315,14 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), checkpoin
 	return checkpoint
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	if o.info.Id == nil {
+		return ""
+	}
+	return *o.info.Id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs     = (*Fs)(nil)
@@ -1331,4 +1334,5 @@ var (
 	_ fs.ChangeNotifier  = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = &Object{}
+	_ fs.IDer            = &Object{}
 )

backend/azureblob/azureblob.go

@@ -8,9 +8,9 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"fmt"
-	gohash "hash"
 	"io"
 	"net/http"
+	"net/url"
 	"path"
 	"regexp"
 	"strconv"
@@ -32,16 +32,16 @@ import (
 )
 
 const (
-	apiVersion         = "2017-04-17"
-	minSleep           = 10 * time.Millisecond
-	maxSleep           = 10 * time.Second
-	decayConstant      = 1    // bigger for slower decay, exponential
-	listChunkSize      = 5000 // number of items to read at once
-	modTimeKey         = "mtime"
-	timeFormatIn       = time.RFC3339
-	timeFormatOut      = "2006-01-02T15:04:05.000000000Z07:00"
-	maxTotalParts      = 50000   // in multipart upload
-	maxUncommittedSize = 9 << 30 // can't upload bigger than this
+	apiVersion    = "2017-04-17"
+	minSleep      = 10 * time.Millisecond
+	maxSleep      = 10 * time.Second
+	decayConstant = 1    // bigger for slower decay, exponential
+	listChunkSize = 5000 // number of items to read at once
+	modTimeKey    = "mtime"
+	timeFormatIn  = time.RFC3339
+	timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
+	maxTotalParts = 50000 // in multipart upload
+	// maxUncommittedSize = 9 << 30 // can't upload bigger than this
 )
 
 // Globals
@@ -60,10 +60,16 @@ func init() {
 		NewFs:       NewFs,
 		Options: []fs.Option{{
 			Name: "account",
-			Help: "Storage Account Name",
+			Help: "Storage Account Name (leave blank to use connection string or SAS URL)",
 		}, {
 			Name: "key",
-			Help: "Storage Account Key",
+			Help: "Storage Account Key (leave blank to use connection string or SAS URL)",
+		}, {
+			Name: "connection_string",
+			Help: "Connection string (leave blank if using account/key or SAS URL)",
+		}, {
+			Name: "sas_url",
+			Help: "SAS URL for container level access only\n(leave blank if using account/key or connection string)",
 		}, {
 			Name: "endpoint",
 			Help: "Endpoint for the service - leave blank normally.",
@@ -80,7 +86,6 @@ type Fs struct {
 	root             string       // the path we are working on if any
 	features         *fs.Features // optional features
 	account          string       // account name
-	key              []byte       // auth key
 	endpoint         string       // name of the starting api endpoint
 	bc               *storage.BlobStorageClient
 	cc               *storage.Container
@@ -96,7 +101,6 @@ type Fs struct {
 type Object struct {
 	fs       *Fs               // what this object is part of
 	remote   string            // The remote path
-	id       string            // azure id of the file
 	modTime  time.Time         // The modified time of the object if known
 	md5      string            // MD5 hash if known
 	size     int64             // Size of the object
@@ -185,36 +189,54 @@ func NewFs(name, root string) (fs.Fs, error) {
 		return nil, err
 	}
 	account := config.FileGet(name, "account")
-	if account == "" {
-		return nil, errors.New("account not found")
-	}
 	key := config.FileGet(name, "key")
-	if key == "" {
-		return nil, errors.New("key not found")
-	}
-	keyBytes, err := base64.StdEncoding.DecodeString(key)
-	if err != nil {
-		return nil, errors.Errorf("malformed storage account key: %v", err)
-	}
-
+	connectionString := config.FileGet(name, "connection_string")
+	sasURL := config.FileGet(name, "sas_url")
 	endpoint := config.FileGet(name, "endpoint", storage.DefaultBaseURL)
 
-	client, err := storage.NewClient(account, key, endpoint, apiVersion, true)
-	if err != nil {
-		return nil, errors.Wrap(err, "failed to make azure storage client")
+	var (
+		oclient storage.Client
+		client  = &oclient
+		cc      *storage.Container
+	)
+	switch {
+	case account != "" && key != "":
+		oclient, err = storage.NewClient(account, key, endpoint, apiVersion, true)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to make azure storage client from account/key")
+		}
+	case connectionString != "":
+		oclient, err = storage.NewClientFromConnectionString(connectionString)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to make azure storage client from connection string")
+		}
+	case sasURL != "":
+		URL, err := url.Parse(sasURL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to parse SAS URL")
+		}
+		cc, err = storage.GetContainerReferenceFromSASURI(*URL)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to make azure storage client from SAS URL")
+		}
+		client = cc.Client()
+	default:
+		return nil, errors.New("Need account+key or connectionString or sasURL")
 	}
 	client.HTTPClient = fshttp.NewClient(fs.Config)
 	bc := client.GetBlobService()
+	if cc == nil {
+		cc = bc.GetContainerReference(container)
+	}
 
 	f := &Fs{
 		name:        name,
 		container:   container,
 		root:        directory,
 		account:     account,
-		key:         keyBytes,
 		endpoint:    endpoint,
 		bc:          &bc,
-		cc:          bc.GetContainerReference(container),
+		cc:          cc,
 		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
 	}
@@ -551,10 +573,19 @@ func (f *Fs) Mkdir(dir string) error {
 	if f.containerOK {
 		return nil
 	}
+	// List the container to see if it exists
+	err := f.list("", false, 1, func(remote string, object *storage.Blob, isDirectory bool) error {
+		return nil
+	})
+	if err == nil {
+		f.markContainerOK()
+		return nil
+	}
+	// now try to create the container
 	options := storage.CreateContainerOptions{
 		Access: storage.ContainerAccessTypePrivate,
 	}
-	err := f.pacer.Call(func() (bool, error) {
+	err = f.pacer.Call(func() (bool, error) {
 		err := f.cc.Create(&options)
 		if err != nil {
 			if storageErr, ok := err.(storage.AzureStorageServiceError); ok {
@@ -847,16 +878,6 @@ func (o *Object) Storable() bool {
 	return true
 }
 
-// openFile represents an Object open for reading
-type openFile struct {
-	o     *Object        // Object we are reading for
-	resp  *http.Response // response of the GET
-	body  io.Reader      // reading from here
-	hash  gohash.Hash    // currently accumulating MD5
-	bytes int64          // number of bytes read on this connection
-	eof   bool           // whether we have read end of file
-}
-
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	getBlobOptions := storage.GetBlobOptions{}
@@ -924,20 +945,6 @@ func init() {
 	}
 }
 
-// urlEncode encodes in with % encoding
-func urlEncode(in string) string {
-	var out bytes.Buffer
-	for i := 0; i < len(in); i++ {
-		c := in[i]
-		if noNeedToEncode[c] {
-			_ = out.WriteByte(c)
-		} else {
-			_, _ = out.WriteString(fmt.Sprintf("%%%2X", c))
-		}
-	}
-	return out.String()
-}
-
 // uploadMultipart uploads a file using multipart upload
 //
 // Write a larger blob, using CreateBlockBlob, PutBlock, and PutBlockList.

backend/b2/b2.go

@@ -1422,6 +1422,11 @@ func (o *Object) MimeType() string {
 	return o.mimeType
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs          = &Fs{}
@@ -1431,4 +1436,5 @@ var (
 	_ fs.ListRer     = &Fs{}
 	_ fs.Object      = &Object{}
 	_ fs.MimeTyper   = &Object{}
+	_ fs.IDer        = &Object{}
 )

backend/b2/upload.go

@@ -357,7 +357,8 @@ outer:
 		buf := up.f.getUploadBlock()
 
 		// Read the chunk
-		n, err := io.ReadFull(up.in, buf)
+		var n int
+		n, err = io.ReadFull(up.in, buf)
 		if err == io.ErrUnexpectedEOF {
 			fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
 			buf = buf[:n]
@@ -366,7 +367,6 @@ outer:
 		} else if err == io.EOF {
 			fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
 			up.f.putUploadBlock(buf)
-			hasMoreParts = false
 			err = nil
 			break outer
 		} else if err != nil {

backend/box/box.go

@@ -16,7 +16,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -135,9 +134,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a box path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an box 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -1052,6 +1048,11 @@ func (o *Object) Remove() error {
 	return o.fs.deleteObject(o.id)
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1062,4 +1063,5 @@ var (
 	_ fs.DirMover        = (*Fs)(nil)
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/cache/cache.go

@@ -473,18 +473,20 @@ func (f *Fs) httpExpireRemote(in rc.Params) (out rc.Params, err error) {
 		withData = true
 	}
 
-	// if it's wrapped by crypt we need to check what format we got
-	if cryptFs, yes := f.isWrappedByCrypt(); yes {
-		_, err := cryptFs.DecryptFileName(remote)
-		// if it failed to decrypt then it is a decrypted format and we need to encrypt it
-		if err != nil {
-			remote = cryptFs.EncryptFileName(remote)
+	if cleanPath(remote) != "" {
+		// if it's wrapped by crypt we need to check what format we got
+		if cryptFs, yes := f.isWrappedByCrypt(); yes {
+			_, err := cryptFs.DecryptFileName(remote)
+			// if it failed to decrypt then it is a decrypted format and we need to encrypt it
+			if err != nil {
+				remote = cryptFs.EncryptFileName(remote)
+			}
+			// else it's an encrypted format and we can use it as it is
 		}
-		// else it's an encrypted format and we can use it as it is
-	}
 
-	if !f.cache.HasEntry(path.Join(f.Root(), remote)) {
-		return out, errors.Errorf("%s doesn't exist in cache", remote)
+		if !f.cache.HasEntry(path.Join(f.Root(), remote)) {
+			return out, errors.Errorf("%s doesn't exist in cache", remote)
+		}
 	}
 
 	co := NewObject(f, remote)
@@ -533,16 +535,12 @@ func (f *Fs) receiveChangeNotify(forgetPath string, entryType fs.EntryType) {
 	if entryType == fs.EntryObject {
 		co := NewObject(f, forgetPath)
 		err := f.cache.GetObject(co)
-		if err == nil {
-			// expire the entry
-			err = f.cache.ExpireObject(co, true)
-			if err != nil {
-				fs.Debugf(forgetPath, "notify: error expiring '%v': %v", co, err)
-			} else {
-				fs.Debugf(forgetPath, "notify: expired %v", co)
-			}
-		} else {
-			fs.Debugf(f, "ignoring change notification for non cached entry %v", co)
+		if err != nil {
+			fs.Debugf(f, "got change notification for non cached entry %v", co)
+		}
+		err = f.cache.ExpireObject(co, true)
+		if err != nil {
+			fs.Debugf(forgetPath, "notify: error expiring '%v': %v", co, err)
 		}
 		cd = NewDirectory(f, cleanPath(path.Dir(co.Remote())))
 	} else {
@@ -647,7 +645,6 @@ func (f *Fs) NewObject(remote string) (fs.Object, error) {
 
 	// search for entry in source or temp fs
 	var obj fs.Object
-	err = nil
 	if f.tempWritePath != "" {
 		obj, err = f.tempFs.NewObject(remote)
 		// not found in temp fs
@@ -726,6 +723,7 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	fs.Debugf(dir, "list: source entries: %v", entries)
 
 	// and then iterate over the ones from source (temp Objects will override source ones)
+	var batchDirectories []*Directory
 	for _, entry := range entries {
 		switch o := entry.(type) {
 		case fs.Object:
@@ -747,18 +745,19 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			cdd := DirectoryFromOriginal(f, o)
 			// check if the dir isn't expired and add it in cache if it isn't
 			if cdd2, err := f.cache.GetDir(cdd.abs()); err != nil || time.Now().Before(cdd2.CacheTs.Add(f.fileAge)) {
-				err := f.cache.AddDir(cdd)
-				if err != nil {
-					fs.Errorf(dir, "list: error caching dir from listing %v", o)
-				} else {
-					fs.Debugf(dir, "list: cached dir: %v", cdd)
-				}
+				batchDirectories = append(batchDirectories, cdd)
 			}
 			cachedEntries = append(cachedEntries, cdd)
 		default:
 			fs.Debugf(entry, "list: Unknown object type %T", entry)
 		}
 	}
+	err = f.cache.AddBatchDir(batchDirectories)
+	if err != nil {
+		fs.Errorf(dir, "list: error caching directories from listing %v", dir)
+	} else {
+		fs.Debugf(dir, "list: cached directories: %v", len(batchDirectories))
+	}
 
 	// cache dir meta
 	t := time.Now()
@@ -1168,8 +1167,14 @@ func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put p
 	}
 
 	// cache the new file
-	cachedObj := ObjectFromOriginal(f, obj).persist()
+	cachedObj := ObjectFromOriginal(f, obj)
+
+	// deleting cached chunks and info to be replaced with new ones
+	_ = f.cache.RemoveObject(cachedObj.abs())
+
+	cachedObj.persist()
 	fs.Debugf(cachedObj, "put: added to cache")
+
 	// expire parent
 	parentCd := NewDirectory(f, cleanPath(path.Dir(cachedObj.Remote())))
 	err = f.cache.ExpireDir(parentCd)

backend/cache/cache_internal_test.go

@@ -50,6 +50,7 @@ const (
 	cryptedTextBase64  = "UkNMT05FAAC320i2xIee0BiNyknSPBn+Qcw3q9FhIFp3tvq6qlqvbsno3PnxmEFeJG3jDBnR/wku2gHWeQ=="     // one content
 	cryptedText2Base64 = "UkNMT05FAAATcQkVsgjBh8KafCKcr0wdTa1fMmV0U8hsCLGFoqcvxKVmvv7wx3Hf5EXxFcki2FFV4sdpmSrb9Q==" // updated content
 	cryptedText3Base64 = "UkNMT05FAAB/f7YtYKbPfmk9+OX/ffN3qG3OEdWT+z74kxCX9V/YZwJ4X2DN3HOnUC3gKQ4Gcoud5UtNvQ=="     // test content
+	letterBytes        = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 )
 
 var (
@@ -132,6 +133,7 @@ func TestInternalListRootAndInnerRemotes(t *testing.T) {
 	require.Len(t, listInner, 1)
 }
 
+/* TODO: is this testing something?
 func TestInternalVfsCache(t *testing.T) {
 	vfsflags.Opt.DirCacheTime = time.Second * 30
 	testSize := int64(524288000)
@@ -229,6 +231,7 @@ func TestInternalVfsCache(t *testing.T) {
 	cacheCh <- true
 	readCh <- true
 }
+*/
 
 func TestInternalObjWrapFsFound(t *testing.T) {
 	id := fmt.Sprintf("tiowff%v", time.Now().Unix())
@@ -308,7 +311,7 @@ func TestInternalCachedWrittenContentMatches(t *testing.T) {
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, chunkSize*4+chunkSize/2)
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 
 	// write the object
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
@@ -385,27 +388,19 @@ func TestInternalWrappedWrittenContentMatches(t *testing.T) {
 
 	// create some rand test data
 	testSize := chunkSize*4 + chunkSize/2
-	testData := runInstance.randomBytes(t, testSize)
+	testData := randStringBytes(int(testSize))
 
 	// write the object
 	o := runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
 	require.Equal(t, o.Size(), int64(testSize))
 	time.Sleep(time.Second * 3)
 
-	data2, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
+	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", 0, int64(testSize), false)
 	require.NoError(t, err)
-	require.Equal(t, int64(len(data2)), o.Size())
-
-	// check sample of data from in-file
-	sampleStart := chunkSize / 2
-	sampleEnd := chunkSize
-	testSample := testData[sampleStart:sampleEnd]
-	checkSample, err := runInstance.readDataFromRemote(t, rootFs, "data.bin", sampleStart, sampleEnd, false)
-	require.NoError(t, err)
-	require.Equal(t, len(checkSample), len(testSample))
+	require.Equal(t, int64(len(checkSample)), o.Size())
 
 	for i := 0; i < len(checkSample); i++ {
-		require.Equal(t, testSample[i], checkSample[i])
+		require.Equal(t, testData[i], checkSample[i])
 	}
 }
 
@@ -424,7 +419,7 @@ func TestInternalLargeWrittenContentMatches(t *testing.T) {
 
 	// create some rand test data
 	testSize := chunkSize*10 + chunkSize/2
-	testData := runInstance.randomBytes(t, testSize)
+	testData := randStringBytes(int(testSize))
 
 	// write the object
 	runInstance.writeObjectBytes(t, cfs.UnWrap(), "data.bin", testData)
@@ -447,7 +442,7 @@ func TestInternalWrappedFsChangeNotSeen(t *testing.T) {
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
@@ -579,6 +574,93 @@ func TestInternalMoveWithNotify(t *testing.T) {
 	require.NoError(t, err)
 }
 
+func TestInternalNotifyCreatesEmptyParts(t *testing.T) {
+	id := fmt.Sprintf("tincep%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+	if !runInstance.wrappedIsExternal {
+		t.Skipf("Not external")
+	}
+	cfs, err := runInstance.getCacheFs(rootFs)
+	require.NoError(t, err)
+
+	srcName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "test")
+	dstName := runInstance.encryptRemoteIfNeeded(t, "test") + "/" + runInstance.encryptRemoteIfNeeded(t, "one") + "/" + runInstance.encryptRemoteIfNeeded(t, "test2")
+	// create some rand test data
+	var testData []byte
+	if runInstance.rootIsCrypt {
+		testData, err = base64.StdEncoding.DecodeString(cryptedTextBase64)
+		require.NoError(t, err)
+	} else {
+		testData = []byte("test content")
+	}
+	err = rootFs.Mkdir("test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("test/one")
+	require.NoError(t, err)
+	srcObj := runInstance.writeObjectBytes(t, cfs.UnWrap(), srcName, testData)
+
+	// list in mount
+	_, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	_, err = runInstance.list(t, rootFs, "test/one")
+	require.NoError(t, err)
+
+	found := boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.True(t, found)
+	boltDb.Purge()
+	found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.False(t, found)
+
+	// move file
+	_, err = cfs.UnWrap().Features().Move(srcObj, dstName)
+	require.NoError(t, err)
+
+	err = runInstance.retryBlock(func() error {
+		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test")))
+		if !found {
+			log.Printf("not found /test")
+			return errors.Errorf("not found /test")
+		}
+		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one")))
+		if !found {
+			log.Printf("not found /test/one")
+			return errors.Errorf("not found /test/one")
+		}
+		found = boltDb.HasEntry(path.Join(cfs.Root(), runInstance.encryptRemoteIfNeeded(t, "test"), runInstance.encryptRemoteIfNeeded(t, "one"), runInstance.encryptRemoteIfNeeded(t, "test2")))
+		if !found {
+			log.Printf("not found /test/one/test2")
+			return errors.Errorf("not found /test/one/test2")
+		}
+		li, err := runInstance.list(t, rootFs, "test/one")
+		if err != nil {
+			log.Printf("err: %v", err)
+			return err
+		}
+		if len(li) != 1 {
+			log.Printf("not expected listing /test/one: %v", li)
+			return errors.Errorf("not expected listing /test/one: %v", li)
+		}
+		if fi, ok := li[0].(os.FileInfo); ok {
+			if fi.Name() != "test2" {
+				log.Printf("not expected name: %v", fi.Name())
+				return errors.Errorf("not expected name: %v", fi.Name())
+			}
+		} else if di, ok := li[0].(fs.DirEntry); ok {
+			if di.Remote() != "test/one/test2" {
+				log.Printf("not expected remote: %v", di.Remote())
+				return errors.Errorf("not expected remote: %v", di.Remote())
+			}
+		} else {
+			log.Printf("unexpected listing: %v", li)
+			return errors.Errorf("unexpected listing: %v", li)
+		}
+		log.Printf("complete listing /test/one/test2")
+		return nil
+	}, 12, time.Second*10)
+	require.NoError(t, err)
+}
+
 func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	id := fmt.Sprintf("ticsadcf%v", time.Now().Unix())
 	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true, nil, nil)
@@ -589,7 +671,7 @@ func TestInternalChangeSeenAfterDirCacheFlush(t *testing.T) {
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
@@ -623,13 +705,16 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	if !runInstance.useMount {
 		t.Skipf("needs mount")
 	}
+	if !runInstance.wrappedIsExternal {
+		t.Skipf("needs drive")
+	}
 
 	cfs, err := runInstance.getCacheFs(rootFs)
 	require.NoError(t, err)
 	chunkSize := cfs.ChunkSize()
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 
 	// update in the wrapped fs
@@ -660,6 +745,31 @@ func TestInternalChangeSeenAfterRc(t *testing.T) {
 	co, err = rootFs.NewObject("data.bin")
 	require.NoError(t, err)
 	require.Equal(t, wrappedTime.Unix(), co.ModTime().Unix())
+	li1, err := runInstance.list(t, rootFs, "")
+
+	// create some rand test data
+	testData2 := randStringBytes(int(chunkSize))
+	runInstance.writeObjectBytes(t, cfs.UnWrap(), runInstance.encryptRemoteIfNeeded(t, "test2"), testData2)
+
+	// list should have 1 item only
+	li1, err = runInstance.list(t, rootFs, "")
+	require.Len(t, li1, 1)
+
+	m = make(map[string]string)
+	res2, err := http.Post("http://localhost:5572/cache/expire?remote=/", "application/json; charset=utf-8", strings.NewReader(""))
+	require.NoError(t, err)
+	defer func() {
+		_ = res2.Body.Close()
+	}()
+	_ = json.NewDecoder(res2.Body).Decode(&m)
+	require.Contains(t, m, "status")
+	require.Contains(t, m, "message")
+	require.Equal(t, "ok", m["status"])
+	require.Contains(t, m["message"], "cached directory cleared")
+
+	// list should have 2 items now
+	li2, err := runInstance.list(t, rootFs, "")
+	require.Len(t, li2, 2)
 }
 
 func TestInternalCacheWrites(t *testing.T) {
@@ -673,7 +783,7 @@ func TestInternalCacheWrites(t *testing.T) {
 
 	// create some rand test data
 	earliestTime := time.Now()
-	testData := runInstance.randomBytes(t, (chunkSize*4 + chunkSize/2))
+	testData := randStringBytes(int(chunkSize*4 + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 	expectedTs := time.Now()
 	ts, err := boltDb.GetChunkTs(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "data.bin")), 0)
@@ -692,7 +802,7 @@ func TestInternalMaxChunkSizeRespected(t *testing.T) {
 	totalChunks := 20
 
 	// create some rand test data
-	testData := runInstance.randomBytes(t, (int64(totalChunks-1)*chunkSize + chunkSize/2))
+	testData := randStringBytes(int(int64(totalChunks-1)*chunkSize + chunkSize/2))
 	runInstance.writeRemoteBytes(t, rootFs, "data.bin", testData)
 	o, err := cfs.NewObject(runInstance.encryptRemoteIfNeeded(t, "data.bin"))
 	require.NoError(t, err)
@@ -805,458 +915,6 @@ func TestInternalBug2117(t *testing.T) {
 	require.Len(t, di, 4)
 }
 
-func TestInternalUploadTempDirCreated(t *testing.T) {
-	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
-	require.NoError(t, err)
-}
-
-func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
-	// create some rand test data
-	testSize := int64(524288000)
-	testReader := runInstance.randomReader(t, testSize)
-	bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
-	runInstance.writeRemoteReader(t, rootFs, "one", testReader)
-	// validate that it exists in temp fs
-	ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
-	require.NoError(t, err)
-
-	if runInstance.rootIsCrypt {
-		require.Equal(t, int64(524416032), ti.Size())
-	} else {
-		require.Equal(t, testSize, ti.Size())
-	}
-	de1, err := runInstance.list(t, rootFs, "")
-	require.NoError(t, err)
-	require.Len(t, de1, 1)
-
-	runInstance.completeBackgroundUpload(t, "one", bu)
-	// check if it was removed from temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
-	require.True(t, os.IsNotExist(err))
-
-	// check if it can be read
-	data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
-	require.NoError(t, err)
-	require.Len(t, data2, 1024)
-}
-
-func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
-	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
-}
-
-func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
-	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
-}
-
-func TestInternalUploadMoveExistingFile(t *testing.T) {
-	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	err := rootFs.Mkdir("one")
-	require.NoError(t, err)
-	err = rootFs.Mkdir("one/test")
-	require.NoError(t, err)
-	err = rootFs.Mkdir("second")
-	require.NoError(t, err)
-
-	// create some rand test data
-	testSize := int64(10485760)
-	testReader := runInstance.randomReader(t, testSize)
-	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
-	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
-
-	de1, err := runInstance.list(t, rootFs, "one/test")
-	require.NoError(t, err)
-	require.Len(t, de1, 1)
-
-	time.Sleep(time.Second * 5)
-	_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
-	require.NoError(t, err)
-
-	err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
-	require.NoError(t, err)
-
-	// check if it can be read
-	de1, err = runInstance.list(t, rootFs, "second/test")
-	require.NoError(t, err)
-	require.Len(t, de1, 1)
-}
-
-func TestInternalUploadQueueMoreFiles(t *testing.T) {
-	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	err := rootFs.Mkdir("test")
-	require.NoError(t, err)
-	minSize := 5242880
-	maxSize := 10485760
-	totalFiles := 10
-	rand.Seed(time.Now().Unix())
-
-	lastFile := ""
-	for i := 0; i < totalFiles; i++ {
-		size := int64(rand.Intn(maxSize-minSize) + minSize)
-		testReader := runInstance.randomReader(t, size)
-		remote := "test/" + strconv.Itoa(i) + ".bin"
-		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
-
-		// validate that it exists in temp fs
-		ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
-		require.NoError(t, err)
-		require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
-
-		if runInstance.wrappedIsExternal && i < totalFiles-1 {
-			time.Sleep(time.Second * 3)
-		}
-		lastFile = remote
-	}
-
-	// check if cache lists all files, likely temp upload didn't finish yet
-	de1, err := runInstance.list(t, rootFs, "test")
-	require.NoError(t, err)
-	require.Len(t, de1, totalFiles)
-
-	// wait for background uploader to do its thing
-	runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
-
-	// retry until we have no more temp files and fail if they don't go down to 0
-	tf, err := ioutil.ReadDir(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
-	require.NoError(t, err)
-	require.Len(t, tf, 0)
-
-	// check if cache lists all files
-	de1, err = runInstance.list(t, rootFs, "test")
-	require.NoError(t, err)
-	require.Len(t, de1, totalFiles)
-}
-
-func TestInternalUploadTempFileOperations(t *testing.T) {
-	id := "tiutfo"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	boltDb.PurgeTempUploads()
-
-	// create some rand test data
-	runInstance.mkdir(t, rootFs, "test")
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// check if it can be read
-	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
-	require.NoError(t, err)
-	require.Equal(t, []byte("one content"), data1)
-	// validate that it exists in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-
-	// test DirMove - allowed
-	err = runInstance.dirMove(t, rootFs, "test", "second")
-	if err != errNotSupported {
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.Error(t, err)
-		_, err = rootFs.NewObject("second/one")
-		require.NoError(t, err)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.Error(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
-		require.NoError(t, err)
-		started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
-		require.Error(t, err)
-		started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
-		require.NoError(t, err)
-		require.False(t, started)
-		runInstance.mkdir(t, rootFs, "test")
-		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-	}
-
-	// test Rmdir - allowed
-	err = runInstance.rm(t, rootFs, "test")
-	require.Error(t, err)
-	require.Contains(t, err.Error(), "directory not empty")
-	_, err = rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	// validate that it exists in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-	started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
-	require.False(t, started)
-	require.NoError(t, err)
-
-	// test Move/Rename -- allowed
-	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
-	if err != errNotSupported {
-		require.NoError(t, err)
-		// try to read from it
-		_, err = rootFs.NewObject("test/one")
-		require.Error(t, err)
-		_, err = rootFs.NewObject("test/second")
-		require.NoError(t, err)
-		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
-		require.NoError(t, err)
-		require.Equal(t, []byte("one content"), data2)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.Error(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
-		require.NoError(t, err)
-		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-	}
-
-	// test Copy -- allowed
-	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
-	if err != errNotSupported {
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/third")
-		require.NoError(t, err)
-		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
-		require.NoError(t, err)
-		require.Equal(t, []byte("one content"), data2)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
-		require.NoError(t, err)
-	}
-
-	// test Remove -- allowed
-	err = runInstance.rm(t, rootFs, "test/one")
-	require.NoError(t, err)
-	_, err = rootFs.NewObject("test/one")
-	require.Error(t, err)
-	// validate that it doesn't exist in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.Error(t, err)
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// test Update -- allowed
-	firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	require.NoError(t, err)
-	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
-	require.NoError(t, err)
-	obj2, err := rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
-	require.Equal(t, "one content updated", string(data2))
-	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-	if runInstance.rootIsCrypt {
-		require.Equal(t, int64(67), tmpInfo.Size())
-	} else {
-		require.Equal(t, int64(len(data2)), tmpInfo.Size())
-	}
-
-	// test SetModTime -- allowed
-	secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	require.NoError(t, err)
-	require.NotEqual(t, secondModTime, firstModTime)
-	require.NotEqual(t, time.Time{}, firstModTime)
-	require.NotEqual(t, time.Time{}, secondModTime)
-}
-
-func TestInternalUploadUploadingFileOperations(t *testing.T) {
-	id := "tiuufo"
-	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
-		nil,
-		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
-	defer runInstance.cleanupFs(t, rootFs, boltDb)
-
-	boltDb.PurgeTempUploads()
-
-	// create some rand test data
-	runInstance.mkdir(t, rootFs, "test")
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// check if it can be read
-	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
-	require.NoError(t, err)
-	require.Equal(t, []byte("one content"), data1)
-	// validate that it exists in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-
-	err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
-	require.NoError(t, err)
-
-	// test DirMove
-	err = runInstance.dirMove(t, rootFs, "test", "second")
-	if err != errNotSupported {
-		require.Error(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
-		require.Error(t, err)
-	}
-
-	// test Rmdir
-	err = runInstance.rm(t, rootFs, "test")
-	require.Error(t, err)
-	_, err = rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	// validate that it doesn't exist in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-
-	// test Move/Rename
-	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
-	if err != errNotSupported {
-		require.Error(t, err)
-		// try to read from it
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/second")
-		require.Error(t, err)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
-		require.Error(t, err)
-	}
-
-	// test Copy -- allowed
-	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
-	if err != errNotSupported {
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/one")
-		require.NoError(t, err)
-		_, err = rootFs.NewObject("test/third")
-		require.NoError(t, err)
-		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
-		require.NoError(t, err)
-		require.Equal(t, []byte("one content"), data2)
-		// validate that it exists in temp fs
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-		require.NoError(t, err)
-		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
-		require.NoError(t, err)
-	}
-
-	// test Remove
-	err = runInstance.rm(t, rootFs, "test/one")
-	require.Error(t, err)
-	_, err = rootFs.NewObject("test/one")
-	require.NoError(t, err)
-	// validate that it doesn't exist in temp fs
-	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	require.NoError(t, err)
-	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
-
-	// test Update - this seems to work. Why? FIXME
-	//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	//require.NoError(t, err)
-	//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
-	//	data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
-	//	require.Equal(t, "one content", string(data2))
-	//
-	//	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
-	//	require.NoError(t, err)
-	//	if runInstance.rootIsCrypt {
-	//		require.Equal(t, int64(67), tmpInfo.Size())
-	//	} else {
-	//		require.Equal(t, int64(len(data2)), tmpInfo.Size())
-	//	}
-	//})
-	//require.Error(t, err)
-
-	// test SetModTime -- seems to work cause of previous
-	//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
-	//require.NoError(t, err)
-	//require.Equal(t, secondModTime, firstModTime)
-	//require.NotEqual(t, time.Time{}, firstModTime)
-	//require.NotEqual(t, time.Time{}, secondModTime)
-}
-
-// FIXME, enable this when mount is sorted out
-//func TestInternalFilesMissingInMount1904(t *testing.T) {
-//	t.Skip("Not yet")
-//	if runtime.GOOS == "windows" {
-//		t.Skip("Not yet")
-//	}
-//	id := "tifm1904"
-//	rootFs, _ := newCacheFs(t, RemoteName, id, false,
-//		map[string]string{"chunk_size": "5M", "info_age": "1m", "chunk_total_size": "500M", "cache-writes": "true"})
-//	mntPoint := path.Join("/tmp", "tifm1904-mnt")
-//	testPoint := path.Join(mntPoint, id)
-//	checkOutput := "1 10 100 11 12 13 14 15 16 17 18 19 2 20 21 22 23 24 25 26 27 28 29 3 30 31 32 33 34 35 36 37 38 39 4 40 41 42 43 44 45 46 47 48 49 5 50 51 52 53 54 55 56 57 58 59 6 60 61 62 63 64 65 66 67 68 69 7 70 71 72 73 74 75 76 77 78 79 8 80 81 82 83 84 85 86 87 88 89 9 90 91 92 93 94 95 96 97 98 99 "
-//
-//	_ = os.MkdirAll(mntPoint, os.ModePerm)
-//
-//	list, err := rootFs.List("")
-//	require.NoError(t, err)
-//	found := false
-//	list.ForDir(func(d fs.Directory) {
-//		if strings.Contains(d.Remote(), id) {
-//			found = true
-//		}
-//	})
-//
-//	if !found {
-//		t.Skip("Test folder '%v' doesn't exist", id)
-//	}
-//
-//	mountFs(t, rootFs, mntPoint)
-//	defer unmountFs(t, mntPoint)
-//
-//	for i := 1; i <= 2; i++ {
-//		out, err := exec.Command("ls", testPoint).Output()
-//		require.NoError(t, err)
-//		require.Equal(t, checkOutput, strings.Replace(string(out), "\n", " ", -1))
-//		t.Logf("root path has all files")
-//		_ = writeObjectString(t, rootFs, path.Join(id, strconv.Itoa(i), strconv.Itoa(i), "one_file"), "one content")
-//
-//		for j := 1; j <= 100; j++ {
-//			out, err := exec.Command("ls", path.Join(testPoint, strconv.Itoa(j))).Output()
-//			require.NoError(t, err)
-//			require.Equal(t, checkOutput, strings.Replace(string(out), "\n", " ", -1), "'%v' doesn't match", j)
-//		}
-//		obj, err := rootFs.NewObject(path.Join(id, strconv.Itoa(i), strconv.Itoa(i), "one_file"))
-//		require.NoError(t, err)
-//		err = obj.Remove()
-//		require.NoError(t, err)
-//		t.Logf("folders contain all the files")
-//
-//		out, err = exec.Command("date").Output()
-//		require.NoError(t, err)
-//		t.Logf("check #%v date: '%v'", i, strings.Replace(string(out), "\n", " ", -1))
-//
-//		if i < 2 {
-//			time.Sleep(time.Second * 60)
-//		}
-//	}
-//}
-
 // run holds the remotes for a test run
 type run struct {
 	okDiff            time.Duration
@@ -1395,7 +1053,7 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 		config.FileSet(remote, "type", "cache")
 		config.FileSet(remote, "remote", localRemote+":/var/tmp/"+localRemote)
 	} else {
-		remoteType := fs.ConfigFileGet(remote, "type", "")
+		remoteType := config.FileGet(remote, "type", "")
 		if remoteType == "" {
 			t.Skipf("skipped due to invalid remote type for %v", remote)
 			return nil, nil
@@ -1406,14 +1064,14 @@ func (r *run) newCacheFs(t *testing.T, remote, id string, needRemote, purge bool
 				config.FileSet(remote, "password", cryptPassword1)
 				config.FileSet(remote, "password2", cryptPassword2)
 			}
-			remoteRemote := fs.ConfigFileGet(remote, "remote", "")
+			remoteRemote := config.FileGet(remote, "remote", "")
 			if remoteRemote == "" {
 				t.Skipf("skipped due to invalid remote wrapper for %v", remote)
 				return nil, nil
 			}
 			remoteRemoteParts := strings.Split(remoteRemote, ":")
 			remoteWrapping := remoteRemoteParts[0]
-			remoteType := fs.ConfigFileGet(remoteWrapping, "type", "")
+			remoteType := config.FileGet(remoteWrapping, "type", "")
 			if remoteType != "cache" {
 				t.Skipf("skipped due to invalid remote type for %v: '%v'", remoteWrapping, remoteType)
 				return nil, nil
@@ -1504,15 +1162,6 @@ func (r *run) cleanupFs(t *testing.T, f fs.Fs, b *cache.Persistent) {
 	}
 }
 
-func (r *run) randomBytes(t *testing.T, size int64) []byte {
-	testData := make([]byte, size)
-	testSize, err := rand.Read(testData)
-	require.Equal(t, size, int64(len(testData)))
-	require.Equal(t, size, int64(testSize))
-	require.NoError(t, err)
-	return testData
-}
-
 func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	chunk := int64(1024)
 	cnt := size / chunk
@@ -1521,10 +1170,10 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 	require.NoError(t, err)
 
 	for i := 0; i < int(cnt); i++ {
-		data := r.randomBytes(t, chunk)
+		data := randStringBytes(int(chunk))
 		_, _ = f.Write(data)
 	}
-	data := r.randomBytes(t, int64(left))
+	data := randStringBytes(int(left))
 	_, _ = f.Write(data)
 	_, _ = f.Seek(int64(0), io.SeekStart)
 	r.tempFiles = append(r.tempFiles, f)
@@ -1535,7 +1184,7 @@ func (r *run) randomReader(t *testing.T, size int64) io.ReadCloser {
 func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64) string {
 	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
 	// create some rand test data
-	testData := r.randomBytes(t, size)
+	testData := randStringBytes(int(size))
 
 	r.writeRemoteBytes(t, f, remote, testData)
 	return remote
@@ -1544,7 +1193,7 @@ func (r *run) writeRemoteRandomBytes(t *testing.T, f fs.Fs, p string, size int64
 func (r *run) writeObjectRandomBytes(t *testing.T, f fs.Fs, p string, size int64) fs.Object {
 	remote := path.Join(p, strconv.Itoa(rand.Int())+".bin")
 	// create some rand test data
-	testData := r.randomBytes(t, size)
+	testData := randStringBytes(int(size))
 
 	return r.writeObjectBytes(t, f, remote, testData)
 }
@@ -1662,9 +1311,6 @@ func (r *run) readDataFromRemote(t *testing.T, f fs.Fs, remote string, offset, e
 		if err != nil {
 			return checkSample, err
 		}
-		if !noLengthCheck && size != int64(totalRead) {
-			return checkSample, errors.Errorf("read size doesn't match expected: %v <> %v", totalRead, size)
-		}
 	} else {
 		co, err := f.NewObject(remote)
 		if err != nil {
@@ -1688,7 +1334,7 @@ func (r *run) readDataFromObj(t *testing.T, o fs.Object, offset, end int64, noLe
 		err = nil
 		checkSample = checkSample[:totalRead]
 	}
-	require.NoError(t, err)
+	require.NoError(t, err, "with string -%v-", string(checkSample))
 	_ = reader.Close()
 	return checkSample
 }
@@ -1897,16 +1543,11 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		if err != nil {
 			return err
 		}
-		_, err = f.WriteString(data + append)
-		if err != nil {
+		defer func() {
 			_ = f.Close()
-			return err
-		}
-		err = f.Close()
-		if err != nil {
-			return err
-		}
-		r.vfs.WaitForWriters(10 * time.Second)
+			r.vfs.WaitForWriters(10 * time.Second)
+		}()
+		_, err = f.WriteString(data + append)
 	} else {
 		obj1, err := rootFs.NewObject(src)
 		if err != nil {
@@ -1916,9 +1557,6 @@ func (r *run) updateData(t *testing.T, rootFs fs.Fs, src, data, append string) e
 		r := bytes.NewReader(data1)
 		objInfo1 := object.NewStaticObjectInfo(src, time.Now(), int64(len(data1)), true, nil, rootFs)
 		err = obj1.Update(r, objInfo1)
-		if err != nil {
-			return err
-		}
 	}
 
 	return err
@@ -2055,6 +1693,14 @@ func (r *run) getCacheFs(f fs.Fs) (*cache.Fs, error) {
 	return nil, errors.New("didn't found a cache fs")
 }
 
+func randStringBytes(n int) []byte {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return b
+}
+
 var (
 	_ fs.Fs = (*cache.Fs)(nil)
 	_ fs.Fs = (*local.Fs)(nil)

backend/cache/cache_mount_unix_test.go

@@ -23,7 +23,7 @@ func (r *run) mountFs(t *testing.T, f fs.Fs) {
 		fuse.FSName(device), fuse.VolumeName(device),
 		fuse.NoAppleDouble(),
 		fuse.NoAppleXattr(),
-		fuse.AllowOther(),
+		//fuse.AllowOther(),
 	}
 	err := os.MkdirAll(r.mntDir, os.ModePerm)
 	require.NoError(t, err)

backend/cache/cache_upload_test.go

@@ -0,0 +1,455 @@
+// +build !plan9
+
+package cache_test
+
+import (
+	"math/rand"
+	"os"
+	"path"
+	"strconv"
+	"testing"
+	"time"
+
+	"fmt"
+
+	"github.com/ncw/rclone/backend/cache"
+	_ "github.com/ncw/rclone/backend/drive"
+	"github.com/ncw/rclone/fs"
+	"github.com/stretchr/testify/require"
+)
+
+func TestInternalUploadTempDirCreated(t *testing.T) {
+	id := fmt.Sprintf("tiutdc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, false, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id)})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	_, err := os.Stat(path.Join(runInstance.tmpUploadDir, id))
+	require.NoError(t, err)
+}
+
+func testInternalUploadQueueOneFile(t *testing.T, id string, rootFs fs.Fs, boltDb *cache.Persistent) {
+	// create some rand test data
+	testSize := int64(524288000)
+	testReader := runInstance.randomReader(t, testSize)
+	bu := runInstance.listenForBackgroundUpload(t, rootFs, "one")
+	runInstance.writeRemoteReader(t, rootFs, "one", testReader)
+	// validate that it exists in temp fs
+	ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.NoError(t, err)
+
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(524416032), ti.Size())
+	} else {
+		require.Equal(t, testSize, ti.Size())
+	}
+	de1, err := runInstance.list(t, rootFs, "")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	runInstance.completeBackgroundUpload(t, "one", bu)
+	// check if it was removed from temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if it can be read
+	data2, err := runInstance.readDataFromRemote(t, rootFs, "one", 0, int64(1024), false)
+	require.NoError(t, err)
+	require.Len(t, data2, 1024)
+}
+
+func TestInternalUploadQueueOneFileNoRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofnr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "0s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadQueueOneFileWithRest(t *testing.T) {
+	id := fmt.Sprintf("tiuqofwr%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1m"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	testInternalUploadQueueOneFile(t, id, rootFs, boltDb)
+}
+
+func TestInternalUploadMoveExistingFile(t *testing.T) {
+	id := fmt.Sprintf("tiumef%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "3s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(10485760)
+	testReader := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	time.Sleep(time.Second * 5)
+	//_ = os.Remove(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	//require.NoError(t, err)
+
+	err = runInstance.dirMove(t, rootFs, "one/test", "second/test")
+	require.NoError(t, err)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadTempPathCleaned(t *testing.T) {
+	id := fmt.Sprintf("tiutpc%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "5s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("one")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("one/test")
+	require.NoError(t, err)
+	err = rootFs.Mkdir("second")
+	require.NoError(t, err)
+
+	// create some rand test data
+	testSize := int64(1048576)
+	testReader := runInstance.randomReader(t, testSize)
+	testReader2 := runInstance.randomReader(t, testSize)
+	runInstance.writeObjectReader(t, rootFs, "one/test/data.bin", testReader)
+	runInstance.writeObjectReader(t, rootFs, "second/data.bin", testReader2)
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "one/test/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one/test")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "one")))
+	require.True(t, os.IsNotExist(err))
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second")))
+	require.False(t, os.IsNotExist(err))
+
+	runInstance.completeAllBackgroundUploads(t, rootFs, "second/data.bin")
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/data.bin")))
+	require.True(t, os.IsNotExist(err))
+
+	de1, err := runInstance.list(t, rootFs, "one/test")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+
+	// check if it can be read
+	de1, err = runInstance.list(t, rootFs, "second")
+	require.NoError(t, err)
+	require.Len(t, de1, 1)
+}
+
+func TestInternalUploadQueueMoreFiles(t *testing.T) {
+	id := fmt.Sprintf("tiuqmf%v", time.Now().Unix())
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1s"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	err := rootFs.Mkdir("test")
+	require.NoError(t, err)
+	minSize := 5242880
+	maxSize := 10485760
+	totalFiles := 10
+	rand.Seed(time.Now().Unix())
+
+	lastFile := ""
+	for i := 0; i < totalFiles; i++ {
+		size := int64(rand.Intn(maxSize-minSize) + minSize)
+		testReader := runInstance.randomReader(t, size)
+		remote := "test/" + strconv.Itoa(i) + ".bin"
+		runInstance.writeRemoteReader(t, rootFs, remote, testReader)
+
+		// validate that it exists in temp fs
+		ti, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, remote)))
+		require.NoError(t, err)
+		require.Equal(t, size, runInstance.cleanSize(t, ti.Size()))
+
+		if runInstance.wrappedIsExternal && i < totalFiles-1 {
+			time.Sleep(time.Second * 3)
+		}
+		lastFile = remote
+	}
+
+	// check if cache lists all files, likely temp upload didn't finish yet
+	de1, err := runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+
+	// wait for background uploader to do its thing
+	runInstance.completeAllBackgroundUploads(t, rootFs, lastFile)
+
+	// retry until we have no more temp files and fail if they don't go down to 0
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test")))
+	require.True(t, os.IsNotExist(err))
+
+	// check if cache lists all files
+	de1, err = runInstance.list(t, rootFs, "test")
+	require.NoError(t, err)
+	require.Len(t, de1, totalFiles)
+}
+
+func TestInternalUploadTempFileOperations(t *testing.T) {
+	id := "tiutfo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove - allowed
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("second/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.NoError(t, err)
+		_, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+		require.Error(t, err)
+		var started bool
+		started, err = boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "second/one")))
+		require.NoError(t, err)
+		require.False(t, started)
+		runInstance.mkdir(t, rootFs, "test")
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Rmdir - allowed
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "directory not empty")
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	started, err := boltDb.SearchPendingUpload(runInstance.encryptRemoteIfNeeded(t, path.Join(id, "test/one")))
+	require.False(t, started)
+	require.NoError(t, err)
+
+	// test Move/Rename -- allowed
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/second", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.Error(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.NoError(t, err)
+		runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove -- allowed
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.NoError(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.Error(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.Error(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update -- allowed
+	firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated")
+	require.NoError(t, err)
+	obj2, err := rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	data2 := runInstance.readDataFromObj(t, obj2, 0, int64(len("one content updated")), false)
+	require.Equal(t, "one content updated", string(data2))
+	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	if runInstance.rootIsCrypt {
+		require.Equal(t, int64(67), tmpInfo.Size())
+	} else {
+		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	}
+
+	// test SetModTime -- allowed
+	secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	require.NoError(t, err)
+	require.NotEqual(t, secondModTime, firstModTime)
+	require.NotEqual(t, time.Time{}, firstModTime)
+	require.NotEqual(t, time.Time{}, secondModTime)
+}
+
+func TestInternalUploadUploadingFileOperations(t *testing.T) {
+	id := "tiuufo"
+	rootFs, boltDb := runInstance.newCacheFs(t, remoteName, id, true, true,
+		nil,
+		map[string]string{"cache-tmp-upload-path": path.Join(runInstance.tmpUploadDir, id), "cache-tmp-wait-time": "1h"})
+	defer runInstance.cleanupFs(t, rootFs, boltDb)
+
+	boltDb.PurgeTempUploads()
+
+	// create some rand test data
+	runInstance.mkdir(t, rootFs, "test")
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// check if it can be read
+	data1, err := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len([]byte("one content"))), false)
+	require.NoError(t, err)
+	require.Equal(t, []byte("one content"), data1)
+	// validate that it exists in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	err = boltDb.SetPendingUploadToStarted(runInstance.encryptRemoteIfNeeded(t, path.Join(rootFs.Root(), "test/one")))
+	require.NoError(t, err)
+
+	// test DirMove
+	err = runInstance.dirMove(t, rootFs, "test", "second")
+	if err != errNotSupported {
+		require.Error(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "second/one")))
+		require.Error(t, err)
+	}
+
+	// test Rmdir
+	err = runInstance.rm(t, rootFs, "test")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+
+	// test Move/Rename
+	err = runInstance.move(t, rootFs, path.Join("test", "one"), path.Join("test", "second"))
+	if err != errNotSupported {
+		require.Error(t, err)
+		// try to read from it
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/second")
+		require.Error(t, err)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/second")))
+		require.Error(t, err)
+	}
+
+	// test Copy -- allowed
+	err = runInstance.copy(t, rootFs, path.Join("test", "one"), path.Join("test", "third"))
+	if err != errNotSupported {
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/one")
+		require.NoError(t, err)
+		_, err = rootFs.NewObject("test/third")
+		require.NoError(t, err)
+		data2, err := runInstance.readDataFromRemote(t, rootFs, "test/third", 0, int64(len([]byte("one content"))), false)
+		require.NoError(t, err)
+		require.Equal(t, []byte("one content"), data2)
+		// validate that it exists in temp fs
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+		require.NoError(t, err)
+		_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/third")))
+		require.NoError(t, err)
+	}
+
+	// test Remove
+	err = runInstance.rm(t, rootFs, "test/one")
+	require.Error(t, err)
+	_, err = rootFs.NewObject("test/one")
+	require.NoError(t, err)
+	// validate that it doesn't exist in temp fs
+	_, err = os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	require.NoError(t, err)
+	runInstance.writeRemoteString(t, rootFs, "test/one", "one content")
+
+	// test Update - this seems to work. Why? FIXME
+	//firstModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//err = runInstance.updateData(t, rootFs, "test/one", "one content", " updated", func() {
+	//	data2 := runInstance.readDataFromRemote(t, rootFs, "test/one", 0, int64(len("one content updated")), true)
+	//	require.Equal(t, "one content", string(data2))
+	//
+	//	tmpInfo, err := os.Stat(path.Join(runInstance.tmpUploadDir, id, runInstance.encryptRemoteIfNeeded(t, "test/one")))
+	//	require.NoError(t, err)
+	//	if runInstance.rootIsCrypt {
+	//		require.Equal(t, int64(67), tmpInfo.Size())
+	//	} else {
+	//		require.Equal(t, int64(len(data2)), tmpInfo.Size())
+	//	}
+	//})
+	//require.Error(t, err)
+
+	// test SetModTime -- seems to work cause of previous
+	//secondModTime, err := runInstance.modTime(t, rootFs, "test/one")
+	//require.NoError(t, err)
+	//require.Equal(t, secondModTime, firstModTime)
+	//require.NotEqual(t, time.Time{}, firstModTime)
+	//require.NotEqual(t, time.Time{}, secondModTime)
+}

backend/cache/handle.go

@@ -147,10 +147,9 @@ func (r *Handle) scaleWorkers(desired int) {
 
 func (r *Handle) confirmExternalReading() {
 	// if we have a max value of workers
-	// or there's no external confirmation available
 	// then we skip this step
 	if len(r.workers) > 1 ||
-		!r.cacheFs().plexConnector.isConnected() {
+		!r.cacheFs().plexConnector.isConfigured() {
 		return
 	}
 	if !r.cacheFs().plexConnector.isPlaying(r.cachedObject) {
@@ -446,7 +445,6 @@ func (w *worker) run() {
 					continue
 				}
 			}
-			err = nil
 		} else {
 			if w.r.storage().HasChunk(w.r.cachedObject, chunkStart) {
 				continue
@@ -493,7 +491,7 @@ func (w *worker) download(chunkStart, chunkEnd int64, retry int) {
 	}
 
 	data = make([]byte, chunkEnd-chunkStart)
-	sourceRead := 0
+	var sourceRead int
 	sourceRead, err = io.ReadFull(w.rc, data)
 	if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 		fs.Errorf(w, "failed to read chunk %v: %v", chunkStart, err)
@@ -630,6 +628,23 @@ func (b *backgroundWriter) run() {
 			fs.Errorf(remote, "background upload: %v", err)
 			continue
 		}
+		// clean empty dirs up to root
+		thisDir := cleanPath(path.Dir(remote))
+		for thisDir != "" {
+			thisList, err := b.fs.tempFs.List(thisDir)
+			if err != nil {
+				break
+			}
+			if len(thisList) > 0 {
+				break
+			}
+			err = b.fs.tempFs.Rmdir(thisDir)
+			fs.Debugf(thisDir, "cleaned from temp path")
+			if err != nil {
+				break
+			}
+			thisDir = cleanPath(path.Dir(thisDir))
+		}
 		fs.Infof(remote, "background upload: uploaded entry")
 		err = b.fs.cache.removePendingUpload(absPath)
 		if err != nil && !strings.Contains(err.Error(), "pending upload not found") {

backend/cache/object.go

@@ -256,6 +256,8 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 
 	// deleting cached chunks and info to be replaced with new ones
 	_ = o.CacheFs.cache.RemoveObject(o.abs())
+	// advertise to ChangeNotify if wrapped doesn't do that
+	o.CacheFs.notifyChangeUpstreamIfNeeded(o.Remote(), fs.EntryObject)
 
 	o.CacheModTime = src.ModTime().UnixNano()
 	o.CacheSize = src.Size()

backend/cache/plex.go

@@ -107,8 +107,11 @@ func (p *plexConnector) closeWebsocket() {
 }
 
 func (p *plexConnector) listenWebsocket() {
+	p.runningMu.Lock()
+	defer p.runningMu.Unlock()
+
 	u := strings.Replace(p.url.String(), "http://", "ws://", 1)
-	u = strings.Replace(u, "https://", "ws://", 1)
+	u = strings.Replace(u, "https://", "wss://", 1)
 	conn, err := websocket.Dial(fmt.Sprintf(defPlexNotificationURL, strings.TrimRight(u, "/"), p.token),
 		"", "http://localhost")
 	if err != nil {
@@ -127,8 +130,8 @@ func (p *plexConnector) listenWebsocket() {
 			err := websocket.JSON.Receive(conn, notif)
 			if err != nil {
 				fs.Debugf("plex", "%v", err)
-				time.Sleep(time.Second)
-				continue
+				p.closeWebsocket()
+				break
 			}
 			// we're only interested in play events
 			if notif.Container.Type == "playing" {
@@ -229,6 +232,9 @@ func (p *plexConnector) isConfigured() bool {
 
 func (p *plexConnector) isPlaying(co *Object) bool {
 	var err error
+	if !p.isConnected() {
+		p.listenWebsocket()
+	}
 
 	remote := co.Remote()
 	if cr, yes := p.f.isWrappedByCrypt(); yes {

backend/cache/storage_persistent.go

@@ -192,19 +192,46 @@ func (b *Persistent) GetDir(remote string) (*Directory, error) {
 
 // AddDir will update a CachedDirectory metadata and all its entries
 func (b *Persistent) AddDir(cachedDir *Directory) error {
+	return b.AddBatchDir([]*Directory{cachedDir})
+}
+
+// AddBatchDir will update a list of CachedDirectory metadata and all their entries
+func (b *Persistent) AddBatchDir(cachedDirs []*Directory) error {
+	if len(cachedDirs) == 0 {
+		return nil
+	}
+
 	return b.db.Update(func(tx *bolt.Tx) error {
-		bucket := b.getBucket(cachedDir.abs(), true, tx)
+		var bucket *bolt.Bucket
+		if cachedDirs[0].Dir == "" {
+			bucket = tx.Bucket([]byte(RootBucket))
+		} else {
+			bucket = b.getBucket(cachedDirs[0].Dir, true, tx)
+		}
 		if bucket == nil {
-			return errors.Errorf("couldn't open bucket (%v)", cachedDir)
+			return errors.Errorf("couldn't open bucket (%v)", cachedDirs[0].Dir)
 		}
 
-		encoded, err := json.Marshal(cachedDir)
-		if err != nil {
-			return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
-		}
-		err = bucket.Put([]byte("."), encoded)
-		if err != nil {
-			return err
+		for _, cachedDir := range cachedDirs {
+			var b *bolt.Bucket
+			var err error
+			if cachedDir.Name == "" {
+				b = bucket
+			} else {
+				b, err = bucket.CreateBucketIfNotExists([]byte(cachedDir.Name))
+			}
+			if err != nil {
+				return err
+			}
+
+			encoded, err := json.Marshal(cachedDir)
+			if err != nil {
+				return errors.Errorf("couldn't marshal object (%v): %v", cachedDir, err)
+			}
+			err = b.Put([]byte("."), encoded)
+			if err != nil {
+				return err
+			}
 		}
 		return nil
 	})
@@ -1070,10 +1097,3 @@ func itob(v int64) []byte {
 func btoi(d []byte) int64 {
 	return int64(binary.BigEndian.Uint64(d))
 }
-
-// cloneBytes returns a copy of a given slice.
-func cloneBytes(v []byte) []byte {
-	var clone = make([]byte, len(v))
-	copy(clone, v)
-	return clone
-}

backend/crypt/crypt.go

@@ -168,7 +168,6 @@ type Fs struct {
 	root     string
 	features *fs.Features // optional features
 	cipher   Cipher
-	mode     NameEncryptionMode
 }
 
 // Name of the remote (as passed into NewFs)
@@ -291,14 +290,48 @@ type putFn func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.O
 
 // put implements Put or PutStream
 func (f *Fs) put(in io.Reader, src fs.ObjectInfo, options []fs.OpenOption, put putFn) (fs.Object, error) {
+	// Encrypt the data into wrappedIn
 	wrappedIn, err := f.cipher.EncryptData(in)
 	if err != nil {
 		return nil, err
 	}
+
+	// Find a hash the destination supports to compute a hash of
+	// the encrypted data
+	ht := f.Fs.Hashes().GetOne()
+	var hasher *hash.MultiHasher
+	if ht != hash.None {
+		hasher, err = hash.NewMultiHasherTypes(hash.NewHashSet(ht))
+		if err != nil {
+			return nil, err
+		}
+		wrappedIn = io.TeeReader(wrappedIn, hasher)
+	}
+
+	// Transfer the data
 	o, err := put(wrappedIn, f.newObjectInfo(src), options...)
 	if err != nil {
 		return nil, err
 	}
+
+	// Check the hashes of the encrypted data if we were comparing them
+	if ht != hash.None && hasher != nil {
+		srcHash := hasher.Sums()[ht]
+		var dstHash string
+		dstHash, err = o.Hash(ht)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to read destination hash")
+		}
+		if srcHash != "" && dstHash != "" && srcHash != dstHash {
+			// remove object
+			err = o.Remove()
+			if err != nil {
+				fs.Errorf(o, "Failed to remove corrupted object: %v", err)
+			}
+			return nil, errors.Errorf("corrupted on transfer: %v crypted hash differ %q vs %q", ht, srcHash, dstHash)
+		}
+	}
+
 	return f.newObject(o), nil
 }
 
@@ -636,11 +669,11 @@ func (o *Object) Open(options ...fs.OpenOption) (rc io.ReadCloser, err error) {
 
 // Update in to the object with the modTime given of the given size
 func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
-	wrappedIn, err := o.f.cipher.EncryptData(in)
-	if err != nil {
-		return err
+	update := func(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
+		return o.Object, o.Object.Update(in, src, options...)
 	}
-	return o.Object.Update(wrappedIn, o.f.newObjectInfo(src))
+	_, err := o.f.put(in, src, options, update)
+	return err
 }
 
 // newDir returns a dir with the Name decrypted

backend/drive/drive.go

@@ -54,15 +54,18 @@ const (
 // Globals
 var (
 	// Flags
-	driveAuthOwnerOnly  = flags.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user.")
-	driveUseTrash       = flags.BoolP("drive-use-trash", "", true, "Send files to the trash instead of deleting permanently.")
-	driveSkipGdocs      = flags.BoolP("drive-skip-gdocs", "", false, "Skip google documents in all listings.")
-	driveSharedWithMe   = flags.BoolP("drive-shared-with-me", "", false, "Only show files that are shared with me")
-	driveTrashedOnly    = flags.BoolP("drive-trashed-only", "", false, "Only show files that are in the trash")
-	driveExtensions     = flags.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
-	driveUseCreatedDate = flags.BoolP("drive-use-created-date", "", false, "Use created date instead of modified date.")
-	driveListChunk      = flags.Int64P("drive-list-chunk", "", 1000, "Size of listing chunk 100-1000. 0 to disable.")
-	driveImpersonate    = flags.StringP("drive-impersonate", "", "", "Impersonate this user when using a service account.")
+	driveAuthOwnerOnly       = flags.BoolP("drive-auth-owner-only", "", false, "Only consider files owned by the authenticated user.")
+	driveUseTrash            = flags.BoolP("drive-use-trash", "", true, "Send files to the trash instead of deleting permanently.")
+	driveSkipGdocs           = flags.BoolP("drive-skip-gdocs", "", false, "Skip google documents in all listings.")
+	driveSharedWithMe        = flags.BoolP("drive-shared-with-me", "", false, "Only show files that are shared with me")
+	driveTrashedOnly         = flags.BoolP("drive-trashed-only", "", false, "Only show files that are in the trash")
+	driveExtensions          = flags.StringP("drive-formats", "", defaultExtensions, "Comma separated list of preferred formats for downloading Google docs.")
+	driveUseCreatedDate      = flags.BoolP("drive-use-created-date", "", false, "Use created date instead of modified date.")
+	driveListChunk           = flags.Int64P("drive-list-chunk", "", 1000, "Size of listing chunk 100-1000. 0 to disable.")
+	driveImpersonate         = flags.StringP("drive-impersonate", "", "", "Impersonate this user when using a service account.")
+	driveAlternateExport     = flags.BoolP("drive-alternate-export", "", false, "Use alternate export URLs for google documents export.")
+	driveAcknowledgeAbuse    = flags.BoolP("drive-acknowledge-abuse", "", false, "Set to allow files which return cannotDownloadAbusiveFile to be downloaded.")
+	driveKeepRevisionForever = flags.BoolP("drive-keep-revision-forever", "", false, "Keep new head revision forever.")
 	// chunkSize is the size of the chunks created during a resumable upload and should be a power of two.
 	// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
 	chunkSize         = fs.SizeSuffix(8 * 1024 * 1024)
@@ -292,8 +295,7 @@ func (f *Fs) list(dirID string, title string, directoriesOnly bool, filesOnly bo
 		searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
 		// Convert ／ to / for search
 		searchTitle = strings.Replace(searchTitle, "／", "/", -1)
-		// use contains to work around #1675
-		query = append(query, fmt.Sprintf("name contains '%s'", searchTitle))
+		query = append(query, fmt.Sprintf("name='%s'", searchTitle))
 	}
 	if directoriesOnly {
 		query = append(query, fmt.Sprintf("mimeType='%s'", driveFolderType))
@@ -341,7 +343,8 @@ OUTER:
 		for _, item := range files.Files {
 			// Convert / to ／ for listing purposes
 			item.Name = strings.Replace(item.Name, "/", "／", -1)
-			// skip items introduced by workaround (#1675)
+			// Check the case of items is correct since
+			// the `=` operator is case insensitive.
 			if title != "" && title != item.Name {
 				continue
 			}
@@ -399,7 +402,7 @@ func configTeamDrive(name string) error {
 	} else {
 		fmt.Printf("Change current team drive ID %q?\n", teamDrive)
 	}
-	if !config.Confirm() {
+	if !config.ConfirmWithDefault(false) {
 		return nil
 	}
 	client, err := createOAuthClient(name)
@@ -757,6 +760,18 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 			}
 			obj := o.(*Object)
 			obj.url = fmt.Sprintf("%sfiles/%s/export?mimeType=%s", f.svc.BasePath, item.Id, url.QueryEscape(exportMimeType))
+			if *driveAlternateExport {
+				switch item.MimeType {
+				case "application/vnd.google-apps.drawing":
+					obj.url = fmt.Sprintf("https://docs.google.com/drawings/d/%s/export/%s", item.Id, extension)
+				case "application/vnd.google-apps.document":
+					obj.url = fmt.Sprintf("https://docs.google.com/document/d/%s/export?format=%s", item.Id, extension)
+				case "application/vnd.google-apps.spreadsheet":
+					obj.url = fmt.Sprintf("https://docs.google.com/spreadsheets/d/%s/export?format=%s", item.Id, extension)
+				case "application/vnd.google-apps.presentation":
+					obj.url = fmt.Sprintf("https://docs.google.com/presentation/d/%s/export/%s", item.Id, extension)
+				}
+			}
 			obj.isDocument = true
 			obj.mimeType = exportMimeType
 			obj.bytes = -1
@@ -843,7 +858,7 @@ func (f *Fs) PutUnchecked(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOpt
 		// Make the API request to upload metadata and file data.
 		// Don't retry, return a retry error instead
 		err = f.pacer.CallNoRetry(func() (bool, error) {
-			info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do()
+			info, err = f.svc.Files.Create(createInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -1000,7 +1015,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 
 	var info *drive.File
 	err = o.fs.pacer.Call(func() (bool, error) {
-		info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).Do()
+		info, err = o.fs.svc.Files.Copy(srcObj.id, createInfo).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(f.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
 		return shouldRetry(err)
 	})
 	if err != nil {
@@ -1057,6 +1072,10 @@ func (f *Fs) CleanUp() error {
 
 // About gets quota information
 func (f *Fs) About() (*fs.Usage, error) {
+	if f.isTeamDrive {
+		// Teamdrives don't appear to have a usage API so just return empty
+		return &fs.Usage{}, nil
+	}
 	var about *drive.About
 	var err error
 	err = f.pacer.Call(func() (bool, error) {
@@ -1298,9 +1317,14 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
 		err = f.pacer.Call(func() (bool, error) {
 			changesCall := f.svc.Changes.List(pageToken).Fields("nextPageToken,newStartPageToken,changes(fileId,file(name,parents,mimeType))")
 			if *driveListChunk > 0 {
-				changesCall = changesCall.PageSize(*driveListChunk)
+				changesCall.PageSize(*driveListChunk)
 			}
-			changeList, err = changesCall.SupportsTeamDrives(f.isTeamDrive).Do()
+			if f.isTeamDrive {
+				changesCall.TeamDriveId(f.teamDriveID)
+				changesCall.SupportsTeamDrives(true)
+				changesCall.IncludeTeamDriveItems(true)
+			}
+			changeList, err = changesCall.Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -1323,7 +1347,12 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
 				continue
 			}
 
-			if change.File != nil && change.File.MimeType != driveFolderType {
+			if change.File != nil {
+				changeType := fs.EntryDirectory
+				if change.File.MimeType != driveFolderType {
+					changeType = fs.EntryObject
+				}
+
 				// translate the parent dir of this object
 				if len(change.File.Parents) > 0 {
 					if path, ok := f.dirCache.GetInv(change.File.Parents[0]); ok {
@@ -1334,10 +1363,10 @@ func (f *Fs) changeNotifyRunner(notifyFunc func(string, fs.EntryType), pollInter
 							path = change.File.Name
 						}
 						// this will now clear the actual file too
-						pathsToClear = append(pathsToClear, entryType{path: path, entryType: fs.EntryObject})
+						pathsToClear = append(pathsToClear, entryType{path: path, entryType: changeType})
 					}
 				} else { // a true root object that is changed
-					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: fs.EntryObject})
+					pathsToClear = append(pathsToClear, entryType{path: change.File.Name, entryType: changeType})
 				}
 			}
 		}
@@ -1571,11 +1600,39 @@ func (file *openFile) Close() (err error) {
 // Check it satisfies the interfaces
 var _ io.ReadCloser = &openFile{}
 
+// Checks to see if err is a googleapi.Error with of type what
+func isGoogleError(err error, what string) bool {
+	if gerr, ok := err.(*googleapi.Error); ok {
+		for _, error := range gerr.Errors {
+			if error.Reason == what {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // Open an object for read
 func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 	_, res, err := o.httpResponse("GET", options)
 	if err != nil {
-		return nil, errors.Wrap(err, "open file failed")
+		if isGoogleError(err, "cannotDownloadAbusiveFile") {
+			if *driveAcknowledgeAbuse {
+				// Retry acknowledging abuse
+				if strings.ContainsRune(o.url, '?') {
+					o.url += "&"
+				} else {
+					o.url += "?"
+				}
+				o.url += "acknowledgeAbuse=true"
+				_, res, err = o.httpResponse("GET", options)
+			} else {
+				err = errors.Wrap(err, "Use the --drive-acknowledge-abuse flag to download this file")
+			}
+		}
+		if err != nil {
+			return nil, errors.Wrap(err, "open file failed")
+		}
 	}
 	// If it is a document, update the size with what we are
 	// reading as it can change from the HEAD in the listing to
@@ -1609,7 +1666,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	if size == 0 || size < int64(driveUploadCutoff) {
 		// Don't retry, return a retry error instead
 		err = o.fs.pacer.CallNoRetry(func() (bool, error) {
-			info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).Do()
+			info, err = o.fs.svc.Files.Update(o.id, updateInfo).Media(in, googleapi.ContentType("")).Fields(googleapi.Field(partialFields)).SupportsTeamDrives(o.fs.isTeamDrive).KeepRevisionForever(*driveKeepRevisionForever).Do()
 			return shouldRetry(err)
 		})
 		if err != nil {
@@ -1656,6 +1713,11 @@ func (o *Object) MimeType() string {
 	return o.mimeType
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1673,4 +1735,5 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/drive/upload.go

@@ -30,9 +30,6 @@ import (
 const (
 	// statusResumeIncomplete is the code returned by the Google uploader when the transfer is not yet complete.
 	statusResumeIncomplete = 308
-
-	// Number of times to try each chunk
-	maxTries = 10
 )
 
 // resumableUpload is used by the generated APIs to provide resumable uploads.
@@ -61,6 +58,9 @@ func (f *Fs) Upload(in io.Reader, size int64, contentType string, fileID string,
 	if f.isTeamDrive {
 		params.Set("supportsTeamDrives", "true")
 	}
+	if *driveKeepRevisionForever {
+		params.Set("keepRevisionForever", "true")
+	}
 	urls := "https://www.googleapis.com/upload/drive/v3/files"
 	method := "POST"
 	if fileID != "" {
@@ -192,7 +192,7 @@ func (rx *resumableUpload) transferChunk(start int64, chunk io.ReadSeeker, chunk
 }
 
 // Upload uploads the chunks from the input
-// It retries each chunk maxTries times (with a pause of uploadPause between attempts).
+// It retries each chunk using the pacer and --low-level-retries
 func (rx *resumableUpload) Upload() (*drive.File, error) {
 	start := int64(0)
 	var StatusCode int

backend/dropbox/dropbox.go

@@ -31,6 +31,7 @@ import (
 	"time"
 
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/common"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/sharing"
 	"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
@@ -128,11 +129,12 @@ type Fs struct {
 	root           string         // the path we are working on
 	features       *fs.Features   // optional features
 	srv            files.Client   // the connection to the dropbox server
-	sharingClient  sharing.Client // as above, but for generating sharing links
+	sharing        sharing.Client // as above, but for generating sharing links
 	users          users.Client   // as above, but for accessing user information
 	slashRoot      string         // root with "/" prefix, lowercase
 	slashRootSlash string         // root with "/" prefix and postfix, lowercase
 	pacer          *pacer.Pacer   // To pace the API calls
+	ns             string         // The namespace we are using or "" for none
 }
 
 // Object describes a dropbox object
@@ -202,24 +204,21 @@ func NewFs(name, root string) (fs.Fs, error) {
 
 	oAuthClient, _, err := oauthutil.NewClient(name, dropboxConfig)
 	if err != nil {
-		log.Fatalf("Failed to configure dropbox: %v", err)
+		return nil, errors.Wrap(err, "failed to configure dropbox")
 	}
 
-	config := dropbox.Config{
-		LogLevel: dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
-		Client:   oAuthClient,    // maybe???
-	}
-	srv := files.New(config)
-	sharingClient := sharing.New(config)
-	users := users.New(config)
-
 	f := &Fs{
-		name:          name,
-		srv:           srv,
-		sharingClient: sharingClient,
-		users:         users,
-		pacer:         pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
+		name:  name,
+		pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 	}
+	config := dropbox.Config{
+		LogLevel:        dropbox.LogOff, // logging in the SDK: LogOff, LogDebug, LogInfo
+		Client:          oAuthClient,    // maybe???
+		HeaderGenerator: f.headerGenerator,
+	}
+	f.srv = files.New(config)
+	f.sharing = sharing.New(config)
+	f.users = users.New(config)
 	f.features = (&fs.Features{
 		CaseInsensitive:         true,
 		ReadMimeType:            true,
@@ -227,6 +226,27 @@ func NewFs(name, root string) (fs.Fs, error) {
 	}).Fill(f)
 	f.setRoot(root)
 
+	// If root starts with / then use the actual root
+	if strings.HasPrefix(root, "/") {
+		var acc *users.FullAccount
+		err = f.pacer.Call(func() (bool, error) {
+			acc, err = f.users.GetCurrentAccount()
+			return shouldRetry(err)
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "get current account failed")
+		}
+		switch x := acc.RootInfo.(type) {
+		case *common.TeamRootInfo:
+			f.ns = x.RootNamespaceId
+		case *common.UserRootInfo:
+			f.ns = x.RootNamespaceId
+		default:
+			return nil, errors.Errorf("unknown RootInfo type %v %T", acc.RootInfo, acc.RootInfo)
+		}
+		fs.Debugf(f, "Using root namespace %q", f.ns)
+	}
+
 	// See if the root is actually an object
 	_, err = f.getFileMetadata(f.slashRoot)
 	if err == nil {
@@ -241,6 +261,16 @@ func NewFs(name, root string) (fs.Fs, error) {
 	return f, nil
 }
 
+// headerGenerator for dropbox sdk
+func (f *Fs) headerGenerator(hostType string, style string, namespace string, route string) map[string]string {
+	if f.ns == "" {
+		return map[string]string{}
+	}
+	return map[string]string{
+		"Dropbox-API-Path-Root": `{".tag": "namespace_id", "namespace_id": "` + f.ns + `"}`,
+	}
+}
+
 // Sets root in f
 func (f *Fs) setRoot(root string) {
 	f.root = strings.Trim(root, "/")
@@ -419,21 +449,6 @@ func (f *Fs) List(dir string) (entries fs.DirEntries, err error) {
 	return entries, nil
 }
 
-// A read closer which doesn't close the input
-type readCloser struct {
-	in io.Reader
-}
-
-// Read bytes from the object - see io.Reader
-func (rc *readCloser) Read(p []byte) (n int, err error) {
-	return rc.in.Read(p)
-}
-
-// Dummy close function
-func (rc *readCloser) Close() error {
-	return nil
-}
-
 // Put the object
 //
 // Copy the reader in to the new object which is returned
@@ -651,7 +666,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 	}
 	var linkRes sharing.IsSharedLinkMetadata
 	err = f.pacer.Call(func() (bool, error) {
-		linkRes, err = f.sharingClient.CreateSharedLinkWithSettings(&createArg)
+		linkRes, err = f.sharing.CreateSharedLinkWithSettings(&createArg)
 		return shouldRetry(err)
 	})
 
@@ -663,7 +678,7 @@ func (f *Fs) PublicLink(remote string) (link string, err error) {
 		}
 		var listRes *sharing.ListSharedLinksResult
 		err = f.pacer.Call(func() (bool, error) {
-			listRes, err = f.sharingClient.ListSharedLinks(&listArg)
+			listRes, err = f.sharing.ListSharedLinks(&listArg)
 			return shouldRetry(err)
 		})
 		if err != nil {

backend/ftp/ftp.go

@@ -480,6 +480,8 @@ func (f *Fs) mkdir(abspath string) error {
 		switch errX.Code {
 		case ftp.StatusFileUnavailable: // dir already exists: see issue #2181
 			err = nil
+		case 521: // dir already exists: error number according to RFC 959: issue #2363
+			err = nil
 		}
 	}
 	return err

backend/googlecloudstorage/googlecloudstorage.go

@@ -31,10 +31,12 @@ import (
 	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/flags"
 	"github.com/ncw/rclone/fs/config/obscure"
+	"github.com/ncw/rclone/fs/fserrors"
 	"github.com/ncw/rclone/fs/fshttp"
 	"github.com/ncw/rclone/fs/hash"
 	"github.com/ncw/rclone/fs/walk"
 	"github.com/ncw/rclone/lib/oauthutil"
+	"github.com/ncw/rclone/lib/pacer"
 	"github.com/pkg/errors"
 	"golang.org/x/oauth2"
 	"golang.org/x/oauth2/google"
@@ -49,6 +51,7 @@ const (
 	timeFormatOut               = "2006-01-02T15:04:05.000000000Z07:00"
 	metaMtime                   = "mtime" // key to store mtime under in metadata
 	listChunks                  = 1000    // chunk size to read directory listings
+	minSleep                    = 10 * time.Millisecond
 )
 
 var (
@@ -219,6 +222,7 @@ type Fs struct {
 	bucketACL     string           // used when creating new buckets
 	location      string           // location of new buckets
 	storageClass  string           // storage class of new buckets
+	pacer         *pacer.Pacer     // To pace the API calls
 }
 
 // Object describes a storage object
@@ -262,6 +266,30 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
+// shouldRetry determines whehter a given err rates being retried
+func shouldRetry(err error) (again bool, errOut error) {
+	again = false
+	if err != nil {
+		if fserrors.ShouldRetry(err) {
+			again = true
+		} else {
+			switch gerr := err.(type) {
+			case *googleapi.Error:
+				if gerr.Code >= 500 && gerr.Code < 600 {
+					// All 5xx errors should be retried
+					again = true
+				} else if len(gerr.Errors) > 0 {
+					reason := gerr.Errors[0].Reason
+					if reason == "rateLimitExceeded" || reason == "userRateLimitExceeded" {
+						again = true
+					}
+				}
+			}
+		}
+	}
+	return again, err
+}
+
 // Pattern to match a storage path
 var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
 
@@ -327,6 +355,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 		bucketACL:     config.FileGet(name, "bucket_acl"),
 		location:      config.FileGet(name, "location"),
 		storageClass:  config.FileGet(name, "storage_class"),
+		pacer:         pacer.New().SetMinSleep(minSleep).SetPacer(pacer.GoogleDrivePacer),
 	}
 	f.features = (&fs.Features{
 		ReadMimeType:  true,
@@ -356,7 +385,10 @@ func NewFs(name, root string) (fs.Fs, error) {
 	if f.root != "" {
 		f.root += "/"
 		// Check to see if the object exists
-		_, err = f.svc.Objects.Get(bucket, directory).Do()
+		err = f.pacer.Call(func() (bool, error) {
+			_, err = f.svc.Objects.Get(bucket, directory).Do()
+			return shouldRetry(err)
+		})
 		if err == nil {
 			f.root = path.Dir(directory)
 			if f.root == "." {
@@ -404,7 +436,7 @@ type listFn func(remote string, object *storage.Object, isDirectory bool) error
 // dir is the starting directory, "" for root
 //
 // Set recurse to read sub directories
-func (f *Fs) list(dir string, recurse bool, fn listFn) error {
+func (f *Fs) list(dir string, recurse bool, fn listFn) (err error) {
 	root := f.root
 	rootLength := len(root)
 	if dir != "" {
@@ -415,7 +447,11 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 		list = list.Delimiter("/")
 	}
 	for {
-		objects, err := list.Do()
+		var objects *storage.Objects
+		err = f.pacer.Call(func() (bool, error) {
+			objects, err = list.Do()
+			return shouldRetry(err)
+		})
 		if err != nil {
 			if gErr, ok := err.(*googleapi.Error); ok {
 				if gErr.Code == http.StatusNotFound {
@@ -444,7 +480,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 			remote := object.Name[rootLength:]
 			// is this a directory marker?
 			if (strings.HasSuffix(remote, "/") || remote == "") && object.Size == 0 {
-				if recurse {
+				if recurse && remote != "" {
 					// add a directory in if --fast-list since will have no prefixes
 					err = fn(remote[:len(remote)-1], object, true)
 					if err != nil {
@@ -519,7 +555,11 @@ func (f *Fs) listBuckets(dir string) (entries fs.DirEntries, err error) {
 	}
 	listBuckets := f.svc.Buckets.List(f.projectNumber).MaxResults(listChunks)
 	for {
-		buckets, err := listBuckets.Do()
+		var buckets *storage.Buckets
+		err = f.pacer.Call(func() (bool, error) {
+			buckets, err = listBuckets.Do()
+			return shouldRetry(err)
+		})
 		if err != nil {
 			return nil, err
 		}
@@ -607,7 +647,7 @@ func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption
 }
 
 // Mkdir creates the bucket if it doesn't exist
-func (f *Fs) Mkdir(dir string) error {
+func (f *Fs) Mkdir(dir string) (err error) {
 	f.bucketOKMu.Lock()
 	defer f.bucketOKMu.Unlock()
 	if f.bucketOK {
@@ -615,7 +655,11 @@ func (f *Fs) Mkdir(dir string) error {
 	}
 	// List something from the bucket to see if it exists.  Doing it like this enables the use of a
 	// service account that only has the "Storage Object Admin" role.  See #2193 for details.
-	_, err := f.svc.Objects.List(f.bucket).MaxResults(1).Do()
+
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.svc.Objects.List(f.bucket).MaxResults(1).Do()
+		return shouldRetry(err)
+	})
 	if err == nil {
 		// Bucket already exists
 		f.bucketOK = true
@@ -637,7 +681,10 @@ func (f *Fs) Mkdir(dir string) error {
 		Location:     f.location,
 		StorageClass: f.storageClass,
 	}
-	_, err = f.svc.Buckets.Insert(f.projectNumber, &bucket).PredefinedAcl(f.bucketACL).Do()
+	err = f.pacer.Call(func() (bool, error) {
+		_, err = f.svc.Buckets.Insert(f.projectNumber, &bucket).PredefinedAcl(f.bucketACL).Do()
+		return shouldRetry(err)
+	})
 	if err == nil {
 		f.bucketOK = true
 	}
@@ -648,13 +695,16 @@ func (f *Fs) Mkdir(dir string) error {
 //
 // Returns an error if it isn't empty: Error 409: The bucket you tried
 // to delete was not empty.
-func (f *Fs) Rmdir(dir string) error {
+func (f *Fs) Rmdir(dir string) (err error) {
 	f.bucketOKMu.Lock()
 	defer f.bucketOKMu.Unlock()
 	if f.root != "" || dir != "" {
 		return nil
 	}
-	err := f.svc.Buckets.Delete(f.bucket).Do()
+	err = f.pacer.Call(func() (bool, error) {
+		err = f.svc.Buckets.Delete(f.bucket).Do()
+		return shouldRetry(err)
+	})
 	if err == nil {
 		f.bucketOK = false
 	}
@@ -696,7 +746,11 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) {
 	srcObject := srcObj.fs.root + srcObj.remote
 	dstBucket := f.bucket
 	dstObject := f.root + remote
-	newObject, err := f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
+	var newObject *storage.Object
+	err = f.pacer.Call(func() (bool, error) {
+		newObject, err = f.svc.Objects.Copy(srcBucket, srcObject, dstBucket, dstObject, nil).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -784,7 +838,11 @@ func (o *Object) readMetaData() (err error) {
 	if !o.modTime.IsZero() {
 		return nil
 	}
-	object, err := o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
+	var object *storage.Object
+	err = o.fs.pacer.Call(func() (bool, error) {
+		object, err = o.fs.svc.Objects.Get(o.fs.bucket, o.fs.root+o.remote).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		if gErr, ok := err.(*googleapi.Error); ok {
 			if gErr.Code == http.StatusNotFound {
@@ -818,14 +876,18 @@ func metadataFromModTime(modTime time.Time) map[string]string {
 }
 
 // SetModTime sets the modification time of the local fs object
-func (o *Object) SetModTime(modTime time.Time) error {
+func (o *Object) SetModTime(modTime time.Time) (err error) {
 	// This only adds metadata so will perserve other metadata
 	object := storage.Object{
 		Bucket:   o.fs.bucket,
 		Name:     o.fs.root + o.remote,
 		Metadata: metadataFromModTime(modTime),
 	}
-	newObject, err := o.fs.svc.Objects.Patch(o.fs.bucket, o.fs.root+o.remote, &object).Do()
+	var newObject *storage.Object
+	err = o.fs.pacer.Call(func() (bool, error) {
+		newObject, err = o.fs.svc.Objects.Patch(o.fs.bucket, o.fs.root+o.remote, &object).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return err
 	}
@@ -845,7 +907,17 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 		return nil, err
 	}
 	fs.OpenOptionAddHTTPHeaders(req.Header, options)
-	res, err := o.fs.client.Do(req)
+	var res *http.Response
+	err = o.fs.pacer.Call(func() (bool, error) {
+		res, err = o.fs.client.Do(req)
+		if err == nil {
+			err = googleapi.CheckResponse(res)
+			if err != nil {
+				_ = res.Body.Close() // ignore error
+			}
+		}
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return nil, err
 	}
@@ -874,7 +946,11 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 		Updated:     modTime.Format(timeFormatOut), // Doesn't get set
 		Metadata:    metadataFromModTime(modTime),
 	}
-	newObject, err := o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.objectACL).Do()
+	var newObject *storage.Object
+	err = o.fs.pacer.CallNoRetry(func() (bool, error) {
+		newObject, err = o.fs.svc.Objects.Insert(o.fs.bucket, &object).Media(in, googleapi.ContentType("")).Name(object.Name).PredefinedAcl(o.fs.objectACL).Do()
+		return shouldRetry(err)
+	})
 	if err != nil {
 		return err
 	}
@@ -884,8 +960,12 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 }
 
 // Remove an object
-func (o *Object) Remove() error {
-	return o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do()
+func (o *Object) Remove() (err error) {
+	err = o.fs.pacer.Call(func() (bool, error) {
+		err = o.fs.svc.Objects.Delete(o.fs.bucket, o.fs.root+o.remote).Do()
+		return shouldRetry(err)
+	})
+	return err
 }
 
 // MimeType of an Object if known, "" otherwise

backend/http/http_internal_test.go

@@ -151,6 +151,7 @@ func TestOpen(t *testing.T) {
 	fd, err := o.Open()
 	require.NoError(t, err)
 	data, err := ioutil.ReadAll(fd)
+	require.NoError(t, err)
 	require.NoError(t, fd.Close())
 	assert.Equal(t, "beetroot\n", string(data))
 
@@ -158,6 +159,7 @@ func TestOpen(t *testing.T) {
 	fd, err = o.Open(&fs.RangeOption{Start: 1, End: 5})
 	require.NoError(t, err)
 	data, err = ioutil.ReadAll(fd)
+	require.NoError(t, err)
 	require.NoError(t, fd.Close())
 	assert.Equal(t, "eetro", string(data))
 }

backend/hubic/auth.go

@@ -36,7 +36,7 @@ func (a *auth) Response(resp *http.Response) error {
 
 // The public storage URL - set Internal to true to read
 // internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string {
+func (a *auth) StorageUrl(Internal bool) string { // nolint
 	return a.f.credentials.Endpoint
 }
 
@@ -46,7 +46,7 @@ func (a *auth) Token() string {
 }
 
 // The CDN url if available
-func (a *auth) CdnUrl() string {
+func (a *auth) CdnUrl() string { // nolint
 	return ""
 }
 

backend/local/local.go

@@ -593,7 +593,6 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	o.fs.objectHashesMu.Unlock()
 
 	if !o.modTime.Equal(oldtime) || oldsize != o.size || hashes == nil {
-		hashes = make(map[hash.Type]string)
 		in, err := os.Open(o.path)
 		if err != nil {
 			return "", errors.Wrap(err, "hash: failed to open")
@@ -643,11 +642,6 @@ func (o *Object) Storable() bool {
 		}
 	}
 	mode := o.mode
-	// On windows a file with os.ModeSymlink represents a file with reparse points
-	if runtime.GOOS == "windows" && (mode&os.ModeSymlink) != 0 {
-		fs.Debugf(o, "Clearing symlink bit to allow a file with reparse points to be copied")
-		mode &^= os.ModeSymlink
-	}
 	if mode&os.ModeSymlink != 0 {
 		if !*skipSymlinks {
 			fs.Logf(o, "Can't follow symlink without -L/--copy-links")

backend/local/local_internal_test.go

@@ -44,9 +44,11 @@ func TestUpdatingCheck(t *testing.T) {
 	}
 
 	fi, err := fd.Stat()
+	require.NoError(t, err)
 	o := &Object{size: fi.Size(), modTime: fi.ModTime()}
 	wrappedFd := readers.NewLimitedReadCloser(fd, -1)
 	hash, err := hash.NewMultiHasherTypes(hash.Supported)
+	require.NoError(t, err)
 	in := localOpenFile{
 		o:    o,
 		in:   wrappedFd,

backend/mega/mega.go

@@ -19,12 +19,12 @@ import (
 	"fmt"
 	"io"
 	"path"
-	"regexp"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/fs/config"
 	"github.com/ncw/rclone/fs/config/flags"
 	"github.com/ncw/rclone/fs/config/obscure"
 	"github.com/ncw/rclone/fs/fshttp"
@@ -38,6 +38,7 @@ import (
 const (
 	minSleep      = 10 * time.Millisecond
 	maxSleep      = 2 * time.Second
+	eventWaitTime = 500 * time.Millisecond
 	decayConstant = 2    // bigger for slower decay, exponential
 	useTrash      = true // FIXME make configurable - rclone global
 )
@@ -114,9 +115,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a mega path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an mega 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -148,8 +146,8 @@ func (f *Fs) readMetaDataForPath(remote string) (info *mega.Node, err error) {
 
 // NewFs constructs an Fs from the path, container:path
 func NewFs(name, root string) (fs.Fs, error) {
-	user := fs.ConfigFileGet(name, "user")
-	pass := fs.ConfigFileGet(name, "pass")
+	user := config.FileGet(name, "user")
+	pass := config.FileGet(name, "pass")
 	if pass != "" {
 		var err error
 		pass, err = obscure.Reveal(pass)
@@ -573,6 +571,8 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 		}
 	}
 
+	waitEvent := f.srv.WaitEventsStart()
+
 	err = f.deleteNode(dirNode)
 	if err != nil {
 		return errors.Wrap(err, "delete directory node failed")
@@ -582,7 +582,8 @@ func (f *Fs) purgeCheck(dir string, check bool) error {
 	if dirNode == rootNode {
 		f.clearRoot()
 	}
-	time.Sleep(100 * time.Millisecond) // FIXME give the callback a chance
+
+	f.srv.WaitEvents(waitEvent, eventWaitTime)
 	return nil
 }
 
@@ -656,6 +657,8 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 		}
 	}
 
+	waitEvent := f.srv.WaitEventsStart()
+
 	// rename the object if required
 	if srcLeaf != dstLeaf {
 		//log.Printf("rename %q to %q", srcLeaf, dstLeaf)
@@ -668,7 +671,8 @@ func (f *Fs) move(dstRemote string, srcFs *Fs, srcRemote string, info *mega.Node
 		}
 	}
 
-	time.Sleep(100 * time.Millisecond) // FIXME give events a chance...
+	f.srv.WaitEvents(waitEvent, eventWaitTime)
+
 	return nil
 }
 
@@ -1114,6 +1118,11 @@ func (o *Object) Remove() error {
 	return nil
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.info.GetHash()
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1126,4 +1135,5 @@ var (
 	_ fs.MergeDirser     = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/onedrive/onedrive.go

@@ -12,7 +12,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -98,7 +97,15 @@ func init() {
 					log.Fatalf("Failed to configure token: %v", err)
 				}
 			} else {
-				err := oauthutil.Config("onedrive", name, oauthBusinessConfig, oauthBusinessResource)
+				err := oauthutil.ConfigErrorCheck("onedrive", name, func(req *http.Request) oauthutil.AuthError {
+					var resp oauthutil.AuthError
+
+					resp.Name = req.URL.Query().Get("error")
+					resp.Code = strings.Split(req.URL.Query().Get("error_description"), ":")[0] // error_description begins with XXXXXXXXXXXX:
+					resp.Description = strings.Join(strings.Split(req.URL.Query().Get("error_description"), ":")[1:], ":")
+					resp.HelpURL = "https://rclone.org/onedrive/#troubleshooting"
+					return resp
+				}, oauthBusinessConfig, oauthBusinessResource)
 				if err != nil {
 					log.Fatalf("Failed to configure token: %v", err)
 					return
@@ -280,9 +287,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a one drive path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an one drive 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -1306,6 +1310,11 @@ func (o *Object) MimeType() string {
 	return o.mimeType
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs     = (*Fs)(nil)
@@ -1317,4 +1326,5 @@ var (
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
 	_ fs.MimeTyper       = &Object{}
+	_ fs.IDer            = &Object{}
 )

backend/onedrive/quickxorhash/quickxorhash.go

@@ -44,7 +44,6 @@ const (
 	Size           = 20
 	bitsInLastCell = 32
 	shift          = 11
-	threshold      = 600
 	widthInBits    = 8 * Size
 	dataSize       = (widthInBits-1)/64 + 1
 )

backend/opendrive/opendrive_test.go

@@ -0,0 +1,17 @@
+// Test Opendrive filesystem interface
+package opendrive_test
+
+import (
+	"testing"
+
+	"github.com/ncw/rclone/backend/opendrive"
+	"github.com/ncw/rclone/fstest/fstests"
+)
+
+// TestIntegration runs integration tests against the remote
+func TestIntegration(t *testing.T) {
+	fstests.Run(t, &fstests.Opt{
+		RemoteName: "TestOpenDrive:",
+		NilObject:  (*opendrive.Object)(nil),
+	})
+}

backend/opendrive/replace.go

@@ -0,0 +1,78 @@
+/*
+Translate file names for OpenDrive
+
+OpenDrive reserved characters
+
+The following characters are OpenDrive reserved characters, and can't
+be used in OpenDrive folder and file names.
+
+\ / : * ? " < > |
+
+OpenDrive files and folders can't have leading or trailing spaces also.
+
+*/
+
+package opendrive
+
+import (
+	"regexp"
+	"strings"
+)
+
+// charMap holds replacements for characters
+//
+// OpenDrive has a restricted set of characters compared to other cloud
+// storage systems, so we to map these to the FULLWIDTH unicode
+// equivalents
+//
+// http://unicode-search.net/unicode-namesearch.pl?term=SOLIDUS
+var (
+	charMap = map[rune]rune{
+		'\\': '＼', // FULLWIDTH REVERSE SOLIDUS
+		':':  '：', // FULLWIDTH COLON
+		'*':  '＊', // FULLWIDTH ASTERISK
+		'?':  '？', // FULLWIDTH QUESTION MARK
+		'"':  '＂', // FULLWIDTH QUOTATION MARK
+		'<':  '＜', // FULLWIDTH LESS-THAN SIGN
+		'>':  '＞', // FULLWIDTH GREATER-THAN SIGN
+		'|':  '｜', // FULLWIDTH VERTICAL LINE
+		' ':  '␠', // SYMBOL FOR SPACE
+	}
+	fixStartingWithSpace = regexp.MustCompile(`(/|^) `)
+	fixEndingWithSpace   = regexp.MustCompile(` (/|$)`)
+	invCharMap           map[rune]rune
+)
+
+func init() {
+	// Create inverse charMap
+	invCharMap = make(map[rune]rune, len(charMap))
+	for k, v := range charMap {
+		invCharMap[v] = k
+	}
+}
+
+// replaceReservedChars takes a path and substitutes any reserved
+// characters in it
+func replaceReservedChars(in string) string {
+	// Filenames can't start with space
+	in = fixStartingWithSpace.ReplaceAllString(in, "$1"+string(charMap[' ']))
+	// Filenames can't end with space
+	in = fixEndingWithSpace.ReplaceAllString(in, string(charMap[' '])+"$1")
+	return strings.Map(func(c rune) rune {
+		if replacement, ok := charMap[c]; ok && c != ' ' {
+			return replacement
+		}
+		return c
+	}, in)
+}
+
+// restoreReservedChars takes a path and undoes any substitutions
+// made by replaceReservedChars
+func restoreReservedChars(in string) string {
+	return strings.Map(func(c rune) rune {
+		if replacement, ok := invCharMap[c]; ok {
+			return replacement
+		}
+		return c
+	}, in)
+}

backend/opendrive/replace_test.go

@@ -0,0 +1,28 @@
+package opendrive
+
+import "testing"
+
+func TestReplace(t *testing.T) {
+	for _, test := range []struct {
+		in  string
+		out string
+	}{
+		{"", ""},
+		{"abc 123", "abc 123"},
+		{`\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~`},
+		{`\*<>?:|#%".~/\*<>?:|#%".~`, `＼＊＜＞？：｜#%＂.~/＼＊＜＞？：｜#%＂.~`},
+		{" leading space", "␠leading space"},
+		{" path/  leading spaces", "␠path/␠ leading spaces"},
+		{"trailing space ", "trailing space␠"},
+		{"trailing spaces  /path ", "trailing spaces ␠/path␠"},
+	} {
+		got := replaceReservedChars(test.in)
+		if got != test.out {
+			t.Errorf("replaceReservedChars(%q) want %q got %q", test.in, test.out, got)
+		}
+		got2 := restoreReservedChars(got)
+		if got2 != test.in {
+			t.Errorf("restoreReservedChars(%q) want %q got %q", got, test.in, got2)
+		}
+	}
+}

backend/opendrive/types.go

@@ -0,0 +1,214 @@
+package opendrive
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// Error describes an openDRIVE error response
+type Error struct {
+	Info struct {
+		Code    int    `json:"code"`
+		Message string `json:"message"`
+	} `json:"error"`
+}
+
+// Error statisfies the error interface
+func (e *Error) Error() string {
+	return fmt.Sprintf("%s (Error %d)", e.Info.Message, e.Info.Code)
+}
+
+// Account describes a OpenDRIVE account
+type Account struct {
+	Username string `json:"username"`
+	Password string `json:"passwd"`
+}
+
+// UserSessionInfo describes a OpenDRIVE session
+type UserSessionInfo struct {
+	Username string `json:"username"`
+	Password string `json:"passwd"`
+
+	SessionID          string          `json:"SessionID"`
+	UserName           string          `json:"UserName"`
+	UserFirstName      string          `json:"UserFirstName"`
+	UserLastName       string          `json:"UserLastName"`
+	AccType            string          `json:"AccType"`
+	UserLang           string          `json:"UserLang"`
+	UserID             string          `json:"UserID"`
+	IsAccountUser      json.RawMessage `json:"IsAccountUser"`
+	DriveName          string          `json:"DriveName"`
+	UserLevel          string          `json:"UserLevel"`
+	UserPlan           string          `json:"UserPlan"`
+	FVersioning        string          `json:"FVersioning"`
+	UserDomain         string          `json:"UserDomain"`
+	PartnerUsersDomain string          `json:"PartnerUsersDomain"`
+}
+
+// FolderList describes a OpenDRIVE listing
+type FolderList struct {
+	// DirUpdateTime    string   `json:"DirUpdateTime,string"`
+	Name             string   `json:"Name"`
+	ParentFolderID   string   `json:"ParentFolderID"`
+	DirectFolderLink string   `json:"DirectFolderLink"`
+	ResponseType     int      `json:"ResponseType"`
+	Folders          []Folder `json:"Folders"`
+	Files            []File   `json:"Files"`
+}
+
+// Folder describes a OpenDRIVE folder
+type Folder struct {
+	FolderID      string `json:"FolderID"`
+	Name          string `json:"Name"`
+	DateCreated   int    `json:"DateCreated"`
+	DirUpdateTime int    `json:"DirUpdateTime"`
+	Access        int    `json:"Access"`
+	DateModified  int64  `json:"DateModified"`
+	Shared        string `json:"Shared"`
+	ChildFolders  int    `json:"ChildFolders"`
+	Link          string `json:"Link"`
+	Encrypted     string `json:"Encrypted"`
+}
+
+type createFolder struct {
+	SessionID           string `json:"session_id"`
+	FolderName          string `json:"folder_name"`
+	FolderSubParent     string `json:"folder_sub_parent"`
+	FolderIsPublic      int64  `json:"folder_is_public"`      // (0 = private, 1 = public, 2 = hidden)
+	FolderPublicUpl     int64  `json:"folder_public_upl"`     // (0 = disabled, 1 = enabled)
+	FolderPublicDisplay int64  `json:"folder_public_display"` // (0 = disabled, 1 = enabled)
+	FolderPublicDnl     int64  `json:"folder_public_dnl"`     // (0 = disabled, 1 = enabled).
+}
+
+type createFolderResponse struct {
+	FolderID      string `json:"FolderID"`
+	Name          string `json:"Name"`
+	DateCreated   int    `json:"DateCreated"`
+	DirUpdateTime int    `json:"DirUpdateTime"`
+	Access        int    `json:"Access"`
+	DateModified  int    `json:"DateModified"`
+	Shared        string `json:"Shared"`
+	Description   string `json:"Description"`
+	Link          string `json:"Link"`
+}
+
+type moveCopyFolder struct {
+	SessionID     string `json:"session_id"`
+	FolderID      string `json:"folder_id"`
+	DstFolderID   string `json:"dst_folder_id"`
+	Move          string `json:"move"`
+	NewFolderName string `json:"new_folder_name"` // New name for destination folder.
+}
+
+type moveCopyFolderResponse struct {
+	FolderID string `json:"FolderID"`
+}
+
+type removeFolder struct {
+	SessionID string `json:"session_id"`
+	FolderID  string `json:"folder_id"`
+}
+
+// File describes a OpenDRIVE file
+type File struct {
+	FileID            string `json:"FileId"`
+	FileHash          string `json:"FileHash"`
+	Name              string `json:"Name"`
+	GroupID           int    `json:"GroupID"`
+	Extension         string `json:"Extension"`
+	Size              int64  `json:"Size,string"`
+	Views             string `json:"Views"`
+	Version           string `json:"Version"`
+	Downloads         string `json:"Downloads"`
+	DateModified      int64  `json:"DateModified,string"`
+	Access            string `json:"Access"`
+	Link              string `json:"Link"`
+	DownloadLink      string `json:"DownloadLink"`
+	StreamingLink     string `json:"StreamingLink"`
+	TempStreamingLink string `json:"TempStreamingLink"`
+	EditLink          string `json:"EditLink"`
+	ThumbLink         string `json:"ThumbLink"`
+	Password          string `json:"Password"`
+	EditOnline        int    `json:"EditOnline"`
+}
+
+type moveCopyFile struct {
+	SessionID         string `json:"session_id"`
+	SrcFileID         string `json:"src_file_id"`
+	DstFolderID       string `json:"dst_folder_id"`
+	Move              string `json:"move"`
+	OverwriteIfExists string `json:"overwrite_if_exists"`
+	NewFileName       string `json:"new_file_name"` // New name for destination file.
+}
+
+type moveCopyFileResponse struct {
+	FileID string `json:"FileID"`
+	Size   string `json:"Size"`
+}
+
+type createFile struct {
+	SessionID string `json:"session_id"`
+	FolderID  string `json:"folder_id"`
+	Name      string `json:"file_name"`
+}
+
+type createFileResponse struct {
+	FileID             string `json:"FileId"`
+	Name               string `json:"Name"`
+	GroupID            int    `json:"GroupID"`
+	Extension          string `json:"Extension"`
+	Size               string `json:"Size"`
+	Views              string `json:"Views"`
+	Downloads          string `json:"Downloads"`
+	DateModified       string `json:"DateModified"`
+	Access             string `json:"Access"`
+	Link               string `json:"Link"`
+	DownloadLink       string `json:"DownloadLink"`
+	StreamingLink      string `json:"StreamingLink"`
+	TempStreamingLink  string `json:"TempStreamingLink"`
+	DirUpdateTime      int    `json:"DirUpdateTime"`
+	TempLocation       string `json:"TempLocation"`
+	SpeedLimit         int    `json:"SpeedLimit"`
+	RequireCompression int    `json:"RequireCompression"`
+	RequireHash        int    `json:"RequireHash"`
+	RequireHashOnly    int    `json:"RequireHashOnly"`
+}
+
+type modTimeFile struct {
+	SessionID            string `json:"session_id"`
+	FileID               string `json:"file_id"`
+	FileModificationTime string `json:"file_modification_time"`
+}
+
+type openUpload struct {
+	SessionID string `json:"session_id"`
+	FileID    string `json:"file_id"`
+	Size      int64  `json:"file_size"`
+}
+
+type openUploadResponse struct {
+	TempLocation       string `json:"TempLocation"`
+	RequireCompression bool   `json:"RequireCompression"`
+	RequireHash        bool   `json:"RequireHash"`
+	RequireHashOnly    bool   `json:"RequireHashOnly"`
+	SpeedLimit         int    `json:"SpeedLimit"`
+}
+
+type closeUpload struct {
+	SessionID    string `json:"session_id"`
+	FileID       string `json:"file_id"`
+	Size         int64  `json:"file_size"`
+	TempLocation string `json:"temp_location"`
+}
+
+type closeUploadResponse struct {
+	FileID   string `json:"FileID"`
+	FileHash string `json:"FileHash"`
+	Size     int64  `json:"Size"`
+}
+
+type permissions struct {
+	SessionID    string `json:"session_id"`
+	FileID       string `json:"file_id"`
+	FileIsPublic int64  `json:"file_ispublic"`
+}

backend/pcloud/pcloud.go

@@ -17,7 +17,6 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -130,9 +129,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a pcloud path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
 // parsePath parses an pcloud 'url'
 func parsePath(path string) (root string) {
 	root = strings.Trim(path, "/")
@@ -1122,6 +1118,11 @@ func (o *Object) Remove() error {
 	})
 }
 
+// ID returns the ID of the Object if known, or "" if not
+func (o *Object) ID() string {
+	return o.id
+}
+
 // Check the interfaces are satisfied
 var (
 	_ fs.Fs              = (*Fs)(nil)
@@ -1133,4 +1134,5 @@ var (
 	_ fs.DirCacheFlusher = (*Fs)(nil)
 	_ fs.Abouter         = (*Fs)(nil)
 	_ fs.Object          = (*Object)(nil)
+	_ fs.IDer            = (*Object)(nil)
 )

backend/qingstor/upload.go

@@ -19,10 +19,10 @@ import (
 )
 
 const (
-	maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
-	maxMultiPartSize  = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
-	minMultiPartSize  = 1024 * 1024 * 4        // The minimum allowed part size when uploading a part to QingStor
-	maxMultiParts     = 10000                  // The maximum allowed number of parts in an multi-part upload
+	// maxSinglePartSize = 1024 * 1024 * 1024 * 5 // The maximum allowed size when uploading a single object to QingStor
+	// maxMultiPartSize = 1024 * 1024 * 1024 * 1 // The maximum allowed part size when uploading a part to QingStor
+	minMultiPartSize = 1024 * 1024 * 4 // The minimum allowed part size when uploading a part to QingStor
+	maxMultiParts    = 10000           // The maximum allowed number of parts in an multi-part upload
 )
 
 const (

backend/s3/s3.go

@@ -537,10 +537,11 @@ const (
 // Globals
 var (
 	// Flags
-	s3ACL             = flags.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
-	s3StorageClass    = flags.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA)")
-	s3ChunkSize       = fs.SizeSuffix(s3manager.MinUploadPartSize)
-	s3DisableChecksum = flags.BoolP("s3-disable-checksum", "", false, "Don't store MD5 checksum with object metadata")
+	s3ACL               = flags.StringP("s3-acl", "", "", "Canned ACL used when creating buckets and/or storing objects in S3")
+	s3StorageClass      = flags.StringP("s3-storage-class", "", "", "Storage class to use when uploading S3 objects (STANDARD|REDUCED_REDUNDANCY|STANDARD_IA|ONEZONE_IA)")
+	s3ChunkSize         = fs.SizeSuffix(s3manager.MinUploadPartSize)
+	s3DisableChecksum   = flags.BoolP("s3-disable-checksum", "", false, "Don't store MD5 checksum with object metadata")
+	s3UploadConcurrency = flags.IntP("s3-upload-concurrency", "", 2, "Concurrency for multipart uploads")
 )
 
 // Fs represents a remote s3 server
@@ -863,7 +864,7 @@ func (f *Fs) list(dir string, recurse bool, fn listFn) error {
 			remote := key[rootLength:]
 			// is this a directory marker?
 			if (strings.HasSuffix(remote, "/") || remote == "") && *object.Size == 0 {
-				if recurse {
+				if recurse && remote != "" {
 					// add a directory in if --fast-list since will have no prefixes
 					remote = remote[:len(remote)-1]
 					err = fn(remote, &s3.Object{Key: &remote}, true)
@@ -1352,11 +1353,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio
 	size := src.Size()
 
 	uploader := s3manager.NewUploader(o.fs.ses, func(u *s3manager.Uploader) {
-		u.Concurrency = 2
+		u.Concurrency = *s3UploadConcurrency
 		u.LeavePartsOnError = false
 		u.S3 = o.fs.c
 		u.PartSize = int64(s3ChunkSize)
-		u.PartSize = s3manager.MinUploadPartSize
 
 		if size == -1 {
 			// Make parts as small as possible while still being able to upload to the

backend/sftp/sftp.go

@@ -1,6 +1,6 @@
 // Package sftp provides a filesystem interface using github.com/pkg/sftp
 
-// +build !plan9,go1.8
+// +build !plan9,go1.9
 
 package sftp
 
@@ -41,6 +41,7 @@ var (
 
 	// Flags
 	sftpAskPassword = flags.BoolP("sftp-ask-password", "", false, "Allow asking for SFTP password when needed.")
+	sshPathOverride = flags.StringP("ssh-path-override", "", "", "Override path used by SSH connection.")
 )
 
 func init() {
@@ -75,7 +76,7 @@ func init() {
 			Optional: true,
 		}, {
 			Name:     "use_insecure_cipher",
-			Help:     "Enable the user of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
+			Help:     "Enable the use of the aes128-cbc cipher. This cipher is insecure and may allow plaintext data to be recovered by an attacker.",
 			Optional: true,
 			Examples: []fs.OptionExample{
 				{
@@ -124,12 +125,6 @@ type Object struct {
 	sha1sum *string     // Cached SHA1 checksum
 }
 
-// ObjectReader holds the sftp.File interface to a remote SFTP file opened for reading
-type ObjectReader struct {
-	object   *Object
-	sftpFile *sftp.File
-}
-
 // readCurrentUser finds the current user name or "" if not found
 func readCurrentUser() (userName string) {
 	usr, err := user.Current()
@@ -339,7 +334,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 
 	// Ask for password if none was defined and we're allowed to
 	if pass == "" && *sftpAskPassword {
-		fmt.Fprint(os.Stderr, "Enter SFTP password: ")
+		_, _ = fmt.Fprint(os.Stderr, "Enter SFTP password: ")
 		clearpass := config.ReadPassword()
 		sshConfig.Auth = append(sshConfig.Auth, ssh.Password(clearpass))
 	}
@@ -763,6 +758,9 @@ func (o *Object) Hash(r hash.Type) (string, error) {
 	session.Stdout = &stdout
 	session.Stderr = &stderr
 	escapedPath := shellEscape(o.path())
+	if *sshPathOverride != "" {
+		escapedPath = shellEscape(path.Join(*sshPathOverride, o.remote))
+	}
 	err = session.Run(hashCmd + " " + escapedPath)
 	if err != nil {
 		_ = session.Close()
@@ -873,15 +871,49 @@ func (o *Object) Storable() bool {
 	return o.mode.IsRegular()
 }
 
+// objectReader represents a file open for reading on the SFTP server
+type objectReader struct {
+	sftpFile   *sftp.File
+	pipeReader *io.PipeReader
+	done       chan struct{}
+}
+
+func newObjectReader(sftpFile *sftp.File) *objectReader {
+	pipeReader, pipeWriter := io.Pipe()
+	file := &objectReader{
+		sftpFile:   sftpFile,
+		pipeReader: pipeReader,
+		done:       make(chan struct{}),
+	}
+
+	go func() {
+		// Use sftpFile.WriteTo to pump data so that it gets a
+		// chance to build the window up.
+		_, err := sftpFile.WriteTo(pipeWriter)
+		// Close the pipeWriter so the pipeReader fails with
+		// the same error or EOF if err == nil
+		_ = pipeWriter.CloseWithError(err)
+		// signal that we've finished
+		close(file.done)
+	}()
+
+	return file
+}
+
 // Read from a remote sftp file object reader
-func (file *ObjectReader) Read(p []byte) (n int, err error) {
-	n, err = file.sftpFile.Read(p)
+func (file *objectReader) Read(p []byte) (n int, err error) {
+	n, err = file.pipeReader.Read(p)
 	return n, err
 }
 
 // Close a reader of a remote sftp file
-func (file *ObjectReader) Close() (err error) {
+func (file *objectReader) Close() (err error) {
+	// Close the sftpFile - this will likely cause the WriteTo to error
 	err = file.sftpFile.Close()
+	// Close the pipeReader so writes to the pipeWriter fail
+	_ = file.pipeReader.Close()
+	// Wait for the background process to finish
+	<-file.done
 	return err
 }
 
@@ -915,10 +947,7 @@ func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
 			return nil, errors.Wrap(err, "Open Seek failed")
 		}
 	}
-	in = readers.NewLimitedReadCloser(&ObjectReader{
-		object:   o,
-		sftpFile: sftpFile,
-	}, limit)
+	in = readers.NewLimitedReadCloser(newObjectReader(sftpFile), limit)
 	return in, nil
 }
 

backend/sftp/sftp_internal_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.8
+// +build !plan9,go1.9
 
 package sftp
 

backend/sftp/sftp_test.go

@@ -1,6 +1,6 @@
 // Test Sftp filesystem interface
 
-// +build !plan9,go1.8
+// +build !plan9,go1.9
 
 package sftp_test
 

backend/sftp/sftp_unsupported.go

@@ -1,6 +1,6 @@
 // Build for sftp for unsupported platforms to stop go complaining
 // about "no buildable Go source files "
 
-// +build plan9 !go1.8
+// +build plan9 !go1.9
 
 package sftp

backend/sftp/stringlock.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.8
+// +build !plan9,go1.9
 
 package sftp
 

backend/sftp/stringlock_test.go

@@ -1,4 +1,4 @@
-// +build !plan9,go1.8
+// +build !plan9,go1.9
 
 package sftp
 

backend/swift/auth.go

@@ -44,7 +44,7 @@ func (a *auth) Response(resp *http.Response) error {
 
 // The public storage URL - set Internal to true to read
 // internal/service net URL
-func (a *auth) StorageUrl(Internal bool) string {
+func (a *auth) StorageUrl(Internal bool) string { // nolint
 	if a.storageURL != "" {
 		return a.storageURL
 	}
@@ -66,7 +66,7 @@ func (a *auth) Token() string {
 }
 
 // The CDN url if available
-func (a *auth) CdnUrl() string {
+func (a *auth) CdnUrl() string { // nolint
 	if a.parentAuth == nil {
 		return ""
 	}

backend/swift/swift.go

@@ -221,7 +221,7 @@ func swiftConnection(name string) (*swift.Connection, error) {
 			return nil, errors.Wrap(err, "failed to read environment variables")
 		}
 	}
-	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken
+	StorageUrl, AuthToken := c.StorageUrl, c.AuthToken // nolint
 	if !c.Authenticated() {
 		if c.UserName == "" && c.UserId == "" {
 			return nil, errors.New("user name or user id not found for authentication (and no storage_url+auth_token is provided)")
@@ -391,7 +391,8 @@ func (f *Fs) list(dir string, recurse bool, fn addEntryFn) error {
 			err = fn(d)
 		} else {
 			// newObjectWithInfo does a full metadata read on 0 size objects which might be dynamic large objects
-			o, err := f.newObjectWithInfo(remote, object)
+			var o fs.Object
+			o, err = f.newObjectWithInfo(remote, object)
 			if err != nil {
 				return err
 			}
@@ -847,7 +848,7 @@ func urlEncode(str string) string {
 // container.  It returns a string which prefixes current segments.
 func (o *Object) updateChunks(in0 io.Reader, headers swift.Headers, size int64, contentType string) (string, error) {
 	// Create the segmentsContainer if it doesn't exist
-	var err error = swift.ContainerNotFound
+	var err error
 	_, _, err = o.fs.c.Container(o.fs.segmentsContainer)
 	if err == swift.ContainerNotFound {
 		err = o.fs.c.ContainerCreate(o.fs.segmentsContainer, nil)

backend/webdav/api/types.go

@@ -5,6 +5,7 @@ import (
 	"encoding/xml"
 	"regexp"
 	"strconv"
+	"strings"
 	"time"
 )
 
@@ -65,8 +66,8 @@ type Prop struct {
 	Modified Time      `xml:"DAV: prop>getlastmodified,omitempty"`
 }
 
-// Parse a status of the form "HTTP/1.1 200 OK",
-var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)\s+(.*)$`)
+// Parse a status of the form "HTTP/1.1 200 OK" or "HTTP/1.1 200"
+var parseStatus = regexp.MustCompile(`^HTTP/[0-9.]+\s+(\d+)`)
 
 // StatusOK examines the Status and returns an OK flag
 func (p *Prop) StatusOK() bool {
@@ -75,7 +76,7 @@ func (p *Prop) StatusOK() bool {
 		return true
 	}
 	match := parseStatus.FindStringSubmatch(p.Status[0])
-	if len(match) < 3 {
+	if len(match) < 2 {
 		return false
 	}
 	code, err := strconv.Atoi(match[1])
@@ -109,16 +110,20 @@ type Error struct {
 
 // Error returns a string for the error and statistifes the error interface
 func (e *Error) Error() string {
+	var out []string
 	if e.Message != "" {
-		return e.Message
+		out = append(out, e.Message)
 	}
 	if e.Exception != "" {
-		return e.Exception
+		out = append(out, e.Exception)
 	}
 	if e.Status != "" {
-		return e.Status
+		out = append(out, e.Status)
 	}
-	return "Webdav Error"
+	if len(out) == 0 {
+		return "Webdav Error"
+	}
+	return strings.Join(out, ": ")
 }
 
 // Time represents represents date and time information for the

backend/webdav/webdav.go

@@ -19,12 +19,12 @@ package webdav
 // For example the ownCloud WebDAV server does it that way.
 
 import (
+	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"path"
-	"regexp"
 	"strings"
 	"time"
 
@@ -87,6 +87,10 @@ func init() {
 			Help:       "Password.",
 			Optional:   true,
 			IsPassword: true,
+		}, {
+			Name:     "bearer_token",
+			Help:     "Bearer token instead of user/pass (eg a Macaroon)",
+			Optional: true,
 		}},
 	})
 }
@@ -117,7 +121,6 @@ type Object struct {
 	hasMetaData bool      // whether info below has been set
 	size        int64     // size of the object
 	modTime     time.Time // modification time of the object
-	id          string    // ID of the object
 	sha1        string    // SHA-1 of the object content
 }
 
@@ -143,15 +146,6 @@ func (f *Fs) Features() *fs.Features {
 	return f.features
 }
 
-// Pattern to match a webdav path
-var matcher = regexp.MustCompile(`^([^/]*)(.*)$`)
-
-// parsePath parses an webdav 'url'
-func parsePath(path string) (root string) {
-	root = strings.Trim(path, "/")
-	return
-}
-
 // retryErrorCodes is a slice of error codes that we will retry
 var retryErrorCodes = []int{
 	429, // Too Many Requests.
@@ -189,6 +183,10 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Prop, err error) {
 	opts := rest.Opts{
 		Method: "PROPFIND",
 		Path:   f.filePath(path),
+		ExtraHeaders: map[string]string{
+			"Depth": "1",
+		},
+		NoRedirect: true,
 	}
 	var result api.Multistatus
 	var resp *http.Response
@@ -198,7 +196,13 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Prop, err error) {
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// does not exist
-		if apiErr.StatusCode == http.StatusNotFound {
+		switch apiErr.StatusCode {
+		case http.StatusNotFound:
+			return nil, fs.ErrorObjectNotFound
+		case http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther:
+			// Some sort of redirect - go doesn't deal with these properly (it resets
+			// the method to GET).  However we can assume that if it was redirected the
+			// object was not found.
 			return nil, fs.ErrorObjectNotFound
 		}
 	}
@@ -220,11 +224,16 @@ func (f *Fs) readMetaDataForPath(path string) (info *api.Prop, err error) {
 
 // errorHandler parses a non 2xx error response into an error
 func errorHandler(resp *http.Response) error {
+	body, err := rest.ReadBody(resp)
+	if err != nil {
+		return errors.Wrap(err, "error when trying to read error from body")
+	}
 	// Decode error response
 	errResponse := new(api.Error)
-	err := rest.DecodeXML(resp, &errResponse)
+	err = xml.Unmarshal(body, &errResponse)
 	if err != nil {
-		fs.Debugf(nil, "Couldn't decode error response: %v", err)
+		// set the Message to be the body if can't parse the XML
+		errResponse.Message = strings.TrimSpace(string(body))
 	}
 	errResponse.Status = resp.Status
 	errResponse.StatusCode = resp.StatusCode
@@ -260,10 +269,12 @@ func NewFs(name, root string) (fs.Fs, error) {
 	if !strings.HasSuffix(endpoint, "/") {
 		endpoint += "/"
 	}
+	rootIsDir := strings.HasSuffix(root, "/")
 	root = strings.Trim(root, "/")
 
 	user := config.FileGet(name, "user")
 	pass := config.FileGet(name, "pass")
+	bearerToken := config.FileGet(name, "bearer_token")
 	if pass != "" {
 		var err error
 		pass, err = obscure.Reveal(pass)
@@ -284,7 +295,7 @@ func NewFs(name, root string) (fs.Fs, error) {
 		root:        root,
 		endpoint:    u,
 		endpointURL: u.String(),
-		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()).SetUserPass(user, pass),
+		srv:         rest.NewClient(fshttp.NewClient(fs.Config)).SetRoot(u.String()),
 		pacer:       pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
 		user:        user,
 		pass:        pass,
@@ -293,13 +304,18 @@ func NewFs(name, root string) (fs.Fs, error) {
 	f.features = (&fs.Features{
 		CanHaveEmptyDirectories: true,
 	}).Fill(f)
+	if user != "" || pass != "" {
+		f.srv.SetUserPass(user, pass)
+	} else if bearerToken != "" {
+		f.srv.SetHeader("Authorization", "BEARER "+bearerToken)
+	}
 	f.srv.SetErrorHandler(errorHandler)
 	err = f.setQuirks(vendor)
 	if err != nil {
 		return nil, err
 	}
 
-	if root != "" {
+	if root != "" && !rootIsDir {
 		// Check to see if the root actually an existing file
 		remote := path.Base(root)
 		f.root = path.Dir(root)
@@ -544,6 +560,11 @@ func (f *Fs) PutStream(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption
 // mkParentDir makes the parent of the native path dirPath if
 // necessary and any directories above that
 func (f *Fs) mkParentDir(dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
+	// chop off trailing / if it exists
+	if strings.HasSuffix(dirPath, "/") {
+		dirPath = dirPath[:len(dirPath)-1]
+	}
 	parent := path.Dir(dirPath)
 	if parent == "." {
 		parent = ""
@@ -553,10 +574,15 @@ func (f *Fs) mkParentDir(dirPath string) error {
 
 // mkdir makes the directory and parents using native paths
 func (f *Fs) mkdir(dirPath string) error {
+	// defer log.Trace(dirPath, "")("")
 	// We assume the root is already ceated
 	if dirPath == "" {
 		return nil
 	}
+	// Collections must end with /
+	if !strings.HasSuffix(dirPath, "/") {
+		dirPath += "/"
+	}
 	opts := rest.Opts{
 		Method:     "MKCOL",
 		Path:       dirPath,
@@ -568,7 +594,7 @@ func (f *Fs) mkdir(dirPath string) error {
 	})
 	if apiErr, ok := err.(*api.Error); ok {
 		// already exists
-		if apiErr.StatusCode == http.StatusMethodNotAllowed {
+		if apiErr.StatusCode == http.StatusMethodNotAllowed || apiErr.StatusCode == http.StatusNotAcceptable {
 			return nil
 		}
 		// parent does not exists

backend/yandex/api/api_request.go

@@ -1,5 +0,0 @@
-package src
-
-type apiRequest interface {
-	Request() *HTTPRequest
-}

backend/yandex/api/http_request.go

@@ -12,10 +12,6 @@ func createGetRequest(client *Client, path string, params map[string]interface{}
 	return createRequest(client, "GET", path, params)
 }
 
-func createPostRequest(client *Client, path string, params map[string]interface{}) *HTTPRequest {
-	return createRequest(client, "POST", path, params)
-}
-
 func createRequest(client *Client, method string, path string, parameters map[string]interface{}) *HTTPRequest {
 	var headers = make(map[string][]string)
 	headers["Authorization"] = []string{"OAuth " + client.token}

backend/yandex/yandex.go

@@ -69,12 +69,11 @@ func init() {
 
 // Fs represents a remote yandex
 type Fs struct {
-	name       string
-	root       string         //root path
-	features   *fs.Features   // optional features
-	yd         *yandex.Client // client for rest api
-	diskRoot   string         //root path with "disk:/" container name
-	mkdircache map[string]int
+	name     string
+	root     string         //root path
+	features *fs.Features   // optional features
+	yd       *yandex.Client // client for rest api
+	diskRoot string         //root path with "disk:/" container name
 }
 
 // Object describes a swift object

bin/cross-compile.go

@@ -15,6 +15,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"sort"
 	"strings"
 	"sync"
 	"text/template"
@@ -66,28 +67,33 @@ var archFlags = map[string][]string{
 }
 
 // runEnv - run a shell command with env
-func runEnv(args, env []string) {
+func runEnv(args, env []string) error {
 	if *debug {
 		args = append([]string{"echo"}, args...)
 	}
 	cmd := exec.Command(args[0], args[1:]...)
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
 	if env != nil {
 		cmd.Env = append(os.Environ(), env...)
 	}
 	if *debug {
 		log.Printf("args = %v, env = %v\n", args, cmd.Env)
 	}
-	err := cmd.Run()
+	out, err := cmd.CombinedOutput()
 	if err != nil {
-		log.Fatalf("Failed to run %v: %v", args, err)
+		log.Print("----------------------------")
+		log.Printf("Failed to run %v: %v", args, err)
+		log.Printf("Command output was:\n%s", out)
+		log.Print("----------------------------")
 	}
+	return err
 }
 
 // run a shell command
 func run(args ...string) {
-	runEnv(args, nil)
+	err := runEnv(args, nil)
+	if err != nil {
+		log.Fatalf("Exiting after error: %v", err)
+	}
 }
 
 // chdir or die
@@ -160,8 +166,8 @@ func buildDebAndRpm(dir, version, goarch string) []string {
 	return artifacts
 }
 
-// build the binary in dir
-func compileArch(version, goos, goarch, dir string) {
+// build the binary in dir returning success or failure
+func compileArch(version, goos, goarch, dir string) bool {
 	log.Printf("Compiling %s/%s", goos, goarch)
 	output := filepath.Join(dir, "rclone")
 	if goos == "windows" {
@@ -191,7 +197,11 @@ func compileArch(version, goos, goarch, dir string) {
 	if flags, ok := archFlags[goarch]; ok {
 		env = append(env, flags...)
 	}
-	runEnv(args, env)
+	err = runEnv(args, env)
+	if err != nil {
+		log.Printf("Error compiling %s/%s: %v", goos, goarch, err)
+		return false
+	}
 	if !*compileOnly {
 		artifacts := []string{buildZip(dir)}
 		// build a .deb and .rpm if appropriate
@@ -207,6 +217,7 @@ func compileArch(version, goos, goarch, dir string) {
 		run("rm", "-rf", dir)
 	}
 	log.Printf("Done compiling %s/%s", goos, goarch)
+	return true
 }
 
 func compile(version string) {
@@ -231,6 +242,8 @@ func compile(version string) {
 		log.Fatalf("Bad -exclude regexp: %v", err)
 	}
 	compiled := 0
+	var failuresMu sync.Mutex
+	var failures []string
 	for _, osarch := range osarches {
 		if excludeRe.MatchString(osarch) || !includeRe.MatchString(osarch) {
 			continue
@@ -246,13 +259,22 @@ func compile(version string) {
 		}
 		dir := filepath.Join("rclone-" + version + "-" + userGoos + "-" + goarch)
 		run <- func() {
-			compileArch(version, goos, goarch, dir)
+			if !compileArch(version, goos, goarch, dir) {
+				failuresMu.Lock()
+				failures = append(failures, goos+"/"+goarch)
+				failuresMu.Unlock()
+			}
 		}
 		compiled++
 	}
 	close(run)
 	wg.Wait()
 	log.Printf("Compiled %d arches in %v", compiled, time.Since(start))
+	if len(failures) > 0 {
+		sort.Strings(failures)
+		log.Printf("%d compile failures:\n  %s\n", len(failures), strings.Join(failures, "\n  "))
+		os.Exit(1)
+	}
 }
 
 func main() {

bin/get-github-release.go

@@ -149,6 +149,9 @@ func getAsset(project string, matchName *regexp.Regexp) (string, string) {
 	log.Printf("Fetching asset info for %q from %q", project, url)
 	user, pass := os.Getenv("GITHUB_USER"), os.Getenv("GITHUB_TOKEN")
 	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		log.Fatalf("Failed to make http request %q: %v", url, err)
+	}
 	if user != "" && pass != "" {
 		log.Printf("Fetching using GITHUB_USER and GITHUB_TOKEN")
 		req.SetBasicAuth(user, pass)

bin/make_manual.py

@@ -38,6 +38,7 @@ docs = [
     "mega.md",
     "azureblob.md",
     "onedrive.md",
+    "opendrive.md",
     "qingstor.md",
     "swift.md",
     "pcloud.md",

bin/make_test_files.go

@@ -93,7 +93,7 @@ func (d *dir) createDirectories() {
 
 // list the directory heirachy
 func (d *dir) list(path string, output []string) []string {
-	dirPath := path + "/" + d.name
+	dirPath := filepath.Join(path, d.name)
 	output = append(output, dirPath)
 	for _, subDir := range d.children {
 		output = subDir.list(dirPath, output)

cmd/all/all.go

@@ -19,6 +19,7 @@ import (
 	_ "github.com/ncw/rclone/cmd/dbhashsum"
 	_ "github.com/ncw/rclone/cmd/dedupe"
 	_ "github.com/ncw/rclone/cmd/delete"
+	_ "github.com/ncw/rclone/cmd/deletefile"
 	_ "github.com/ncw/rclone/cmd/genautocomplete"
 	_ "github.com/ncw/rclone/cmd/gendocs"
 	_ "github.com/ncw/rclone/cmd/hashsum"

cmd/cachestats/cachestats.go

@@ -52,6 +52,9 @@ Print cache stats for a remote in JSON format
 				}
 			}
 			m, err := fsCache.Stats()
+			if err != nil {
+				return err
+			}
 
 			raw, err := json.MarshalIndent(m, "", "  ")
 			if err != nil {

cmd/check/check.go

@@ -9,11 +9,13 @@ import (
 // Globals
 var (
 	download = false
+	oneway   = false
 )
 
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
 	commandDefintion.Flags().BoolVarP(&download, "download", "", download, "Check by downloading rather than with hash.")
+	commandDefintion.Flags().BoolVarP(&oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote")
 }
 
 var commandDefintion = &cobra.Command{
@@ -31,15 +33,19 @@ If you supply the --download flag, it will download the data from
 both remotes and check them against each other on the fly.  This can
 be useful for remotes that don't support hashes or if you really want
 to check all the data.
+
+If you supply the --one-way flag, it will only check that files in source
+match the files in destination, not the other way around. Meaning extra files in
+destination that are not in the source will not trigger an error.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
 		fsrc, fdst := cmd.NewFsSrcDst(args)
 		cmd.Run(false, false, command, func() error {
 			if download {
-				return operations.CheckDownload(fdst, fsrc)
+				return operations.CheckDownload(fdst, fsrc, oneway)
 			}
-			return operations.Check(fdst, fsrc)
+			return operations.Check(fdst, fsrc, oneway)
 		})
 	},
 }

cmd/cmd.go

@@ -39,18 +39,18 @@ import (
 // Globals
 var (
 	// Flags
-	cpuProfile    = flags.StringP("cpuprofile", "", "", "Write cpu profile to file")
-	memProfile    = flags.StringP("memprofile", "", "", "Write memory profile to file")
-	statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
-	dataRateUnit  = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
-	version       bool
-	retries       = flags.IntP("retries", "", 3, "Retry operations this many times if they fail")
+	cpuProfile      = flags.StringP("cpuprofile", "", "", "Write cpu profile to file")
+	memProfile      = flags.StringP("memprofile", "", "", "Write memory profile to file")
+	statsInterval   = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)")
+	dataRateUnit    = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s")
+	version         bool
+	retries         = flags.IntP("retries", "", 3, "Retry operations this many times if they fail")
+	retriesInterval = flags.DurationP("retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)")
 	// Errors
 	errorCommandNotFound    = errors.New("command not found")
 	errorUncategorized      = errors.New("uncategorized error")
 	errorNotEnoughArguments = errors.New("not enough arguments")
 	errorTooManyArguents    = errors.New("too many arguments")
-	errorUsageError         = errors.New("usage error")
 )
 
 const (
@@ -62,6 +62,7 @@ const (
 	exitCodeRetryError
 	exitCodeNoRetryError
 	exitCodeFatalError
+	exitCodeTransferExceeded
 )
 
 // Root is the main rclone command
@@ -85,6 +86,7 @@ from various cloud storage systems and using file transfer services, such as:
   * Mega
   * Microsoft Azure Blob Storage
   * Microsoft OneDrive
+  * OpenDrive
   * Openstack Swift / Rackspace cloud files / Memset Memstore
   * pCloud
   * QingStor
@@ -121,7 +123,7 @@ func runRoot(cmd *cobra.Command, args []string) {
 		resolveExitCode(nil)
 	} else {
 		_ = Root.Usage()
-		fmt.Fprintf(os.Stderr, "Command not found.\n")
+		_, _ = fmt.Fprintf(os.Stderr, "Command not found.\n")
 		resolveExitCode(errorCommandNotFound)
 	}
 }
@@ -144,9 +146,10 @@ func ShowVersion() {
 	fmt.Printf("- go version: %s\n", runtime.Version())
 }
 
-// NewFsFile creates a dst Fs from a name but may point to a file.
+// NewFsFile creates a Fs from a name but may point to a file.
 //
 // It returns a string with the file name if points to a file
+// otherwise "".
 func NewFsFile(remote string) (fs.Fs, string) {
 	fsInfo, configName, fsPath, err := fs.ParseRemote(remote)
 	if err != nil {
@@ -166,12 +169,11 @@ func NewFsFile(remote string) (fs.Fs, string) {
 	return nil, ""
 }
 
-// newFsSrc creates a src Fs from a name
+// newFsFileAddFilter creates a src Fs from a name
 //
-// It returns a string with the file name if limiting to one file
-//
-// This can point to a file
-func newFsSrc(remote string) (fs.Fs, string) {
+// This works the same as NewFsFile however it adds filters to the Fs
+// to limit it to a single file if the remote pointed to a file.
+func newFsFileAddFilter(remote string) (fs.Fs, string) {
 	f, fileName := NewFsFile(remote)
 	if fileName != "" {
 		if !filter.Active.InActive() {
@@ -189,10 +191,19 @@ func newFsSrc(remote string) (fs.Fs, string) {
 	return f, fileName
 }
 
-// newFsDst creates a dst Fs from a name
+// NewFsSrc creates a new src fs from the arguments.
+//
+// The source can be a file or a directory - if a file then it will
+// limit the Fs to a single file.
+func NewFsSrc(args []string) fs.Fs {
+	fsrc, _ := newFsFileAddFilter(args[0])
+	return fsrc
+}
+
+// newFsDir creates an Fs from a name
 //
 // This must point to a directory
-func newFsDst(remote string) fs.Fs {
+func newFsDir(remote string) fs.Fs {
 	f, err := fs.NewFs(remote)
 	if err != nil {
 		fs.CountError(err)
@@ -201,24 +212,40 @@ func newFsDst(remote string) fs.Fs {
 	return f
 }
 
+// NewFsDir creates a new Fs from the arguments
+//
+// The argument must point a directory
+func NewFsDir(args []string) fs.Fs {
+	fdst := newFsDir(args[0])
+	return fdst
+}
+
 // NewFsSrcDst creates a new src and dst fs from the arguments
 func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) {
-	fsrc, _ := newFsSrc(args[0])
-	fdst := newFsDst(args[1])
-	fs.CalculateModifyWindow(fdst, fsrc)
+	fsrc, _ := newFsFileAddFilter(args[0])
+	fdst := newFsDir(args[1])
 	return fsrc, fdst
 }
 
+// NewFsSrcFileDst creates a new src and dst fs from the arguments
+//
+// The source may be a file, in which case the source Fs and file name is returned
+func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs) {
+	fsrc, srcFileName = NewFsFile(args[0])
+	fdst = newFsDir(args[1])
+	return fsrc, srcFileName, fdst
+}
+
 // NewFsSrcDstFiles creates a new src and dst fs from the arguments
 // If src is a file then srcFileName and dstFileName will be non-empty
 func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) {
-	fsrc, srcFileName = newFsSrc(args[0])
+	fsrc, srcFileName = newFsFileAddFilter(args[0])
 	// If copying a file...
 	dstRemote := args[1]
 	// If file exists then srcFileName != "", however if the file
 	// doesn't exist then we assume it is a directory...
 	if srcFileName != "" {
-		dstRemote, dstFileName = fspath.RemoteSplit(dstRemote)
+		dstRemote, dstFileName = fspath.Split(dstRemote)
 		if dstRemote == "" {
 			dstRemote = "."
 		}
@@ -236,37 +263,19 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
 		fs.CountError(err)
 		log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err)
 	}
-	fs.CalculateModifyWindow(fdst, fsrc)
 	return
 }
 
-// NewFsSrc creates a new src fs from the arguments
-func NewFsSrc(args []string) fs.Fs {
-	fsrc, _ := newFsSrc(args[0])
-	fs.CalculateModifyWindow(fsrc)
-	return fsrc
-}
-
-// NewFsDst creates a new dst fs from the arguments
-//
-// Dst fs-es can't point to single files
-func NewFsDst(args []string) fs.Fs {
-	fdst := newFsDst(args[0])
-	fs.CalculateModifyWindow(fdst)
-	return fdst
-}
-
 // NewFsDstFile creates a new dst fs with a destination file name from the arguments
 func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) {
-	dstRemote, dstFileName := fspath.RemoteSplit(args[0])
+	dstRemote, dstFileName := fspath.Split(args[0])
 	if dstRemote == "" {
 		dstRemote = "."
 	}
 	if dstFileName == "" {
 		log.Fatalf("%q is a directory", args[0])
 	}
-	fdst = newFsDst(dstRemote)
-	fs.CalculateModifyWindow(fdst)
+	fdst = newFsDir(dstRemote)
 	return
 }
 
@@ -289,6 +298,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 	if showStats {
 		stopStats = StartStats()
 	}
+	SigInfoHandler()
 	for try := 1; try <= *retries; try++ {
 		err = f()
 		if !Retry || (err == nil && !accounting.Stats.Errored()) {
@@ -313,6 +323,9 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 		if try < *retries {
 			accounting.Stats.ResetErrors()
 		}
+		if *retriesInterval > 0 {
+			time.Sleep(*retriesInterval)
+		}
 	}
 	if showStats {
 		close(stopStats)
@@ -354,12 +367,12 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
 func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) {
 	if len(args) < MinArgs {
 		_ = cmd.Usage()
-		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments mininum\n", cmd.Name(), MinArgs)
+		_, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum\n", cmd.Name(), MinArgs)
 		// os.Exit(1)
 		resolveExitCode(errorNotEnoughArguments)
 	} else if len(args) > MaxArgs {
 		_ = cmd.Usage()
-		fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name(), MaxArgs)
+		_, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum\n", cmd.Name(), MaxArgs)
 		// os.Exit(1)
 		resolveExitCode(errorTooManyArguents)
 	}
@@ -457,19 +470,22 @@ func initConfig() {
 }
 
 func resolveExitCode(err error) {
+	atexit.Run()
 	if err == nil {
 		os.Exit(exitCodeSuccess)
 	}
 
-	err = errors.Cause(err)
+	_, unwrapped := fserrors.Cause(err)
 
 	switch {
-	case err == fs.ErrorDirNotFound:
+	case unwrapped == fs.ErrorDirNotFound:
 		os.Exit(exitCodeDirNotFound)
-	case err == fs.ErrorObjectNotFound:
+	case unwrapped == fs.ErrorObjectNotFound:
 		os.Exit(exitCodeFileNotFound)
-	case err == errorUncategorized:
+	case unwrapped == errorUncategorized:
 		os.Exit(exitCodeUncategorizedError)
+	case unwrapped == accounting.ErrorMaxTransferLimitReached:
+		os.Exit(exitCodeTransferExceeded)
 	case fserrors.ShouldRetry(err):
 		os.Exit(exitCodeRetryError)
 	case fserrors.IsNoRetryError(err):

cmd/cmount/fs.go

@@ -350,7 +350,6 @@ func (fsys *FS) Read(path string, buff []byte, ofst int64, fh uint64) (n int) {
 	}
 	n, err := handle.ReadAt(buff, ofst)
 	if err == io.EOF {
-		err = nil
 	} else if err != nil {
 		return translateError(err)
 	}

cmd/cmount/mount.go

@@ -53,9 +53,13 @@ func mountOptions(device string, mountpoint string) (options []string) {
 
 	// OSX options
 	if runtime.GOOS == "darwin" {
-		options = append(options, "-o", "volname="+device)
-		options = append(options, "-o", "noappledouble")
-		options = append(options, "-o", "noapplexattr")
+		options = append(options, "-o", "volname="+mountlib.VolumeName)
+		if mountlib.NoAppleDouble {
+			options = append(options, "-o", "noappledouble")
+		}
+		if mountlib.NoAppleXattr {
+			options = append(options, "-o", "noapplexattr")
+		}
 	}
 
 	// Windows options

cmd/copy/copy.go

@@ -2,6 +2,7 @@ package copy
 
 import (
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs/operations"
 	"github.com/ncw/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
@@ -52,9 +53,12 @@ source or destination.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
-		fsrc, fdst := cmd.NewFsSrcDst(args)
+		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
-			return sync.CopyDir(fdst, fsrc)
+			if srcFileName == "" {
+				return sync.CopyDir(fdst, fsrc)
+			}
+			return operations.CopyFile(fdst, fsrc, srcFileName, srcFileName)
 		})
 	},
 }

cmd/cryptcheck/cryptcheck.go

@@ -10,8 +10,14 @@ import (
 	"github.com/spf13/cobra"
 )
 
+// Globals
+var (
+	oneway = false
+)
+
 func init() {
 	cmd.Root.AddCommand(commandDefintion)
+	commandDefintion.Flags().BoolVarP(&oneway, "one-way", "", oneway, "Check one way only, source files must exist on destination")
 }
 
 var commandDefintion = &cobra.Command{
@@ -40,6 +46,10 @@ the files in remote:path.
     rclone cryptcheck remote:path encryptedremote:path
 
 After it has run it will log the status of the encryptedremote:.
+
+If you supply the --one-way flag, it will only check that files in source
+match the files in destination, not the other way around. Meaning extra files in
+destination that are not in the source will not trigger an error.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
@@ -100,5 +110,5 @@ func cryptCheck(fdst, fsrc fs.Fs) error {
 		return false, false
 	}
 
-	return operations.CheckFn(fcrypt, fsrc, checkIdentical)
+	return operations.CheckFn(fcrypt, fsrc, checkIdentical, oneway)
 }

cmd/deletefile/deletefile.go

@@ -0,0 +1,37 @@
+package deletefile
+
+import (
+	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs/operations"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	cmd.Root.AddCommand(commandDefintion)
+}
+
+var commandDefintion = &cobra.Command{
+	Use:   "deletefile remote:path",
+	Short: `Remove a single file from remote.`,
+	Long: `
+Remove a single file from remote.  Unlike ` + "`" + `delete` + "`" + ` it cannot be used to
+remove a directory and it doesn't obey include/exclude filters - if the specified file exists,
+it will always be removed.
+`,
+	Run: func(command *cobra.Command, args []string) {
+		cmd.CheckArgs(1, 1, command, args)
+		fs, fileName := cmd.NewFsFile(args[0])
+		cmd.Run(true, false, command, func() error {
+			if fileName == "" {
+				return errors.Errorf("%s is a directory or doesn't exist", args[0])
+			}
+			fileObj, err := fs.NewObject(fileName)
+			if err != nil {
+				return err
+			}
+			return operations.DeleteFile(fileObj)
+		})
+	},
+}

cmd/info/info.go

@@ -48,7 +48,7 @@ a bit of go code for each one.
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1E6, command, args)
 		for i := range args {
-			f := cmd.NewFsDst(args[i : i+1])
+			f := cmd.NewFsDir(args[i : i+1])
 			cmd.Run(false, false, command, func() error {
 				return readInfo(f)
 			})

cmd/listremotes/listremotes.go

@@ -5,7 +5,6 @@ import (
 	"sort"
 
 	"github.com/ncw/rclone/cmd"
-	"github.com/ncw/rclone/fs"
 	"github.com/ncw/rclone/fs/config"
 	"github.com/spf13/cobra"
 )
@@ -40,7 +39,7 @@ When uses with the -l flag it lists the types too.
 		}
 		for _, remote := range remotes {
 			if listLong {
-				remoteType := fs.ConfigFileGet(remote, "type", "UNKNOWN")
+				remoteType := config.FileGet(remote, "type", "UNKNOWN")
 				fmt.Printf("%-*s %s\n", maxlen+1, remote+":", remoteType)
 			} else {
 				fmt.Printf("%s:\n", remote)

cmd/lsf/lsf.go

@@ -23,6 +23,8 @@ var (
 	hashType  = hash.MD5
 	filesOnly bool
 	dirsOnly  bool
+	csv       bool
+	absolute  bool
 )
 
 func init() {
@@ -34,6 +36,8 @@ func init() {
 	flags.VarP(&hashType, "hash", "", "Use this hash when `h` is used in the format MD5|SHA-1|DropboxHash")
 	flags.BoolVarP(&filesOnly, "files-only", "", false, "Only list files.")
 	flags.BoolVarP(&dirsOnly, "dirs-only", "", false, "Only list directories.")
+	flags.BoolVarP(&csv, "csv", "", false, "Output in CSV format.")
+	flags.BoolVarP(&absolute, "absolute", "", false, "Put a leading / in front of path names.")
 	commandDefintion.Flags().BoolVarP(&recurse, "recursive", "R", false, "Recurse into the listing.")
 }
 
@@ -63,6 +67,8 @@ output:
     s - size
     t - modification time
     h - hash
+    i - ID of object if known
+    m - MimeType of object if known
 
 So if you wanted the path, size and modification time, you would use
 --format "pst", or maybe --format "tsp" to put the path last.
@@ -111,11 +117,37 @@ Eg
     2018-04-26 08:52:53,0,,ferejej3gux/
     2016-06-25 18:55:40,37600,8fd37c3810dd660778137ac3a66cc06d,fubuwic
 
+You can output in CSV standard format.  This will escape things in "
+if they contain ,
+
+Eg
+
+    $ rclone lsf --csv --files-only --format ps remote:path
+    test.log,22355
+    test.sh,449
+    "this file contains a comma, in the file name.txt",6
+
+Note that the --absolute parameter is useful for making lists of files
+to pass to an rclone copy with the --files-from flag.
+
+For example to find all the files modified within one day and copy
+those only (without traversing the whole directory structure):
+
+    rclone lsf --absolute --files-only --max-age 1d /path/to/local > new_files
+    rclone copy --files-from new_files /path/to/local remote:path
+
 ` + lshelp.Help,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
 		fsrc := cmd.NewFsSrc(args)
 		cmd.Run(false, false, command, func() error {
+			// Work out if the separatorFlag was supplied or not
+			separatorFlag := command.Flags().Lookup("separator")
+			separatorFlagSupplied := separatorFlag != nil && separatorFlag.Changed
+			// Default the separator to , if using CSV
+			if csv && !separatorFlagSupplied {
+				separator = ","
+			}
 			return Lsf(fsrc, os.Stdout)
 		})
 	},
@@ -126,7 +158,9 @@ Eg
 func Lsf(fsrc fs.Fs, out io.Writer) error {
 	var list operations.ListFormat
 	list.SetSeparator(separator)
+	list.SetCSV(csv)
 	list.SetDirSlash(dirSlash)
+	list.SetAbsolute(absolute)
 
 	for _, char := range format {
 		switch char {
@@ -138,6 +172,10 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 			list.AddSize()
 		case 'h':
 			list.AddHash(hashType)
+		case 'i':
+			list.AddID()
+		case 'm':
+			list.AddMimeType()
 		default:
 			return errors.Errorf("Unknown format character %q", char)
 		}
@@ -160,7 +198,7 @@ func Lsf(fsrc fs.Fs, out io.Writer) error {
 					continue
 				}
 			}
-			fmt.Fprintln(out, operations.ListFormatted(&entry, &list))
+			_, _ = fmt.Fprintln(out, list.Format(entry))
 		}
 		return nil
 	})

cmd/lsjson/lsjson.go

@@ -39,9 +39,11 @@ type lsJSON struct {
 	Name      string
 	Encrypted string `json:",omitempty"`
 	Size      int64
+	MimeType  string    `json:",omitempty"`
 	ModTime   Timestamp //`json:",omitempty"`
 	IsDir     bool
 	Hashes    map[string]string `json:",omitempty"`
+	ID        string            `json:",omitempty"`
 }
 
 // Timestamp a time in RFC3339 format with Nanosecond precision secongs
@@ -69,7 +71,9 @@ The output is an array of Items, where each Item looks like this
          "MD5" : "b1946ac92492d2347c6235b4d2611184",
          "DropboxHash" : "ecb65bb98f9d905b70458986c39fcbad7715e5f2fcc3b1f07767d7c83e2438cc"
       },
+      "ID": "y2djkhiujf83u33",
       "IsDir" : false,
+      "MimeType" : "application/octet-stream",
       "ModTime" : "2017-05-31T16:15:57.034468261+01:00",
       "Name" : "file.txt",
       "Encrypted" : "v0qpsdq8anpci8n929v3uu9338",
@@ -121,9 +125,10 @@ can be processed line by line as each item is written one to a line.
 				}
 				for _, entry := range entries {
 					item := lsJSON{
-						Path: entry.Remote(),
-						Name: path.Base(entry.Remote()),
-						Size: entry.Size(),
+						Path:     entry.Remote(),
+						Name:     path.Base(entry.Remote()),
+						Size:     entry.Size(),
+						MimeType: fs.MimeTypeDirEntry(entry),
 					}
 					if !noModTime {
 						item.ModTime = Timestamp(entry.ModTime())
@@ -138,6 +143,9 @@ can be processed line by line as each item is written one to a line.
 							fs.Errorf(nil, "Unknown type %T in listing", entry)
 						}
 					}
+					if do, ok := entry.(fs.IDer); ok {
+						item.ID = do.ID()
+					}
 					switch x := entry.(type) {
 					case fs.Directory:
 						item.IsDir = true

cmd/mkdir/mkdir.go

@@ -15,7 +15,7 @@ var commandDefintion = &cobra.Command{
 	Short: `Make the path if it doesn't already exist.`,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
-		fdst := cmd.NewFsDst(args)
+		fdst := cmd.NewFsDir(args)
 		cmd.Run(true, false, command, func() error {
 			return operations.Mkdir(fdst, "")
 		})

cmd/mount/file.go

@@ -79,6 +79,9 @@ func (f *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenR
 		resp.Flags |= fuse.OpenNonSeekable
 	}
 
+	// Keep the file in the cache - equivalent of kernel_cache
+	resp.Flags |= fuse.OpenKeepCache
+
 	return &FileHandle{handle}, nil
 }
 
@@ -92,3 +95,37 @@ func (f *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) (err error) {
 	defer log.Trace(f, "")("err=%v", &err)
 	return nil
 }
+
+// Getxattr gets an extended attribute by the given name from the
+// node.
+//
+// If there is no xattr by that name, returns fuse.ErrNoXattr.
+func (f *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
+	return fuse.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeGetxattrer = (*File)(nil)
+
+// Listxattr lists the extended attributes recorded for the node.
+func (f *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
+	return fuse.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeListxattrer = (*File)(nil)
+
+// Setxattr sets an extended attribute with the given name and
+// value for the node.
+func (f *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {
+	return fuse.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeSetxattrer = (*File)(nil)
+
+// Removexattr removes an extended attribute for the name.
+//
+// If there is no xattr by that name, returns fuse.ErrNoXattr.
+func (f *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {
+	return fuse.ENOSYS // we never implement this
+}
+
+var _ fusefs.NodeRemovexattrer = (*File)(nil)

cmd/mount/mount.go

@@ -13,6 +13,7 @@ import (
 	fusefs "bazil.org/fuse/fs"
 	"github.com/ncw/rclone/cmd/mountlib"
 	"github.com/ncw/rclone/fs"
+	"github.com/ncw/rclone/lib/atexit"
 	"github.com/ncw/rclone/vfs"
 	"github.com/ncw/rclone/vfs/vfsflags"
 	"github.com/okzk/sdnotify"
@@ -28,9 +29,8 @@ func mountOptions(device string) (options []fuse.MountOption) {
 	options = []fuse.MountOption{
 		fuse.MaxReadahead(uint32(mountlib.MaxReadAhead)),
 		fuse.Subtype("rclone"),
-		fuse.FSName(device), fuse.VolumeName(device),
-		fuse.NoAppleDouble(),
-		fuse.NoAppleXattr(),
+		fuse.FSName(device),
+		fuse.VolumeName(mountlib.VolumeName),
 
 		// Options from benchmarking in the fuse module
 		//fuse.MaxReadahead(64 * 1024 * 1024),
@@ -39,6 +39,12 @@ func mountOptions(device string) (options []fuse.MountOption) {
 		// which is probably related to errors people are having
 		//fuse.WritebackCache(),
 	}
+	if mountlib.NoAppleDouble {
+		options = append(options, fuse.NoAppleDouble())
+	}
+	if mountlib.NoAppleXattr {
+		options = append(options, fuse.NoAppleXattr())
+	}
 	if mountlib.AllowNonEmpty {
 		options = append(options, fuse.AllowNonEmptyMount())
 	}
@@ -128,6 +134,7 @@ func Mount(f fs.Fs, mountpoint string) error {
 	signal.Notify(sigInt, syscall.SIGINT, syscall.SIGTERM)
 	sigHup := make(chan os.Signal, 1)
 	signal.Notify(sigHup, syscall.SIGHUP)
+	atexit.IgnoreSignals()
 
 	if err := sdnotify.SdNotifyReady(); err != nil && err != sdnotify.SdNotifyNoSocket {
 		return errors.Wrap(err, "failed to notify systemd")

cmd/mountlib/daemon.go

@@ -1,6 +1,6 @@
 // Daemonization interface for non-Unix variants only
 
-// +build windows darwin,!cgo
+// +build windows
 
 package mountlib
 

cmd/mountlib/daemon_unix.go

@@ -1,7 +1,6 @@
 // Daemonization interface for Unix variants only
 
 // +build !windows
-// +build !darwin cgo
 
 package mountlib
 

cmd/mountlib/mount.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"os"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/ncw/rclone/cmd"
@@ -29,6 +30,9 @@ var (
 	ExtraOptions       []string
 	ExtraFlags         []string
 	AttrTimeout        = 1 * time.Second // how long the kernel caches attribute for
+	VolumeName         string
+	NoAppleDouble      = true  // use noappledouble by default
+	NoAppleXattr       = false // do not use noapplexattr by default
 )
 
 // Check is folder is empty
@@ -189,10 +193,29 @@ to use Type=notify. In this case the service will enter the started state
 after the mountpoint has been successfully set up.
 Units having the rclone ` + commandName + ` service specified as a requirement
 will see all files and folders immediately in this mode.
+
+### chunked reading ###
+
+--vfs-read-chunk-size will enable reading the source objects in parts.
+This can reduce the used download quota for some remotes by requesting only chunks
+from the remote that are actually read at the cost of an increased number of requests.
+
+When --vfs-read-chunk-size-limit is also specified and greater than --vfs-read-chunk-size,
+the chunk size for each open file will get doubled for each chunk read, until the
+specified value is reached. A value of -1 will disable the limit and the chunk size will
+grow indefinitely.
+
+With --vfs-read-chunk-size 100M and --vfs-read-chunk-size-limit 0 the following
+parts will be downloaded: 0-100M, 100M-200M, 200M-300M, 300M-400M and so on.
+When --vfs-read-chunk-size-limit 500M is specified, the result would be
+0-100M, 100M-300M, 300M-700M, 700M-1200M, 1200M-1700M and so on.
+
+Chunked reading will only work with --vfs-cache-mode < full, as the file will always
+be copied to the vfs cache before opening with --vfs-cache-mode full.
 ` + vfs.Help,
 		Run: func(command *cobra.Command, args []string) {
 			cmd.CheckArgs(2, 2, command, args)
-			fdst := cmd.NewFsDst(args)
+			fdst := cmd.NewFsDir(args)
 
 			// Show stats if the user has specifically requested them
 			if cmd.ShowStats() {
@@ -209,6 +232,15 @@ will see all files and folders immediately in this mode.
 				}
 			}
 
+			// Work out the volume name, removing special
+			// characters from it if necessary
+			if VolumeName == "" {
+				VolumeName = fdst.Name() + ":" + fdst.Root()
+			}
+			VolumeName = strings.Replace(VolumeName, ":", " ", -1)
+			VolumeName = strings.Replace(VolumeName, "/", " ", -1)
+			VolumeName = strings.TrimSpace(VolumeName)
+
 			// Start background task if --background is specified
 			if Daemon {
 				daemonized := startBackgroundMode()
@@ -241,6 +273,12 @@ will see all files and folders immediately in this mode.
 	flags.StringArrayVarP(flagSet, &ExtraOptions, "option", "o", []string{}, "Option for libfuse/WinFsp. Repeat if required.")
 	flags.StringArrayVarP(flagSet, &ExtraFlags, "fuse-flag", "", []string{}, "Flags or arguments to be passed direct to libfuse/WinFsp. Repeat if required.")
 	flags.BoolVarP(flagSet, &Daemon, "daemon", "", Daemon, "Run mount as a daemon (background mode).")
+	flags.StringVarP(flagSet, &VolumeName, "volname", "", VolumeName, "Set the volume name (not supported by all OSes).")
+
+	if runtime.GOOS == "darwin" {
+		flags.BoolVarP(flagSet, &NoAppleDouble, "noappledouble", "", NoAppleDouble, "Sets the OSXFUSE option noappledouble.")
+		flags.BoolVarP(flagSet, &NoAppleXattr, "noapplexattr", "", NoAppleXattr, "Sets the OSXFUSE option noapplexattr.")
+	}
 
 	// Add in the generic flags
 	vfsflags.AddFlags(flagSet)

cmd/mountlib/mounttest/edge_cases.go

@@ -0,0 +1,59 @@
+package mounttest
+
+import (
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestTouchAndDelete checks that writing a zero byte file and immediately
+// deleting it is not racy. See https://github.com/ncw/rclone/issues/1181
+func TestTouchAndDelete(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	run.checkDir(t, "")
+
+	run.createFile(t, "touched", "")
+	run.rm(t, "touched")
+
+	run.checkDir(t, "")
+}
+
+// TestRenameOpenHandle checks that a file with open writers is successfully
+// renamed after all writers close. See https://github.com/ncw/rclone/issues/2130
+func TestRenameOpenHandle(t *testing.T) {
+	run.skipIfNoFUSE(t)
+	if runtime.GOOS == "windows" {
+		t.Skip("Skipping test on Windows")
+	}
+
+	run.checkDir(t, "")
+
+	// create file
+	example := []byte("Some Data")
+	path := run.path("rename")
+	file, err := osCreate(path)
+	require.NoError(t, err)
+
+	// write some data
+	_, err = file.Write(example)
+	require.NoError(t, err)
+	err = file.Sync()
+	require.NoError(t, err)
+
+	// attempt to rename open file
+	err = os.Rename(path, path+"bla")
+	require.NoError(t, err)
+
+	// close open writers to allow rename on remote to go through
+	err = file.Close()
+	require.NoError(t, err)
+
+	// verify file was renamed properly
+	run.checkDir(t, "renamebla 9")
+
+	// cleanup
+	run.rm(t, "renamebla")
+	run.checkDir(t, "")
+}

cmd/mountlib/mounttest/fs.go

@@ -52,6 +52,8 @@ func RunTests(t *testing.T, fn MountFn) {
 		run.cacheMode(cacheMode)
 		log.Printf("Starting test run with cache mode %v", cacheMode)
 		ok := t.Run(fmt.Sprintf("CacheMode=%v", cacheMode), func(t *testing.T) {
+			t.Run("TestTouchAndDelete", TestTouchAndDelete)
+			t.Run("TestRenameOpenHandle", TestRenameOpenHandle)
 			t.Run("TestDirLs", TestDirLs)
 			t.Run("TestDirCreateAndRemoveDir", TestDirCreateAndRemoveDir)
 			t.Run("TestDirCreateAndRemoveFile", TestDirCreateAndRemoveFile)
@@ -122,32 +124,36 @@ func newRun() *Run {
 		log.Fatalf("Failed to open mkdir %q: %v", *fstest.RemoteName, err)
 	}
 
-	if runtime.GOOS != "windows" {
-		r.mountPath, err = ioutil.TempDir("", "rclonefs-mount")
-		if err != nil {
-			log.Fatalf("Failed to create mount dir: %v", err)
-		}
-	} else {
-		// Find a free drive letter
-		drive := ""
-		for letter := 'E'; letter <= 'Z'; letter++ {
-			drive = string(letter) + ":"
-			_, err := os.Stat(drive + "\\")
-			if os.IsNotExist(err) {
-				goto found
-			}
-		}
-		log.Fatalf("Couldn't find free drive letter for test")
-	found:
-		r.mountPath = drive
-	}
-
+	r.mountPath = findMountPath()
 	// Mount it up
 	r.mount()
 
 	return r
 }
 
+func findMountPath() string {
+	if runtime.GOOS != "windows" {
+		mountPath, err := ioutil.TempDir("", "rclonefs-mount")
+		if err != nil {
+			log.Fatalf("Failed to create mount dir: %v", err)
+		}
+		return mountPath
+	}
+
+	// Find a free drive letter
+	drive := ""
+	for letter := 'E'; letter <= 'Z'; letter++ {
+		drive = string(letter) + ":"
+		_, err := os.Stat(drive + "\\")
+		if os.IsNotExist(err) {
+			goto found
+		}
+	}
+	log.Fatalf("Couldn't find free drive letter for test")
+found:
+	return drive
+}
+
 func (r *Run) mount() {
 	log.Printf("mount %q %q", r.fremote, r.mountPath)
 	var err error

cmd/move/move.go

@@ -2,6 +2,7 @@ package move
 
 import (
 	"github.com/ncw/rclone/cmd"
+	"github.com/ncw/rclone/fs/operations"
 	"github.com/ncw/rclone/fs/sync"
 	"github.com/spf13/cobra"
 )
@@ -41,10 +42,12 @@ If you want to delete empty source directories after move, use the --delete-empt
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(2, 2, command, args)
-		fsrc, fdst := cmd.NewFsSrcDst(args)
+		fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst(args)
 		cmd.Run(true, true, command, func() error {
-
-			return sync.MoveDir(fdst, fsrc, deleteEmptySrcDirs)
+			if srcFileName == "" {
+				return sync.MoveDir(fdst, fsrc, deleteEmptySrcDirs)
+			}
+			return operations.MoveFile(fdst, fsrc, srcFileName, srcFileName)
 		})
 	},
 }

cmd/ncdu/scan/scan.go

@@ -12,16 +12,13 @@ import (
 
 // Dir represents a directory found in the remote
 type Dir struct {
-	parent   *Dir
-	path     string
-	mu       sync.Mutex
-	count    int64
-	size     int64
-	complete bool
-	entries  fs.DirEntries
-	dirs     map[string]*Dir
-	offset   int // current listing offset
-	entry    int // current listing entry
+	parent  *Dir
+	path    string
+	mu      sync.Mutex
+	count   int64
+	size    int64
+	entries fs.DirEntries
+	dirs    map[string]*Dir
 }
 
 // Parent returns the directory above this one

cmd/purge/purge.go

@@ -20,7 +20,7 @@ you want to selectively delete files.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
-		fdst := cmd.NewFsDst(args)
+		fdst := cmd.NewFsDir(args)
 		cmd.Run(true, false, command, func() error {
 			return operations.Purge(fdst, "")
 		})

cmd/rc/rc.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"strings"
@@ -39,7 +40,7 @@ Arguments should be passed in as parameter=value.
 
 The result will be returned as a JSON object by default.
 
-Use "rclone rc list" to see a list of all possible commands.`,
+Use "rclone rc" to see a list of all possible commands.`,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(0, 1E9, command, args)
 		cmd.Run(false, false, command, func() error {
@@ -80,6 +81,19 @@ func doCall(path string, in rc.Params) (out rc.Params, err error) {
 	}
 	defer fs.CheckClose(resp.Body, &err)
 
+	if resp.StatusCode != http.StatusOK {
+		var body []byte
+		body, err = ioutil.ReadAll(resp.Body)
+		var bodyString string
+		if err == nil {
+			bodyString = string(body)
+		} else {
+			bodyString = err.Error()
+		}
+		bodyString = strings.TrimSpace(bodyString)
+		return nil, errors.Errorf("Failed to read rc response: %s: %s", resp.Status, bodyString)
+	}
+
 	// Parse output
 	out = make(rc.Params)
 	err = json.NewDecoder(resp.Body).Decode(&out)

cmd/rcat/rcat.go

@@ -22,7 +22,7 @@ rclone rcat reads from standard input (stdin) and copies it to a
 single remote file.
 
     echo "hello world" | rclone rcat remote:path/to/file
-    ffmpeg - | rclone rcat --checksum remote:path/to/file
+    ffmpeg - | rclone rcat remote:path/to/file
 
 If the remote file already exists, it will be overwritten.
 

cmd/rmdir/rmdir.go

@@ -18,7 +18,7 @@ Remove the path.  Note that you can't remove a path with
 objects in it, use purge for that.`,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
-		fdst := cmd.NewFsDst(args)
+		fdst := cmd.NewFsDir(args)
 		cmd.Run(true, false, command, func() error {
 			return operations.Rmdir(fdst, "")
 		})

cmd/rmdirs/rmdirs.go

@@ -30,7 +30,7 @@ empty directories in.
 `,
 	Run: func(command *cobra.Command, args []string) {
 		cmd.CheckArgs(1, 1, command, args)
-		fdst := cmd.NewFsDst(args)
+		fdst := cmd.NewFsDir(args)
 		cmd.Run(true, false, command, func() error {
 			return operations.Rmdirs(fdst, "", leaveRoot)
 		})

cmd/serve/webdav/webdav.go

@@ -16,6 +16,7 @@ import (
 	"github.com/ncw/rclone/vfs/vfsflags"
 	"github.com/spf13/cobra"
 	"golang.org/x/net/context" // switch to "context" when we stop supporting go1.8
+
 	"golang.org/x/net/webdav"
 )
 

cmd/siginfo_darwin.go

@@ -0,0 +1,23 @@
+//+build darwin
+
+package cmd
+
+import (
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/ncw/rclone/fs/accounting"
+)
+
+// SigInfoHandler creates SigInfo handler
+func SigInfoHandler() {
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGINFO)
+	go func() {
+		for range signals {
+			log.Printf("%v\n", accounting.Stats)
+		}
+	}()
+}

cmd/siginfo_others.go

@@ -0,0 +1,7 @@
+//+build !darwin
+
+package cmd
+
+// SigInfoHandler creates SigInfo handler
+func SigInfoHandler() {
+}

cmd/tree/tree.go

@@ -135,7 +135,7 @@ func Tree(fsrc fs.Fs, outFile io.Writer, opts *tree.Options) error {
 		if !opts.DirsOnly {
 			footer += fmt.Sprintf(", %d files", nf)
 		}
-		fmt.Fprintln(outFile, footer)
+		_, _ = fmt.Fprintln(outFile, footer)
 	}
 	return nil
 }

docs/content/about.md

@@ -32,8 +32,9 @@ Rclone is a command line program to sync files and directories to and from:
 * {{< provider name="Microsoft Azure Blob Storage" home="https://azure.microsoft.com/en-us/services/storage/blobs/" config="/azureblob/" >}}
 * {{< provider name="Microsoft OneDrive" home="https://onedrive.live.com/" config="/onedrive/" >}}
 * {{< provider name="Minio" home="https://www.minio.io/" config="/s3/#minio" >}}
-* {{< provider name="Nextloud" home="https://nextcloud.com/" config="/webdav/#nextcloud" >}}
+* {{< provider name="Nextcloud" home="https://nextcloud.com/" config="/webdav/#nextcloud" >}}
 * {{< provider name="OVH" home="https://www.ovh.co.uk/public-cloud/storage/object-storage/" config="/swift/" >}}
+* {{< provider name="OpenDrive" home="https://www.opendrive.com/" config="/opendrive/" >}}
 * {{< provider name="Openstack Swift" home="https://docs.openstack.org/swift/latest/" config="/swift/" >}}
 * {{< provider name="Oracle Cloud Storage" home="https://cloud.oracle.com/storage-opc" config="/swift/" >}}
 * {{< provider name="ownCloud" home="https://owncloud.org/" config="/webdav/#owncloud" >}}

docs/content/authors.md

@@ -161,3 +161,13 @@ Contributors
   * hensur <me@hensur.de>
   * Oliver Heyme <de8olihe@lego.com>
   * Richard Yang <richard@yenforyang.com>
+  * Piotr Oleszczyk <piotr.oleszczyk@gmail.com>
+  * Rodrigo <rodarima@gmail.com>
+  * NoLooseEnds <NoLooseEnds@users.noreply.github.com>
+  * Jakub Karlicek <jakub@karlicek.me>
+  * John Clayton <john@codemonkeylabs.com>
+  * Kasper Byrdal Nielsen <byrdal76@gmail.com>
+  * Benjamin Joseph Dag <bjdag1234@users.noreply.github.com>
+  * themylogin <themylogin@gmail.com>
+  * Onno Zweers <onno.zweers@surfsara.nl>
+  * Jasper Lievisse Adriaanse <jasper@humppa.nl>

docs/content/azureblob.md

@@ -117,6 +117,39 @@ MD5 hashes are stored with blobs.  However blobs that were uploaded in
 chunks only have an MD5 if the source remote was capable of MD5
 hashes, eg the local disk.
 
+### Authenticating with Azure Blob Storage
+
+Rclone has 3 ways of authenticating with Azure Blob Storage:
+
+#### Account and Key
+
+This is the most straight forward and least flexible way.  Just fill in the `account` and `key` lines and leave the rest blank.
+
+#### Connection string
+
+This supports all the possible connection string variants.  Leave `account`, `key` and `sas_url` blank and put the connection string into the `connection_string` configuration parameter.
+
+Use this method if using an account level SAS; the Azure Portal shows connection strings you can cut and paste.
+
+#### SAS URL
+
+This only for a container level SAS URL - it does not work with an account level SAS URL. For account level SAS use the connection string method.
+
+To use it leave `account`, `key` and `connection_string` blank and fill in `sas_url`.
+
+To get a container level SAS URL right click on a container in the Azure Blob explorer in the Azure portal.
+
+You will only be able to use the container specified in the SAS URL with rclone, eg
+
+    rclone ls azureblob:container
+
+However these will not work
+
+    rclone lsd azureblob:
+    rclone ls azureblob:othercontainer
+
+This would be useful for temporarily allowing third parties access to a single container or putting credentials into an untrusted environment.
+
 ### Multipart uploads ###
 
 Rclone supports multipart uploads with Azure Blob storage.  Files

docs/content/changelog.md

@@ -7,6 +7,85 @@ date: "2018-04-28"
 Changelog
 ---------
 
+  * v1.42 - 2018-06-16
+    * New backends
+      * OpenDrive (Oliver Heyme, Jakub Karlicek, ncw)
+    * New commands
+      * deletefile command (Filip Bartodziej)
+    * New Features
+      * copy, move: Copy single files directly, don't use `--files-from` work-around
+         * this makes them much more efficient
+      * Implement `--max-transfer` flag to quit transferring at a limit
+         * make exit code 8 for `--max-transfer` exceeded
+      * copy: copy empty source directories to destination (Ishuah Kariuki)
+      * check: Add `--one-way` flag (Kasper Byrdal Nielsen)
+      * Add siginfo handler for macOS for ctrl-T stats (kubatasiemski)
+      * rc
+         * add core/gc to run a garbage collection on demand
+         * enable go profiling by default on the `--rc` port
+         * return error from remote on failure
+      * lsf
+         * Add `--absolute` flag to add a leading / onto path names
+         * Add `--csv` flag for compliant CSV output
+         * Add 'm' format specifier to show the MimeType
+         * Implement 'i' format for showing object ID
+      * lsjson
+         * Add MimeType to the output
+         * Add ID field to output to show Object ID
+      * Add `--retries-sleep` flag (Benjamin Joseph Dag)
+      * Oauth tidy up web page and error handling (Henning Surmeier)
+    * Bug Fixes
+      * Password prompt output with `--log-file` fixed for unix (Filip Bartodziej)
+      * Calculate ModifyWindow each time on the fly to fix various problems (Stefan Breunig)
+    * Mount
+      * Only print "File.rename error" if there actually is an error (Stefan Breunig)
+      * Delay rename if file has open writers instead of failing outright (Stefan Breunig)
+      * Ensure atexit gets run on interrupt
+      * macOS enhancements
+         * Make `--noappledouble` `--noapplexattr`
+         * Add `--volname` flag and remove special chars from it
+         * Make Get/List/Set/Remove xattr return ENOSYS for efficiency
+         * Make `--daemon` work for macOS without CGO
+    * VFS
+        * Add `--vfs-read-chunk-size` and `--vfs-read-chunk-size-limit` (Fabian Möller)
+        * Fix ChangeNotify for new or changed folders (Fabian Möller)
+    * Local
+      * Fix symlink/junction point directory handling under Windows
+         * **NB** you will need to add `-L` to your command line to copy files with reparse points
+    * Cache
+      * Add non cached dirs on notifications (Remus Bunduc)
+      * Allow root to be expired from rc (Remus Bunduc)
+      * Clean remaining empty folders from temp upload path (Remus Bunduc)
+      * Cache lists using batch writes (Remus Bunduc)
+      * Use secure websockets for HTTPS Plex addresses (John Clayton)
+      * Reconnect plex websocket on failures (Remus Bunduc)
+      * Fix panic when running without plex configs (Remus Bunduc)
+      * Fix root folder caching (Remus Bunduc)
+    * Crypt
+      * Check the crypted hash of files when uploading for extra data security
+    * Dropbox
+      * Make Dropbox for business folders accessible using an initial `/` in the path
+    * Google Cloud Storage
+      * Low level retry all operations if necessary
+    * Google Drive
+      * Add `--drive-acknowledge-abuse` to download flagged files
+      * Add `--drive-alternate-export` to fix large doc export
+      * Don't attempt to choose Team Drives when using rclone config create
+      * Fix change list polling with team drives
+      * Fix ChangeNotify for folders (Fabian Möller)
+      * Fix about (and df on a mount) for team drives
+    * Onedrive
+      * Errorhandler for onedrive for business requests (Henning Surmeier)
+    * S3
+      * Adjust upload concurrency with `--s3-upload-concurrency` (themylogin)
+      * Fix `--s3-chunk-size` which was always using the minimum
+    * SFTP
+      * Add `--ssh-path-override` flag (Piotr Oleszczyk)
+      * Fix slow downloads for long latency connections
+    * Webdav
+      * Add workarounds for biz.mail.ru
+      * Ignore Reason-Phrase in status line to fix 4shared (Rodrigo)
+      * Better error message generation
   * v1.41 - 2018-04-28
     * New backends
       * Mega support added