Compare commits
42
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d8144a7e84 | ||
|
|
21f5b1076f | ||
|
|
a8cee91929 | ||
|
|
14edaefaa3 | ||
|
|
946b73c5fe | ||
|
|
07f7854a25 | ||
|
|
dfc7393e9c | ||
|
|
7f50143805 | ||
|
|
37308832b1 | ||
|
|
d5d124c9ee | ||
|
|
8b1f737271 | ||
|
|
ed013228ea | ||
|
|
a9b7974196 | ||
|
|
38453bd8ff | ||
|
|
51ba73afe4 | ||
|
|
b782a15f17 | ||
|
|
1d2edf0789 | ||
|
|
8367261a93 | ||
|
|
31dc78905e | ||
|
|
f9493ff1db | ||
|
|
a5cf531b94 | ||
|
|
04d59fe6bd | ||
|
|
cbc448039d | ||
|
|
7a8ade4949 | ||
|
|
8ab6e2bd1f | ||
|
|
d5c5811a54 | ||
|
|
43d21b7e64 | ||
|
|
0501b12e17 | ||
|
|
723c1dfdc5 | ||
|
|
d6894408b9 | ||
|
|
ac0e831090 | ||
|
|
b93ed77495 | ||
|
|
d0a3c40f9a | ||
|
|
6a9a571d87 | ||
|
|
6b68d1f79a | ||
|
|
088756ff8a | ||
|
|
4c1ffcc36d | ||
|
|
e5bd3f1c26 | ||
|
|
e03b4e2ce9 | ||
|
|
6054476c9c | ||
|
|
6cd8d3c4a0 | ||
|
|
78b0329928 |
No files matched your search
+20
-21
@@ -19,13 +19,13 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'race', 'go1.11', 'go1.12', 'go1.13']
|
||||
job_name: ['linux', 'mac', 'windows_amd64', 'windows_386', 'other_os', 'modules_race', 'go1.11', 'go1.12', 'go1.13']
|
||||
|
||||
include:
|
||||
- job_name: linux
|
||||
os: ubuntu-latest
|
||||
go: '1.14.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^linux/"'
|
||||
check: true
|
||||
@@ -35,7 +35,7 @@ jobs:
|
||||
- job_name: mac
|
||||
os: macOS-latest
|
||||
go: '1.14.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
gotags: '' # cmount doesn't work on osx travis for some reason
|
||||
build_flags: '-include "^darwin/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
- job_name: windows_amd64
|
||||
os: windows-latest
|
||||
go: '1.14.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
gotags: cmount
|
||||
build_flags: '-include "^windows/amd64" -cgo'
|
||||
quicktest: true
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
- job_name: windows_386
|
||||
os: windows-latest
|
||||
go: '1.14.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
gotags: cmount
|
||||
goarch: '386'
|
||||
cgo: '1'
|
||||
@@ -66,12 +66,12 @@ jobs:
|
||||
- job_name: other_os
|
||||
os: ubuntu-latest
|
||||
go: '1.14.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
build_flags: '-exclude "^(windows/|darwin/amd64|linux/)"'
|
||||
compile_all: true
|
||||
deploy: true
|
||||
|
||||
- job_name: race
|
||||
- job_name: modules_race
|
||||
os: ubuntu-latest
|
||||
go: '1.14.x'
|
||||
modules: 'on'
|
||||
@@ -81,19 +81,19 @@ jobs:
|
||||
- job_name: go1.11
|
||||
os: ubuntu-latest
|
||||
go: '1.11.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
quicktest: true
|
||||
|
||||
- job_name: go1.12
|
||||
os: ubuntu-latest
|
||||
go: '1.12.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
quicktest: true
|
||||
|
||||
- job_name: go1.13
|
||||
os: ubuntu-latest
|
||||
go: '1.13.x'
|
||||
modules: 'on'
|
||||
modules: 'off'
|
||||
quicktest: true
|
||||
|
||||
name: ${{ matrix.job_name }}
|
||||
@@ -197,9 +197,8 @@ jobs:
|
||||
- name: Deploy built binaries
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
|
||||
if [[ "${{ matrix.os }}" == "windows-latest" ]]; then make release_dep_windows ; fi
|
||||
make ci_beta
|
||||
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep ; fi
|
||||
make travis_beta
|
||||
env:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# working-directory: '$(modulePath)'
|
||||
@@ -229,12 +228,12 @@ jobs:
|
||||
run: |
|
||||
docker pull billziss/xgo-cgofuse
|
||||
GO111MODULE=off go get -v github.com/karalabe/xgo # don't add to go.mod
|
||||
# xgo \
|
||||
# -image=billziss/xgo-cgofuse \
|
||||
# -targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
|
||||
# -tags cmount \
|
||||
# -dest build \
|
||||
# .
|
||||
xgo \
|
||||
-image=billziss/xgo-cgofuse \
|
||||
-targets=darwin/386,darwin/amd64,linux/386,linux/amd64,windows/386,windows/amd64 \
|
||||
-tags cmount \
|
||||
-dest build \
|
||||
.
|
||||
xgo \
|
||||
-image=billziss/xgo-cgofuse \
|
||||
-targets=android/*,ios/* \
|
||||
@@ -244,11 +243,11 @@ jobs:
|
||||
- name: Build rclone
|
||||
run: |
|
||||
docker pull golang
|
||||
docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=mod -v
|
||||
docker run --rm -v "$PWD":/usr/src/rclone -w /usr/src/rclone golang go build -mod=vendor -v
|
||||
|
||||
- name: Upload artifacts
|
||||
run: |
|
||||
make ci_upload
|
||||
make circleci_upload
|
||||
env:
|
||||
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
|
||||
# Upload artifacts if not a PR && not a fork
|
||||
|
||||
+24
-14
@@ -79,15 +79,8 @@ request](https://help.github.com/articles/creating-a-pull-request/).
|
||||
|
||||
You patch will get reviewed and you might get asked to fix some stuff.
|
||||
|
||||
If so, then make the changes in the same branch, squash the commits (make multiple commits one commit) by running:
|
||||
```
|
||||
git log # See how many commits you want to squash
|
||||
git reset --soft HEAD~2 # This squashes the 2 latest commits together.
|
||||
git status # Check what will happen, if you made a mistake resetting, you can run git reset 'HEAD@{1}' to undo.
|
||||
git commit # Add a new commit message.
|
||||
git push --force # Push the squashed commit to your GitHub repo.
|
||||
# For more, see Stack Overflow, Git docs, or generally Duck around the web. jtagcat also reccommends wizardzines.com
|
||||
```
|
||||
If so, then make the changes in the same branch, squash the commits,
|
||||
rebase it to master then push it to GitHub with `--force`.
|
||||
|
||||
## CI for your fork ##
|
||||
|
||||
@@ -186,6 +179,7 @@ with modules beneath.
|
||||
* pacer - retries with backoff and paces operations
|
||||
* readers - a selection of useful io.Readers
|
||||
* rest - a thin abstraction over net/http for REST
|
||||
* vendor - 3rd party code managed by `go mod`
|
||||
* vfs - Virtual FileSystem layer for implementing rclone mount and similar
|
||||
|
||||
## Writing Documentation ##
|
||||
@@ -265,25 +259,41 @@ rclone uses the [go
|
||||
modules](https://tip.golang.org/cmd/go/#hdr-Modules__module_versions__and_more)
|
||||
support in go1.11 and later to manage its dependencies.
|
||||
|
||||
rclone can be built with modules outside of the GOPATH
|
||||
**NB** you must be using go1.11 or above to add a dependency to
|
||||
rclone. Rclone will still build with older versions of go, but we use
|
||||
the `go mod` command for dependencies which is only in go1.11 and
|
||||
above.
|
||||
|
||||
rclone can be built with modules outside of the GOPATH, but for
|
||||
backwards compatibility with older go versions, rclone also maintains
|
||||
a `vendor` directory with all the external code rclone needs for
|
||||
building.
|
||||
|
||||
The `vendor` directory is entirely managed by the `go mod` tool, do
|
||||
not add things manually.
|
||||
|
||||
To add a dependency `github.com/ncw/new_dependency` see the
|
||||
instructions below. These will fetch the dependency and add it to
|
||||
`go.mod` and `go.sum`.
|
||||
instructions below. These will fetch the dependency, add it to
|
||||
`go.mod` and `go.sum` and vendor it for older go versions.
|
||||
|
||||
GO111MODULE=on go get github.com/ncw/new_dependency
|
||||
GO111MODULE=on go mod vendor
|
||||
|
||||
You can add constraints on that package when doing `go get` (see the
|
||||
go docs linked above), but don't unless you really need to.
|
||||
|
||||
Please check in the changes generated by `go mod` including `go.mod`
|
||||
and `go.sum` in the same commit as your other changes.
|
||||
Please check in the changes generated by `go mod` including the
|
||||
`vendor` directory and `go.mod` and `go.sum` in a single commit
|
||||
separate from any other code changes with the title "vendor: add
|
||||
github.com/ncw/new_dependency". Remember to `git add` any new files
|
||||
in `vendor`.
|
||||
|
||||
## Updating a dependency ##
|
||||
|
||||
If you need to update a dependency then run
|
||||
|
||||
GO111MODULE=on go get -u github.com/pkg/errors
|
||||
GO111MODULE=on go mod vendor
|
||||
|
||||
Check in a single commit as above.
|
||||
|
||||
|
||||
Generated
+134
-31
@@ -5,7 +5,6 @@
|
||||
<meta name="generator" content="pandoc" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
|
||||
<meta name="author" content="Nick Craig-Wood" />
|
||||
<meta name="dcterms.date" content="2020-05-27" />
|
||||
<title>rclone(1) User Manual</title>
|
||||
<style type="text/css">
|
||||
code{white-space: pre-wrap;}
|
||||
@@ -18,31 +17,31 @@
|
||||
<header id="title-block-header">
|
||||
<h1 class="title">rclone(1) User Manual</h1>
|
||||
<p class="author">Nick Craig-Wood</p>
|
||||
<p class="date">May 27, 2020</p>
|
||||
<p class="date">Jun 24, 2020</p>
|
||||
</header>
|
||||
<h1 id="rclone-syncs-your-files-to-cloud-storage">Rclone syncs your files to cloud storage</h1>
|
||||
<p><img width="50%" src="https://rclone.org/img/logo_on_light__horizontal_color.svg" alt="rclone logo" style="float:right; padding: 5px;" ></p>
|
||||
<ul>
|
||||
<li><a href="#about">About rclone</a></li>
|
||||
<li><a href="#what">What can rclone do for you</a></li>
|
||||
<li><a href="#features">What features does rclone have</a></li>
|
||||
<li><a href="#providers">What providers does rclone support</a></li>
|
||||
<li><a href="#what">What can rclone do for you?</a></li>
|
||||
<li><a href="#features">What features does rclone have?</a></li>
|
||||
<li><a href="#providers">What providers does rclone support?</a></li>
|
||||
<li><a href="https://rclone.org/downloads/">Download</a></li>
|
||||
<li><a href="https://rclone.org/install/">Install</a></li>
|
||||
<li><a href="https://rclone.org/donate/">Donate.</a></li>
|
||||
</ul>
|
||||
<h2 id="about">About rclone</h2>
|
||||
<p>Rclone is a command line program to manage files on cloud storage. It is a feature rich alternative to cloud vendors’ web storage interfaces. <a href="#providers">Over 40 cloud storage products</a> support rclone including S3 object stores, business & consumer file storage services, as well as standard transfer protocols.</p>
|
||||
<p>Rclone has powerful cloud equivalents to the unix commands rsync, cp, mv, mount, ls, ncdu, tree, rm, and cat. Rclone’s familiar syntax includes shell pipeline support, and <code>--dry-run</code> protection. It can be used at the command line, in scripts or via its <a href="/rc">API</a>.</p>
|
||||
<p>Users have called rclone <em>“The Swiss army knife of cloud storage”</em> and <em>“Technology indistinguishable from magic”</em>.</p>
|
||||
<p>Rclone really looks after your data. It preserves timestamps and verifies your data at all times. Transfers over limited bandwidth; intermittent connections, or subject to quota can be restarted, from the last good file transferred. You can <a href="https://rclone.org/commands/rclone_check/">check</a> the integrity of your files. Where possible, rclone employs server side transfers to minimise local bandwidth use and transfers from one provider to another without using your local disk.</p>
|
||||
<p>Rclone has powerful cloud equivalents to the unix commands rsync, cp, mv, mount, ls, ncdu, tree, rm, and cat. Rclone’s familiar syntax includes shell pipeline support, and <code>--dry-run</code> protection. It is used at the command line, in scripts or via its <a href="/rc">API</a>.</p>
|
||||
<p>Users call rclone <em>“The Swiss army knife of cloud storage”</em>, and <em>“Technology indistinguishable from magic”</em>.</p>
|
||||
<p>Rclone really looks after your data. It preserves timestamps and verifies checksums at all times. Transfers over limited bandwidth; intermittent connections, or subject to quota can be restarted, from the last good file transferred. You can <a href="https://rclone.org/commands/rclone_check/">check</a> the integrity of your files. Where possible, rclone employs server side transfers to minimise local bandwidth use and transfers from one provider to another without using local disk.</p>
|
||||
<p>Virtual backends wrap local and cloud file systems to apply <a href="https://rclone.org/crypt/">encryption</a>, <a href="https://rclone.org/cache/">caching</a>, <a href="https://rclone.org/chunker/">chunking</a> and <a href="https://rclone.org/union/">joining</a>.</p>
|
||||
<p>Rclone can <a href="https://rclone.org/commands/rclone_mount/">mount</a> any local, cloud or virtual filesystem as a disk on Windows, macOS, linux and FreeBSD, and also serve these over <a href="https://rclone.org/commands/rclone_serve_sftp/">SFTP</a>, <a href="https://rclone.org/commands/rclone_serve_http/">HTTP</a>, <a href="https://rclone.org/commands/rclone_serve_webdav/">WebDAV</a>, <a href="https://rclone.org/commands/rclone_serve_ftp/">FTP</a> and <a href="https://rclone.org/commands/rclone_serve_dlna/">DLNA</a>.</p>
|
||||
<p>Rclone <a href="https://rclone.org/commands/rclone_mount/">mounts</a> any local, cloud or virtual filesystem as a disk on Windows, macOS, linux and FreeBSD, and also serves these over <a href="https://rclone.org/commands/rclone_serve_sftp/">SFTP</a>, <a href="https://rclone.org/commands/rclone_serve_http/">HTTP</a>, <a href="https://rclone.org/commands/rclone_serve_webdav/">WebDAV</a>, <a href="https://rclone.org/commands/rclone_serve_ftp/">FTP</a> and <a href="https://rclone.org/commands/rclone_serve_dlna/">DLNA</a>.</p>
|
||||
<p>Rclone is mature, open source software originally inspired by rsync and written in <a href="https://golang.org">Go</a>. The friendly support community are familiar with varied use cases. Official Ubuntu, Debian, Fedora, Brew and Chocolatey repos. include rclone. For the latest version <a href="https://rclone.org/downloads/">downloading from rclone.org</a> is recommended.</p>
|
||||
<p>Rclone is widely used on Linux, Windows and Mac. Third party developers have built innovative backup, restore, GUI and business process solutions using the rclone command line or API.</p>
|
||||
<p>Let rclone do the heavy lifting of communicating with cloud storage.</p>
|
||||
<h2 id="what">What can rclone do for you</h2>
|
||||
<p>Rclone can help you:</p>
|
||||
<p>Rclone is widely used on Linux, Windows and Mac. Third party developers create innovative backup, restore, GUI and business process solutions using the rclone command line or API.</p>
|
||||
<p>Rclone does the heavy lifting of communicating with cloud storage.</p>
|
||||
<h2 id="what">What can rclone do for you?</h2>
|
||||
<p>Rclone helps you:</p>
|
||||
<ul>
|
||||
<li>Backup (and encrypt) files to cloud storage</li>
|
||||
<li>Restore (and decrypt) files from cloud storage</li>
|
||||
@@ -71,7 +70,7 @@
|
||||
<li>Experimental <a href="https://rclone.org/gui/">Web based GUI</a></li>
|
||||
</ul>
|
||||
<h2 id="providers">Supported providers</h2>
|
||||
<p>(There are many other providers, built on standard protocols such as WebDAV or S3, that work out of the box.)</p>
|
||||
<p>(There are many others, built on standard protocols such as WebDAV or S3, that work out of the box.)</p>
|
||||
<ul>
|
||||
<li>1Fichier</li>
|
||||
<li>Alibaba Cloud (Aliyun) Object Storage System (OSS)</li>
|
||||
@@ -1564,9 +1563,11 @@ if src is directory
|
||||
<li><a href="https://rclone.org/commands/rclone/">rclone</a> - Show help for rclone commands, flags and backends.</li>
|
||||
</ul>
|
||||
<h1 id="rclone-obscure">rclone obscure</h1>
|
||||
<p>Obscure password for use in the rclone.conf</p>
|
||||
<p>Obscure password for use in the rclone config file</p>
|
||||
<h2 id="synopsis-52">Synopsis</h2>
|
||||
<p>Obscure password for use in the rclone.conf</p>
|
||||
<p>In the rclone config file, human readable passwords are obscured. Obscuring them is done by encrypting them and writing them out in base64. This is <strong>not</strong> a secure way of encrypting these passwords as rclone can decrypt them - it is to prevent “eyedropping” - namely someone seeing a password in the rclone config file by accident.</p>
|
||||
<p>Many equally important things (like access tokens) are not obscured in the config file. However it is very hard to shoulder surf a 64 character hex token.</p>
|
||||
<p>If you want to encrypt the config file then please use config file encryption - see <a href="https://rclone.org/commands/rclone_config/">rclone config</a> for more info.</p>
|
||||
<pre><code>rclone obscure password [flags]</code></pre>
|
||||
<h2 id="options-52">Options</h2>
|
||||
<pre><code> -h, --help help for obscure</code></pre>
|
||||
@@ -5865,7 +5866,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--use-json-log Use json log format.
|
||||
--use-mmap Use mmap allocator (see docs).
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.0")
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.2")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)</code></pre>
|
||||
<h2 id="backend-flags">Backend Flags</h2>
|
||||
<p>These flags are available for every command. They control the backends and may be set in the config file.</p>
|
||||
@@ -5920,7 +5921,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s)
|
||||
--cache-info-age Duration How long to cache file structure information (directory listings, file size, times etc). (default 6h0m0s)
|
||||
--cache-plex-insecure string Skip all certificate verification when connecting to the Plex server
|
||||
--cache-plex-password string The password of the Plex user
|
||||
--cache-plex-password string The password of the Plex user (obscured)
|
||||
--cache-plex-url string The URL of the Plex server
|
||||
--cache-plex-username string The username of the Plex user
|
||||
--cache-read-retries int How many times to retry a read from a cache storage. (default 10)
|
||||
@@ -5940,8 +5941,8 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
-L, --copy-links Follow symlinks and copy the pointed to item.
|
||||
--crypt-directory-name-encryption Option to either encrypt directory names or leave them intact. (default true)
|
||||
--crypt-filename-encryption string How to encrypt the filenames. (default "standard")
|
||||
--crypt-password string Password or pass phrase for encryption.
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended.
|
||||
--crypt-password string Password or pass phrase for encryption. (obscured)
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended. (obscured)
|
||||
--crypt-remote string Remote to encrypt/decrypt.
|
||||
--crypt-show-mapping For all files listed show how the names encrypt.
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
|
||||
@@ -5992,7 +5993,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--ftp-encoding MultiEncoder This sets the encoding for the backend. (default Slash,Del,Ctl,RightSpace,Dot)
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-pass string FTP password
|
||||
--ftp-pass string FTP password (obscured)
|
||||
--ftp-port string FTP port, leave blank to use default (21)
|
||||
--ftp-tls Use FTP over TLS (Implicit)
|
||||
--ftp-user string FTP username, leave blank for current username, $USER
|
||||
@@ -6029,7 +6030,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--koofr-encoding MultiEncoder This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--koofr-endpoint string The Koofr API endpoint to use (default "https://app.koofr.net")
|
||||
--koofr-mountid string Mount ID of the mount to use. If omitted, the primary mount is used.
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured)
|
||||
--koofr-setmtime Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend. (default true)
|
||||
--koofr-user string Your Koofr user name
|
||||
-l, --links Translate symlinks to/from regular files with a '.rclonelink' extension
|
||||
@@ -6042,7 +6043,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--local-nounc string Disable UNC (long path names) conversion on Windows
|
||||
--mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true)
|
||||
--mailru-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--mailru-pass string Password
|
||||
--mailru-pass string Password (obscured)
|
||||
--mailru-speedup-enable Skip full upload if there is another file with same data hash. (default true)
|
||||
--mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash). (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf")
|
||||
--mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3G)
|
||||
@@ -6051,7 +6052,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--mega-debug Output more debug from Mega.
|
||||
--mega-encoding MultiEncoder This sets the encoding for the backend. (default Slash,InvalidUtf8,Dot)
|
||||
--mega-hard-delete Delete files permanently rather than putting them into the trash.
|
||||
--mega-pass string Password.
|
||||
--mega-pass string Password. (obscured)
|
||||
--mega-user string User name
|
||||
-x, --one-file-system Don't cross filesystem boundaries (unix/macOS only).
|
||||
--onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes). (default 10M)
|
||||
@@ -6064,7 +6065,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--onedrive-server-side-across-configs Allow server side operations (eg copy) to work across different onedrive configs.
|
||||
--opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size. (default 10M)
|
||||
--opendrive-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
|
||||
--opendrive-password string Password.
|
||||
--opendrive-password string Password. (obscured)
|
||||
--opendrive-username string Username
|
||||
--pcloud-client-id string Pcloud App Client Id
|
||||
--pcloud-client-secret string Pcloud App Client Secret
|
||||
@@ -6115,19 +6116,19 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--seafile-create-library Should rclone create a library if it doesn't exist
|
||||
--seafile-encoding MultiEncoder This sets the encoding for the backend. (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
|
||||
--seafile-library string Name of the library. Leave blank to access all non-encrypted libraries.
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line.
|
||||
--seafile-pass string Password
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line. (obscured)
|
||||
--seafile-pass string Password (obscured)
|
||||
--seafile-url string URL of seafile host to connect to
|
||||
--seafile-user string User name (usually email address)
|
||||
--sftp-ask-password Allow asking for SFTP password when needed.
|
||||
--sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available.
|
||||
--sftp-host string SSH host to connect to
|
||||
--sftp-key-file string Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file. (obscured)
|
||||
--sftp-key-pem string Raw PEM-encoded private key, If specified, will override key_file parameter.
|
||||
--sftp-key-use-agent When set forces the usage of the ssh-agent.
|
||||
--sftp-md5sum-command string The command used to read md5 hashes. Leave blank for autodetect.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent. (obscured)
|
||||
--sftp-path-override string Override path used by SSH connection.
|
||||
--sftp-port string SSH port, leave blank to use default (22)
|
||||
--sftp-set-modtime Set the modified time on the remote if set. (default true)
|
||||
@@ -6185,7 +6186,7 @@ Showing nodes accounting for 1537.03kB, 100% of 1537.03kB total
|
||||
--union-upstreams string List of space separated upstreams.
|
||||
--webdav-bearer-token string Bearer token instead of user/pass (eg a Macaroon)
|
||||
--webdav-bearer-token-command string Command to run to get a bearer token
|
||||
--webdav-pass string Password.
|
||||
--webdav-pass string Password. (obscured)
|
||||
--webdav-url string URL of http host to connect to
|
||||
--webdav-user string User name
|
||||
--webdav-vendor string Name of the Webdav site/service/software you are using
|
||||
@@ -9304,6 +9305,7 @@ chunk_total_size = 10G</code></pre>
|
||||
</ul>
|
||||
<h4 id="cache-plex-password">–cache-plex-password</h4>
|
||||
<p>The password of the Plex user</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: plex_password</li>
|
||||
<li>Env Var: RCLONE_CACHE_PLEX_PASSWORD</li>
|
||||
@@ -10195,6 +10197,7 @@ $ rclone -q ls secret:
|
||||
</ul>
|
||||
<h4 id="crypt-password">–crypt-password</h4>
|
||||
<p>Password or pass phrase for encryption.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: password</li>
|
||||
<li>Env Var: RCLONE_CRYPT_PASSWORD</li>
|
||||
@@ -10203,6 +10206,7 @@ $ rclone -q ls secret:
|
||||
</ul>
|
||||
<h4 id="crypt-password2">–crypt-password2</h4>
|
||||
<p>Password or pass phrase for salt. Optional but recommended. Should be different to the previous password.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: password2</li>
|
||||
<li>Env Var: RCLONE_CRYPT_PASSWORD2</li>
|
||||
@@ -10621,6 +10625,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="ftp-pass">–ftp-pass</h4>
|
||||
<p>FTP password</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: pass</li>
|
||||
<li>Env Var: RCLONE_FTP_PASS</li>
|
||||
@@ -11379,7 +11384,7 @@ trashed=false and 'c' in parents</code></pre>
|
||||
<li>When deleting with <code>rclone rmdir</code> or <code>rclone purge</code> the shortcut is deleted not the linked folder.</li>
|
||||
<li><strong>NB</strong> When deleting with <code>rclone remove</code> or <code>rclone mount</code> the contents of the linked folder will be deleted.</li>
|
||||
</ul>
|
||||
<p>It isn’t currently possible to create shortcuts with rclone.</p>
|
||||
<p>The <a href="https://rclone.org/commands/rclone_backend/">rclone backend</a> command can be used to create shortcuts.</p>
|
||||
<p>Shortcuts can be completely ignored with the <code>--drive-skip-shortcuts</code> flag or the corresponding <code>skip_shortcuts</code> configuration setting.</p>
|
||||
<h3 id="emptying-trash">Emptying trash</h3>
|
||||
<p>If you wish to empty your trash you can use the <code>rclone cleanup remote:</code> command which will permanently delete all your trashed files. This command does not take any path arguments.</p>
|
||||
@@ -11810,6 +11815,7 @@ trashed=false and 'c' in parents</code></pre>
|
||||
</ul>
|
||||
<h4 id="drive-impersonate">–drive-impersonate</h4>
|
||||
<p>Impersonate this user when using a service account.</p>
|
||||
<p>Note that if this is used then “root_folder_id” will be ignored.</p>
|
||||
<ul>
|
||||
<li>Config: impersonate</li>
|
||||
<li>Env Var: RCLONE_DRIVE_IMPERSONATE</li>
|
||||
@@ -12776,6 +12782,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="koofr-password">–koofr-password</h4>
|
||||
<p>Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: password</li>
|
||||
<li>Env Var: RCLONE_KOOFR_PASSWORD</li>
|
||||
@@ -12976,6 +12983,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="mailru-pass">–mailru-pass</h4>
|
||||
<p>Password</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: pass</li>
|
||||
<li>Env Var: RCLONE_MAILRU_PASS</li>
|
||||
@@ -13223,6 +13231,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="mega-pass">–mega-pass</h4>
|
||||
<p>Password.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: pass</li>
|
||||
<li>Env Var: RCLONE_MEGA_PASS</li>
|
||||
@@ -14022,6 +14031,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="opendrive-password">–opendrive-password</h4>
|
||||
<p>Password.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: password</li>
|
||||
<li>Env Var: RCLONE_OPENDRIVE_PASSWORD</li>
|
||||
@@ -15244,6 +15254,7 @@ http://my.seafile.server/d/9ea2455f6f55478bbb0d/</code></pre>
|
||||
</ul>
|
||||
<h4 id="seafile-pass">–seafile-pass</h4>
|
||||
<p>Password</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: pass</li>
|
||||
<li>Env Var: RCLONE_SEAFILE_PASS</li>
|
||||
@@ -15268,6 +15279,7 @@ http://my.seafile.server/d/9ea2455f6f55478bbb0d/</code></pre>
|
||||
</ul>
|
||||
<h4 id="seafile-library-key">–seafile-library-key</h4>
|
||||
<p>Library password (for encrypted libraries only). Leave blank if you pass it through the command line.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: library_key</li>
|
||||
<li>Env Var: RCLONE_SEAFILE_LIBRARY_KEY</li>
|
||||
@@ -15426,6 +15438,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="sftp-pass">–sftp-pass</h4>
|
||||
<p>SSH password, leave blank to use ssh-agent.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: pass</li>
|
||||
<li>Env Var: RCLONE_SFTP_PASS</li>
|
||||
@@ -15451,6 +15464,7 @@ y/e/d> y</code></pre>
|
||||
<h4 id="sftp-key-file-pass">–sftp-key-file-pass</h4>
|
||||
<p>The passphrase to decrypt the PEM-encoded private key file.</p>
|
||||
<p>Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys in the new OpenSSH format can’t be used.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: key_file_pass</li>
|
||||
<li>Env Var: RCLONE_SFTP_KEY_FILE_PASS</li>
|
||||
@@ -16368,6 +16382,7 @@ y/e/d> y</code></pre>
|
||||
</ul>
|
||||
<h4 id="webdav-pass">–webdav-pass</h4>
|
||||
<p>Password.</p>
|
||||
<p><strong>NB</strong> Input to this must be obscured - see <a href="https://rclone.org/commands/rclone_obscure/">rclone obscure</a>.</p>
|
||||
<ul>
|
||||
<li>Config: pass</li>
|
||||
<li>Env Var: RCLONE_WEBDAV_PASS</li>
|
||||
@@ -17045,6 +17060,94 @@ $ tree /tmp/b
|
||||
<li>“error”: return an error based on option value</li>
|
||||
</ul>
|
||||
<h1 id="changelog">Changelog</h1>
|
||||
<h2 id="v1.52.2---2020-06-24">v1.52.2 - 2020-06-24</h2>
|
||||
<p><a href="https://github.com/rclone/rclone/compare/v1.52.1...v1.52.2">See commits</a></p>
|
||||
<ul>
|
||||
<li>Bug Fixes
|
||||
<ul>
|
||||
<li>build
|
||||
<ul>
|
||||
<li>Fix docker release build action (Nick Craig-Wood)</li>
|
||||
<li>Fix custom timezone in Docker image (NoLooseEnds)</li>
|
||||
</ul></li>
|
||||
<li>check: Fix misleading message which printed errors instead of differences (Nick Craig-Wood)</li>
|
||||
<li>errors: Add WSAECONNREFUSED and more to the list of retriable Windows errors (Nick Craig-Wood)</li>
|
||||
<li>rcd: Fix incorrect prometheus metrics (Gary Kim)</li>
|
||||
<li>serve restic: Fix flags so they use environment variables (Nick Craig-Wood)</li>
|
||||
<li>serve webdav: Fix flags so they use environment variables (Nick Craig-Wood)</li>
|
||||
<li>sync: Fix –track-renames-strategy modtime (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Drive
|
||||
<ul>
|
||||
<li>Fix not being able to delete a directory with a trashed shortcut (Nick Craig-Wood)</li>
|
||||
<li>Fix creating a directory inside a shortcut (Nick Craig-Wood)</li>
|
||||
<li>Fix –drive-impersonate with cached root_folder_id (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>SFTP
|
||||
<ul>
|
||||
<li>Fix SSH key PEM loading (Zac Rubin)</li>
|
||||
</ul></li>
|
||||
<li>Swift
|
||||
<ul>
|
||||
<li>Speed up deletes by not retrying segment container deletes (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Tardigrade
|
||||
<ul>
|
||||
<li>Upgrade to uplink v1.1.1 (Caleb Case)</li>
|
||||
</ul></li>
|
||||
<li>WebDAV
|
||||
<ul>
|
||||
<li>Fix free/used display for rclone about/df for certain backends (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
</ul>
|
||||
<h2 id="v1.52.1---2020-06-10">v1.52.1 - 2020-06-10</h2>
|
||||
<p><a href="https://github.com/rclone/rclone/compare/v1.52.0...v1.52.1">See commits</a></p>
|
||||
<ul>
|
||||
<li>Bug Fixes
|
||||
<ul>
|
||||
<li>lib/file: Fix SetSparse on Windows 7 which fixes downloads of files > 250MB (Nick Craig-Wood)</li>
|
||||
<li>build
|
||||
<ul>
|
||||
<li>Update go.mod to go1.14 to enable -mod=vendor build (Nick Craig-Wood)</li>
|
||||
<li>Remove quicktest from Dockerfile (Nick Craig-Wood)</li>
|
||||
<li>Build Docker images with GitHub actions (Matteo Pietro Dazzi)</li>
|
||||
<li>Update Docker build workflows (Nick Craig-Wood)</li>
|
||||
<li>Set user_allow_other in /etc/fuse.conf in the Docker image (Nick Craig-Wood)</li>
|
||||
<li>Fix xgo build after go1.14 go.mod update (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>docs
|
||||
<ul>
|
||||
<li>Add link to source and modified time to footer of every page (Nick Craig-Wood)</li>
|
||||
<li>Remove manually set dates and use git dates instead (Nick Craig-Wood)</li>
|
||||
<li>Minor tense, punctuation, brevity and positivity changes for the home page (edwardxml)</li>
|
||||
<li>Remove leading slash in page reference in footer when present (Nick Craig-Wood)</li>
|
||||
<li>Note commands which need obscured input in the docs (Nick Craig-Wood)</li>
|
||||
<li>obscure: Write more help as we are referencing it elsewhere (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
</ul></li>
|
||||
<li>VFS
|
||||
<ul>
|
||||
<li>Fix OS vs Unix path confusion - fixes ChangeNotify on Windows (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Drive
|
||||
<ul>
|
||||
<li>Fix missing items when listing using –fast-list / ListR (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Putio
|
||||
<ul>
|
||||
<li>Fix panic on Object.Open (Cenk Alti)</li>
|
||||
</ul></li>
|
||||
<li>S3
|
||||
<ul>
|
||||
<li>Fix upload of single files into buckets without create permission (Nick Craig-Wood)</li>
|
||||
<li>Fix –header-upload (Nick Craig-Wood)</li>
|
||||
</ul></li>
|
||||
<li>Tardigrade
|
||||
<ul>
|
||||
<li>Fix listing bug by upgrading to v1.0.7</li>
|
||||
<li>Set UserAgent to rclone (Caleb Case)</li>
|
||||
</ul></li>
|
||||
</ul>
|
||||
<h2 id="v1.52.0---2020-05-27">v1.52.0 - 2020-05-27</h2>
|
||||
<p>Special thanks to Martin Michlmayr for proof reading and correcting all the docs and Edward Barker for helping re-write the front page.</p>
|
||||
<p><a href="https://github.com/rclone/rclone/compare/v1.51.0...v1.52.0">See commits</a></p>
|
||||
|
||||
@@ -1,15 +1,15 @@
|
||||
% rclone(1) User Manual
|
||||
% Nick Craig-Wood
|
||||
% May 27, 2020
|
||||
% Jun 24, 2020
|
||||
|
||||
# Rclone syncs your files to cloud storage
|
||||
|
||||
<img width="50%" src="https://rclone.org/img/logo_on_light__horizontal_color.svg" alt="rclone logo" style="float:right; padding: 5px;" >
|
||||
|
||||
- [About rclone](#about)
|
||||
- [What can rclone do for you](#what)
|
||||
- [What features does rclone have](#features)
|
||||
- [What providers does rclone support](#providers)
|
||||
- [What can rclone do for you?](#what)
|
||||
- [What features does rclone have?](#features)
|
||||
- [What providers does rclone support?](#providers)
|
||||
- [Download](https://rclone.org/downloads/)
|
||||
- [Install](https://rclone.org/install/)
|
||||
- [Donate.](https://rclone.org/donate/)
|
||||
@@ -24,20 +24,20 @@ services, as well as standard transfer protocols.
|
||||
|
||||
Rclone has powerful cloud equivalents to the unix commands rsync, cp,
|
||||
mv, mount, ls, ncdu, tree, rm, and cat. Rclone's familiar syntax
|
||||
includes shell pipeline support, and `--dry-run` protection. It can be
|
||||
includes shell pipeline support, and `--dry-run` protection. It is
|
||||
used at the command line, in scripts or via its [API](/rc).
|
||||
|
||||
Users have called rclone *"The Swiss army knife of cloud storage"* and
|
||||
Users call rclone *"The Swiss army knife of cloud storage"*, and
|
||||
*"Technology indistinguishable from magic"*.
|
||||
|
||||
Rclone really looks after your data. It preserves timestamps and
|
||||
verifies your data at all times. Transfers over limited bandwidth;
|
||||
verifies checksums at all times. Transfers over limited bandwidth;
|
||||
intermittent connections, or subject to quota can be restarted, from
|
||||
the last good file transferred. You can
|
||||
[check](https://rclone.org/commands/rclone_check/) the integrity of your files. Where
|
||||
possible, rclone employs server side transfers to minimise local
|
||||
bandwidth use and transfers from one provider to another without
|
||||
using your local disk.
|
||||
using local disk.
|
||||
|
||||
Virtual backends wrap local and cloud file systems to apply
|
||||
[encryption](https://rclone.org/crypt/),
|
||||
@@ -45,9 +45,9 @@ Virtual backends wrap local and cloud file systems to apply
|
||||
[chunking](https://rclone.org/chunker/) and
|
||||
[joining](https://rclone.org/union/).
|
||||
|
||||
Rclone can [mount](https://rclone.org/commands/rclone_mount/) any local, cloud or
|
||||
Rclone [mounts](https://rclone.org/commands/rclone_mount/) any local, cloud or
|
||||
virtual filesystem as a disk on Windows,
|
||||
macOS, linux and FreeBSD, and also serve these over
|
||||
macOS, linux and FreeBSD, and also serves these over
|
||||
[SFTP](https://rclone.org/commands/rclone_serve_sftp/),
|
||||
[HTTP](https://rclone.org/commands/rclone_serve_http/),
|
||||
[WebDAV](https://rclone.org/commands/rclone_serve_webdav/),
|
||||
@@ -61,14 +61,14 @@ Fedora, Brew and Chocolatey repos. include rclone. For the latest
|
||||
version [downloading from rclone.org](https://rclone.org/downloads/) is recommended.
|
||||
|
||||
Rclone is widely used on Linux, Windows and Mac. Third party
|
||||
developers have built innovative backup, restore, GUI and business
|
||||
developers create innovative backup, restore, GUI and business
|
||||
process solutions using the rclone command line or API.
|
||||
|
||||
Let rclone do the heavy lifting of communicating with cloud storage.
|
||||
Rclone does the heavy lifting of communicating with cloud storage.
|
||||
|
||||
## What can rclone do for you {#what}
|
||||
## What can rclone do for you? {#what}
|
||||
|
||||
Rclone can help you:
|
||||
Rclone helps you:
|
||||
|
||||
- Backup (and encrypt) files to cloud storage
|
||||
- Restore (and decrypt) files from cloud storage
|
||||
@@ -96,7 +96,7 @@ Rclone can help you:
|
||||
|
||||
## Supported providers {#providers}
|
||||
|
||||
(There are many other providers, built on standard protocols such as
|
||||
(There are many others, built on standard protocols such as
|
||||
WebDAV or S3, that work out of the box.)
|
||||
|
||||
|
||||
@@ -3126,11 +3126,24 @@ See the [global flags page](https://rclone.org/flags/) for global options not li
|
||||
|
||||
# rclone obscure
|
||||
|
||||
Obscure password for use in the rclone.conf
|
||||
Obscure password for use in the rclone config file
|
||||
|
||||
## Synopsis
|
||||
|
||||
Obscure password for use in the rclone.conf
|
||||
In the rclone config file, human readable passwords are
|
||||
obscured. Obscuring them is done by encrypting them and writing them
|
||||
out in base64. This is **not** a secure way of encrypting these
|
||||
passwords as rclone can decrypt them - it is to prevent "eyedropping"
|
||||
- namely someone seeing a password in the rclone config file by
|
||||
accident.
|
||||
|
||||
Many equally important things (like access tokens) are not obscured in
|
||||
the config file. However it is very hard to shoulder surf a 64
|
||||
character hex token.
|
||||
|
||||
If you want to encrypt the config file then please use config file
|
||||
encryption - see [rclone config](https://rclone.org/commands/rclone_config/) for more
|
||||
info.
|
||||
|
||||
```
|
||||
rclone obscure password [flags]
|
||||
@@ -9670,7 +9683,7 @@ These flags are available for every command.
|
||||
--use-json-log Use json log format.
|
||||
--use-mmap Use mmap allocator (see docs).
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.0")
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.2")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)
|
||||
```
|
||||
|
||||
@@ -9731,7 +9744,7 @@ and may be set in the config file.
|
||||
--cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s)
|
||||
--cache-info-age Duration How long to cache file structure information (directory listings, file size, times etc). (default 6h0m0s)
|
||||
--cache-plex-insecure string Skip all certificate verification when connecting to the Plex server
|
||||
--cache-plex-password string The password of the Plex user
|
||||
--cache-plex-password string The password of the Plex user (obscured)
|
||||
--cache-plex-url string The URL of the Plex server
|
||||
--cache-plex-username string The username of the Plex user
|
||||
--cache-read-retries int How many times to retry a read from a cache storage. (default 10)
|
||||
@@ -9751,8 +9764,8 @@ and may be set in the config file.
|
||||
-L, --copy-links Follow symlinks and copy the pointed to item.
|
||||
--crypt-directory-name-encryption Option to either encrypt directory names or leave them intact. (default true)
|
||||
--crypt-filename-encryption string How to encrypt the filenames. (default "standard")
|
||||
--crypt-password string Password or pass phrase for encryption.
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended.
|
||||
--crypt-password string Password or pass phrase for encryption. (obscured)
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended. (obscured)
|
||||
--crypt-remote string Remote to encrypt/decrypt.
|
||||
--crypt-show-mapping For all files listed show how the names encrypt.
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
|
||||
@@ -9803,7 +9816,7 @@ and may be set in the config file.
|
||||
--ftp-encoding MultiEncoder This sets the encoding for the backend. (default Slash,Del,Ctl,RightSpace,Dot)
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-pass string FTP password
|
||||
--ftp-pass string FTP password (obscured)
|
||||
--ftp-port string FTP port, leave blank to use default (21)
|
||||
--ftp-tls Use FTP over TLS (Implicit)
|
||||
--ftp-user string FTP username, leave blank for current username, $USER
|
||||
@@ -9840,7 +9853,7 @@ and may be set in the config file.
|
||||
--koofr-encoding MultiEncoder This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--koofr-endpoint string The Koofr API endpoint to use (default "https://app.koofr.net")
|
||||
--koofr-mountid string Mount ID of the mount to use. If omitted, the primary mount is used.
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured)
|
||||
--koofr-setmtime Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend. (default true)
|
||||
--koofr-user string Your Koofr user name
|
||||
-l, --links Translate symlinks to/from regular files with a '.rclonelink' extension
|
||||
@@ -9853,7 +9866,7 @@ and may be set in the config file.
|
||||
--local-nounc string Disable UNC (long path names) conversion on Windows
|
||||
--mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true)
|
||||
--mailru-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--mailru-pass string Password
|
||||
--mailru-pass string Password (obscured)
|
||||
--mailru-speedup-enable Skip full upload if there is another file with same data hash. (default true)
|
||||
--mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash). (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf")
|
||||
--mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3G)
|
||||
@@ -9862,7 +9875,7 @@ and may be set in the config file.
|
||||
--mega-debug Output more debug from Mega.
|
||||
--mega-encoding MultiEncoder This sets the encoding for the backend. (default Slash,InvalidUtf8,Dot)
|
||||
--mega-hard-delete Delete files permanently rather than putting them into the trash.
|
||||
--mega-pass string Password.
|
||||
--mega-pass string Password. (obscured)
|
||||
--mega-user string User name
|
||||
-x, --one-file-system Don't cross filesystem boundaries (unix/macOS only).
|
||||
--onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes). (default 10M)
|
||||
@@ -9875,7 +9888,7 @@ and may be set in the config file.
|
||||
--onedrive-server-side-across-configs Allow server side operations (eg copy) to work across different onedrive configs.
|
||||
--opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size. (default 10M)
|
||||
--opendrive-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
|
||||
--opendrive-password string Password.
|
||||
--opendrive-password string Password. (obscured)
|
||||
--opendrive-username string Username
|
||||
--pcloud-client-id string Pcloud App Client Id
|
||||
--pcloud-client-secret string Pcloud App Client Secret
|
||||
@@ -9926,19 +9939,19 @@ and may be set in the config file.
|
||||
--seafile-create-library Should rclone create a library if it doesn't exist
|
||||
--seafile-encoding MultiEncoder This sets the encoding for the backend. (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
|
||||
--seafile-library string Name of the library. Leave blank to access all non-encrypted libraries.
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line.
|
||||
--seafile-pass string Password
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line. (obscured)
|
||||
--seafile-pass string Password (obscured)
|
||||
--seafile-url string URL of seafile host to connect to
|
||||
--seafile-user string User name (usually email address)
|
||||
--sftp-ask-password Allow asking for SFTP password when needed.
|
||||
--sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available.
|
||||
--sftp-host string SSH host to connect to
|
||||
--sftp-key-file string Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file. (obscured)
|
||||
--sftp-key-pem string Raw PEM-encoded private key, If specified, will override key_file parameter.
|
||||
--sftp-key-use-agent When set forces the usage of the ssh-agent.
|
||||
--sftp-md5sum-command string The command used to read md5 hashes. Leave blank for autodetect.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent. (obscured)
|
||||
--sftp-path-override string Override path used by SSH connection.
|
||||
--sftp-port string SSH port, leave blank to use default (22)
|
||||
--sftp-set-modtime Set the modified time on the remote if set. (default true)
|
||||
@@ -9996,7 +10009,7 @@ and may be set in the config file.
|
||||
--union-upstreams string List of space separated upstreams.
|
||||
--webdav-bearer-token string Bearer token instead of user/pass (eg a Macaroon)
|
||||
--webdav-bearer-token-command string Command to run to get a bearer token
|
||||
--webdav-pass string Password.
|
||||
--webdav-pass string Password. (obscured)
|
||||
--webdav-url string URL of http host to connect to
|
||||
--webdav-user string User name
|
||||
--webdav-vendor string Name of the Webdav site/service/software you are using
|
||||
@@ -13622,6 +13635,8 @@ The username of the Plex user
|
||||
|
||||
The password of the Plex user
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: plex_password
|
||||
- Env Var: RCLONE_CACHE_PLEX_PASSWORD
|
||||
- Type: string
|
||||
@@ -14880,6 +14895,8 @@ NB If filename_encryption is "off" then this option will do nothing.
|
||||
|
||||
Password or pass phrase for encryption.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_CRYPT_PASSWORD
|
||||
- Type: string
|
||||
@@ -14890,6 +14907,8 @@ Password or pass phrase for encryption.
|
||||
Password or pass phrase for salt. Optional but recommended.
|
||||
Should be different to the previous password.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: password2
|
||||
- Env Var: RCLONE_CRYPT_PASSWORD2
|
||||
- Type: string
|
||||
@@ -15480,6 +15499,8 @@ FTP port, leave blank to use default (21)
|
||||
|
||||
FTP password
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_FTP_PASS
|
||||
- Type: string
|
||||
@@ -16426,7 +16447,7 @@ For shortcuts pointing to folders:
|
||||
- When deleting with `rclone rmdir` or `rclone purge` the shortcut is deleted not the linked folder.
|
||||
- **NB** When deleting with `rclone remove` or `rclone mount` the contents of the linked folder will be deleted.
|
||||
|
||||
It isn't currently possible to create shortcuts with rclone.
|
||||
The [rclone backend](https://rclone.org/commands/rclone_backend/) command can be used to create shortcuts.
|
||||
|
||||
Shortcuts can be completely ignored with the `--drive-skip-shortcuts` flag
|
||||
or the corresponding `skip_shortcuts` configuration setting.
|
||||
@@ -16812,6 +16833,9 @@ Size of listing chunk 100-1000. 0 to disable.
|
||||
|
||||
Impersonate this user when using a service account.
|
||||
|
||||
Note that if this is used then "root_folder_id" will be ignored.
|
||||
|
||||
|
||||
- Config: impersonate
|
||||
- Env Var: RCLONE_DRIVE_IMPERSONATE
|
||||
- Type: string
|
||||
@@ -18312,6 +18336,8 @@ Your Koofr user name
|
||||
|
||||
Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_KOOFR_PASSWORD
|
||||
- Type: string
|
||||
@@ -18544,6 +18570,8 @@ User name (usually email)
|
||||
|
||||
Password
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_MAILRU_PASS
|
||||
- Type: string
|
||||
@@ -18842,6 +18870,8 @@ User name
|
||||
|
||||
Password.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_MEGA_PASS
|
||||
- Type: string
|
||||
@@ -19818,6 +19848,8 @@ Username
|
||||
|
||||
Password.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_OPENDRIVE_PASSWORD
|
||||
- Type: string
|
||||
@@ -21394,6 +21426,8 @@ User name (usually email address)
|
||||
|
||||
Password
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_SEAFILE_PASS
|
||||
- Type: string
|
||||
@@ -21421,6 +21455,8 @@ Name of the library. Leave blank to access all non-encrypted libraries.
|
||||
|
||||
Library password (for encrypted libraries only). Leave blank if you pass it through the command line.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: library_key
|
||||
- Env Var: RCLONE_SEAFILE_LIBRARY_KEY
|
||||
- Type: string
|
||||
@@ -21650,6 +21686,8 @@ SSH port, leave blank to use default (22)
|
||||
|
||||
SSH password, leave blank to use ssh-agent.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_SFTP_PASS
|
||||
- Type: string
|
||||
@@ -21680,6 +21718,8 @@ The passphrase to decrypt the PEM-encoded private key file.
|
||||
Only PEM encrypted key files (old OpenSSH format) are supported. Encrypted keys
|
||||
in the new OpenSSH format can't be used.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: key_file_pass
|
||||
- Env Var: RCLONE_SFTP_KEY_FILE_PASS
|
||||
- Type: string
|
||||
@@ -22778,6 +22818,8 @@ User name
|
||||
|
||||
Password.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](https://rclone.org/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_WEBDAV_PASS
|
||||
- Type: string
|
||||
@@ -23596,6 +23638,66 @@ Options:
|
||||
|
||||
# Changelog
|
||||
|
||||
## v1.52.2 - 2020-06-24
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.52.1...v1.52.2)
|
||||
|
||||
* Bug Fixes
|
||||
* build
|
||||
* Fix docker release build action (Nick Craig-Wood)
|
||||
* Fix custom timezone in Docker image (NoLooseEnds)
|
||||
* check: Fix misleading message which printed errors instead of differences (Nick Craig-Wood)
|
||||
* errors: Add WSAECONNREFUSED and more to the list of retriable Windows errors (Nick Craig-Wood)
|
||||
* rcd: Fix incorrect prometheus metrics (Gary Kim)
|
||||
* serve restic: Fix flags so they use environment variables (Nick Craig-Wood)
|
||||
* serve webdav: Fix flags so they use environment variables (Nick Craig-Wood)
|
||||
* sync: Fix --track-renames-strategy modtime (Nick Craig-Wood)
|
||||
* Drive
|
||||
* Fix not being able to delete a directory with a trashed shortcut (Nick Craig-Wood)
|
||||
* Fix creating a directory inside a shortcut (Nick Craig-Wood)
|
||||
* Fix --drive-impersonate with cached root_folder_id (Nick Craig-Wood)
|
||||
* SFTP
|
||||
* Fix SSH key PEM loading (Zac Rubin)
|
||||
* Swift
|
||||
* Speed up deletes by not retrying segment container deletes (Nick Craig-Wood)
|
||||
* Tardigrade
|
||||
* Upgrade to uplink v1.1.1 (Caleb Case)
|
||||
* WebDAV
|
||||
* Fix free/used display for rclone about/df for certain backends (Nick Craig-Wood)
|
||||
|
||||
## v1.52.1 - 2020-06-10
|
||||
|
||||
[See commits](https://github.com/rclone/rclone/compare/v1.52.0...v1.52.1)
|
||||
|
||||
* Bug Fixes
|
||||
* lib/file: Fix SetSparse on Windows 7 which fixes downloads of files > 250MB (Nick Craig-Wood)
|
||||
* build
|
||||
* Update go.mod to go1.14 to enable -mod=vendor build (Nick Craig-Wood)
|
||||
* Remove quicktest from Dockerfile (Nick Craig-Wood)
|
||||
* Build Docker images with GitHub actions (Matteo Pietro Dazzi)
|
||||
* Update Docker build workflows (Nick Craig-Wood)
|
||||
* Set user_allow_other in /etc/fuse.conf in the Docker image (Nick Craig-Wood)
|
||||
* Fix xgo build after go1.14 go.mod update (Nick Craig-Wood)
|
||||
* docs
|
||||
* Add link to source and modified time to footer of every page (Nick Craig-Wood)
|
||||
* Remove manually set dates and use git dates instead (Nick Craig-Wood)
|
||||
* Minor tense, punctuation, brevity and positivity changes for the home page (edwardxml)
|
||||
* Remove leading slash in page reference in footer when present (Nick Craig-Wood)
|
||||
* Note commands which need obscured input in the docs (Nick Craig-Wood)
|
||||
* obscure: Write more help as we are referencing it elsewhere (Nick Craig-Wood)
|
||||
* VFS
|
||||
* Fix OS vs Unix path confusion - fixes ChangeNotify on Windows (Nick Craig-Wood)
|
||||
* Drive
|
||||
* Fix missing items when listing using --fast-list / ListR (Nick Craig-Wood)
|
||||
* Putio
|
||||
* Fix panic on Object.Open (Cenk Alti)
|
||||
* S3
|
||||
* Fix upload of single files into buckets without create permission (Nick Craig-Wood)
|
||||
* Fix --header-upload (Nick Craig-Wood)
|
||||
* Tardigrade
|
||||
* Fix listing bug by upgrading to v1.0.7
|
||||
* Set UserAgent to rclone (Caleb Case)
|
||||
|
||||
## v1.52.0 - 2020-05-27
|
||||
|
||||
Special thanks to Martin Michlmayr for proof reading and correcting
|
||||
|
||||
Generated
+158
-36
@@ -1,6 +1,6 @@
|
||||
rclone(1) User Manual
|
||||
Nick Craig-Wood
|
||||
May 27, 2020
|
||||
Jun 24, 2020
|
||||
|
||||
|
||||
|
||||
@@ -8,9 +8,9 @@ RCLONE SYNCS YOUR FILES TO CLOUD STORAGE
|
||||
|
||||
|
||||
- About rclone
|
||||
- What can rclone do for you
|
||||
- What features does rclone have
|
||||
- What providers does rclone support
|
||||
- What can rclone do for you?
|
||||
- What features does rclone have?
|
||||
- What providers does rclone support?
|
||||
- Download
|
||||
- Install
|
||||
- Donate.
|
||||
@@ -26,26 +26,26 @@ transfer protocols.
|
||||
|
||||
Rclone has powerful cloud equivalents to the unix commands rsync, cp,
|
||||
mv, mount, ls, ncdu, tree, rm, and cat. Rclone’s familiar syntax
|
||||
includes shell pipeline support, and --dry-run protection. It can be
|
||||
used at the command line, in scripts or via its API.
|
||||
includes shell pipeline support, and --dry-run protection. It is used at
|
||||
the command line, in scripts or via its API.
|
||||
|
||||
Users have called rclone _“The Swiss army knife of cloud storage”_ and
|
||||
Users call rclone _“The Swiss army knife of cloud storage”_, and
|
||||
_“Technology indistinguishable from magic”_.
|
||||
|
||||
Rclone really looks after your data. It preserves timestamps and
|
||||
verifies your data at all times. Transfers over limited bandwidth;
|
||||
verifies checksums at all times. Transfers over limited bandwidth;
|
||||
intermittent connections, or subject to quota can be restarted, from the
|
||||
last good file transferred. You can check the integrity of your files.
|
||||
Where possible, rclone employs server side transfers to minimise local
|
||||
bandwidth use and transfers from one provider to another without using
|
||||
your local disk.
|
||||
local disk.
|
||||
|
||||
Virtual backends wrap local and cloud file systems to apply encryption,
|
||||
caching, chunking and joining.
|
||||
|
||||
Rclone can mount any local, cloud or virtual filesystem as a disk on
|
||||
Windows, macOS, linux and FreeBSD, and also serve these over SFTP, HTTP,
|
||||
WebDAV, FTP and DLNA.
|
||||
Rclone mounts any local, cloud or virtual filesystem as a disk on
|
||||
Windows, macOS, linux and FreeBSD, and also serves these over SFTP,
|
||||
HTTP, WebDAV, FTP and DLNA.
|
||||
|
||||
Rclone is mature, open source software originally inspired by rsync and
|
||||
written in Go. The friendly support community are familiar with varied
|
||||
@@ -54,15 +54,15 @@ include rclone. For the latest version downloading from rclone.org is
|
||||
recommended.
|
||||
|
||||
Rclone is widely used on Linux, Windows and Mac. Third party developers
|
||||
have built innovative backup, restore, GUI and business process
|
||||
solutions using the rclone command line or API.
|
||||
create innovative backup, restore, GUI and business process solutions
|
||||
using the rclone command line or API.
|
||||
|
||||
Let rclone do the heavy lifting of communicating with cloud storage.
|
||||
Rclone does the heavy lifting of communicating with cloud storage.
|
||||
|
||||
|
||||
What can rclone do for you
|
||||
What can rclone do for you?
|
||||
|
||||
Rclone can help you:
|
||||
Rclone helps you:
|
||||
|
||||
- Backup (and encrypt) files to cloud storage
|
||||
- Restore (and decrypt) files from cloud storage
|
||||
@@ -94,8 +94,8 @@ Features
|
||||
|
||||
Supported providers
|
||||
|
||||
(There are many other providers, built on standard protocols such as
|
||||
WebDAV or S3, that work out of the box.)
|
||||
(There are many others, built on standard protocols such as WebDAV or
|
||||
S3, that work out of the box.)
|
||||
|
||||
- 1Fichier
|
||||
- Alibaba Cloud (Aliyun) Object Storage System (OSS)
|
||||
@@ -3157,12 +3157,23 @@ SEE ALSO
|
||||
RCLONE OBSCURE
|
||||
|
||||
|
||||
Obscure password for use in the rclone.conf
|
||||
Obscure password for use in the rclone config file
|
||||
|
||||
|
||||
Synopsis
|
||||
|
||||
Obscure password for use in the rclone.conf
|
||||
In the rclone config file, human readable passwords are obscured.
|
||||
Obscuring them is done by encrypting them and writing them out in
|
||||
base64. This is NOT a secure way of encrypting these passwords as rclone
|
||||
can decrypt them - it is to prevent “eyedropping” - namely someone
|
||||
seeing a password in the rclone config file by accident.
|
||||
|
||||
Many equally important things (like access tokens) are not obscured in
|
||||
the config file. However it is very hard to shoulder surf a 64 character
|
||||
hex token.
|
||||
|
||||
If you want to encrypt the config file then please use config file
|
||||
encryption - see rclone config for more info.
|
||||
|
||||
rclone obscure password [flags]
|
||||
|
||||
@@ -9741,7 +9752,7 @@ These flags are available for every command.
|
||||
--use-json-log Use json log format.
|
||||
--use-mmap Use mmap allocator (see docs).
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.0")
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.2")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)
|
||||
|
||||
|
||||
@@ -9801,7 +9812,7 @@ and may be set in the config file.
|
||||
--cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s)
|
||||
--cache-info-age Duration How long to cache file structure information (directory listings, file size, times etc). (default 6h0m0s)
|
||||
--cache-plex-insecure string Skip all certificate verification when connecting to the Plex server
|
||||
--cache-plex-password string The password of the Plex user
|
||||
--cache-plex-password string The password of the Plex user (obscured)
|
||||
--cache-plex-url string The URL of the Plex server
|
||||
--cache-plex-username string The username of the Plex user
|
||||
--cache-read-retries int How many times to retry a read from a cache storage. (default 10)
|
||||
@@ -9821,8 +9832,8 @@ and may be set in the config file.
|
||||
-L, --copy-links Follow symlinks and copy the pointed to item.
|
||||
--crypt-directory-name-encryption Option to either encrypt directory names or leave them intact. (default true)
|
||||
--crypt-filename-encryption string How to encrypt the filenames. (default "standard")
|
||||
--crypt-password string Password or pass phrase for encryption.
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended.
|
||||
--crypt-password string Password or pass phrase for encryption. (obscured)
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended. (obscured)
|
||||
--crypt-remote string Remote to encrypt/decrypt.
|
||||
--crypt-show-mapping For all files listed show how the names encrypt.
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
|
||||
@@ -9873,7 +9884,7 @@ and may be set in the config file.
|
||||
--ftp-encoding MultiEncoder This sets the encoding for the backend. (default Slash,Del,Ctl,RightSpace,Dot)
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-pass string FTP password
|
||||
--ftp-pass string FTP password (obscured)
|
||||
--ftp-port string FTP port, leave blank to use default (21)
|
||||
--ftp-tls Use FTP over TLS (Implicit)
|
||||
--ftp-user string FTP username, leave blank for current username, $USER
|
||||
@@ -9910,7 +9921,7 @@ and may be set in the config file.
|
||||
--koofr-encoding MultiEncoder This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--koofr-endpoint string The Koofr API endpoint to use (default "https://app.koofr.net")
|
||||
--koofr-mountid string Mount ID of the mount to use. If omitted, the primary mount is used.
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured)
|
||||
--koofr-setmtime Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend. (default true)
|
||||
--koofr-user string Your Koofr user name
|
||||
-l, --links Translate symlinks to/from regular files with a '.rclonelink' extension
|
||||
@@ -9923,7 +9934,7 @@ and may be set in the config file.
|
||||
--local-nounc string Disable UNC (long path names) conversion on Windows
|
||||
--mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true)
|
||||
--mailru-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--mailru-pass string Password
|
||||
--mailru-pass string Password (obscured)
|
||||
--mailru-speedup-enable Skip full upload if there is another file with same data hash. (default true)
|
||||
--mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash). (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf")
|
||||
--mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3G)
|
||||
@@ -9932,7 +9943,7 @@ and may be set in the config file.
|
||||
--mega-debug Output more debug from Mega.
|
||||
--mega-encoding MultiEncoder This sets the encoding for the backend. (default Slash,InvalidUtf8,Dot)
|
||||
--mega-hard-delete Delete files permanently rather than putting them into the trash.
|
||||
--mega-pass string Password.
|
||||
--mega-pass string Password. (obscured)
|
||||
--mega-user string User name
|
||||
-x, --one-file-system Don't cross filesystem boundaries (unix/macOS only).
|
||||
--onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes). (default 10M)
|
||||
@@ -9945,7 +9956,7 @@ and may be set in the config file.
|
||||
--onedrive-server-side-across-configs Allow server side operations (eg copy) to work across different onedrive configs.
|
||||
--opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size. (default 10M)
|
||||
--opendrive-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
|
||||
--opendrive-password string Password.
|
||||
--opendrive-password string Password. (obscured)
|
||||
--opendrive-username string Username
|
||||
--pcloud-client-id string Pcloud App Client Id
|
||||
--pcloud-client-secret string Pcloud App Client Secret
|
||||
@@ -9996,19 +10007,19 @@ and may be set in the config file.
|
||||
--seafile-create-library Should rclone create a library if it doesn't exist
|
||||
--seafile-encoding MultiEncoder This sets the encoding for the backend. (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
|
||||
--seafile-library string Name of the library. Leave blank to access all non-encrypted libraries.
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line.
|
||||
--seafile-pass string Password
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line. (obscured)
|
||||
--seafile-pass string Password (obscured)
|
||||
--seafile-url string URL of seafile host to connect to
|
||||
--seafile-user string User name (usually email address)
|
||||
--sftp-ask-password Allow asking for SFTP password when needed.
|
||||
--sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available.
|
||||
--sftp-host string SSH host to connect to
|
||||
--sftp-key-file string Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file. (obscured)
|
||||
--sftp-key-pem string Raw PEM-encoded private key, If specified, will override key_file parameter.
|
||||
--sftp-key-use-agent When set forces the usage of the ssh-agent.
|
||||
--sftp-md5sum-command string The command used to read md5 hashes. Leave blank for autodetect.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent. (obscured)
|
||||
--sftp-path-override string Override path used by SSH connection.
|
||||
--sftp-port string SSH port, leave blank to use default (22)
|
||||
--sftp-set-modtime Set the modified time on the remote if set. (default true)
|
||||
@@ -10066,7 +10077,7 @@ and may be set in the config file.
|
||||
--union-upstreams string List of space separated upstreams.
|
||||
--webdav-bearer-token string Bearer token instead of user/pass (eg a Macaroon)
|
||||
--webdav-bearer-token-command string Command to run to get a bearer token
|
||||
--webdav-pass string Password.
|
||||
--webdav-pass string Password. (obscured)
|
||||
--webdav-url string URL of http host to connect to
|
||||
--webdav-user string User name
|
||||
--webdav-vendor string Name of the Webdav site/service/software you are using
|
||||
@@ -13652,6 +13663,8 @@ The username of the Plex user
|
||||
|
||||
The password of the Plex user
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: plex_password
|
||||
- Env Var: RCLONE_CACHE_PLEX_PASSWORD
|
||||
- Type: string
|
||||
@@ -14887,6 +14900,8 @@ NB If filename_encryption is “off” then this option will do nothing.
|
||||
|
||||
Password or pass phrase for encryption.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_CRYPT_PASSWORD
|
||||
- Type: string
|
||||
@@ -14897,6 +14912,8 @@ Password or pass phrase for encryption.
|
||||
Password or pass phrase for salt. Optional but recommended. Should be
|
||||
different to the previous password.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: password2
|
||||
- Env Var: RCLONE_CRYPT_PASSWORD2
|
||||
- Type: string
|
||||
@@ -15478,6 +15495,8 @@ FTP port, leave blank to use default (21)
|
||||
|
||||
FTP password
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_FTP_PASS
|
||||
- Type: string
|
||||
@@ -16418,7 +16437,7 @@ For shortcuts pointing to folders:
|
||||
- NB When deleting with rclone remove or rclone mount the contents of
|
||||
the linked folder will be deleted.
|
||||
|
||||
It isn’t currently possible to create shortcuts with rclone.
|
||||
The rclone backend command can be used to create shortcuts.
|
||||
|
||||
Shortcuts can be completely ignored with the --drive-skip-shortcuts flag
|
||||
or the corresponding skip_shortcuts configuration setting.
|
||||
@@ -16826,6 +16845,8 @@ Size of listing chunk 100-1000. 0 to disable.
|
||||
|
||||
Impersonate this user when using a service account.
|
||||
|
||||
Note that if this is used then “root_folder_id” will be ignored.
|
||||
|
||||
- Config: impersonate
|
||||
- Env Var: RCLONE_DRIVE_IMPERSONATE
|
||||
- Type: string
|
||||
@@ -18315,6 +18336,8 @@ Your Koofr user name
|
||||
Your Koofr password for rclone (generate one at
|
||||
https://app.koofr.net/app/admin/preferences/password)
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_KOOFR_PASSWORD
|
||||
- Type: string
|
||||
@@ -18550,6 +18573,8 @@ User name (usually email)
|
||||
|
||||
Password
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_MAILRU_PASS
|
||||
- Type: string
|
||||
@@ -18848,6 +18873,8 @@ User name
|
||||
|
||||
Password.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_MEGA_PASS
|
||||
- Type: string
|
||||
@@ -19834,6 +19861,8 @@ Username
|
||||
|
||||
Password.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_OPENDRIVE_PASSWORD
|
||||
- Type: string
|
||||
@@ -21379,6 +21408,8 @@ User name (usually email address)
|
||||
|
||||
Password
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_SEAFILE_PASS
|
||||
- Type: string
|
||||
@@ -21407,6 +21438,8 @@ Name of the library. Leave blank to access all non-encrypted libraries.
|
||||
Library password (for encrypted libraries only). Leave blank if you pass
|
||||
it through the command line.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: library_key
|
||||
- Env Var: RCLONE_SEAFILE_LIBRARY_KEY
|
||||
- Type: string
|
||||
@@ -21634,6 +21667,8 @@ SSH port, leave blank to use default (22)
|
||||
|
||||
SSH password, leave blank to use ssh-agent.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_SFTP_PASS
|
||||
- Type: string
|
||||
@@ -21666,6 +21701,8 @@ The passphrase to decrypt the PEM-encoded private key file.
|
||||
Only PEM encrypted key files (old OpenSSH format) are supported.
|
||||
Encrypted keys in the new OpenSSH format can’t be used.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: key_file_pass
|
||||
- Env Var: RCLONE_SFTP_KEY_FILE_PASS
|
||||
- Type: string
|
||||
@@ -22826,6 +22863,8 @@ User name
|
||||
|
||||
Password.
|
||||
|
||||
NB Input to this must be obscured - see rclone obscure.
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_WEBDAV_PASS
|
||||
- Type: string
|
||||
@@ -23597,6 +23636,89 @@ Options:
|
||||
CHANGELOG
|
||||
|
||||
|
||||
v1.52.2 - 2020-06-24
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- build
|
||||
- Fix docker release build action (Nick Craig-Wood)
|
||||
- Fix custom timezone in Docker image (NoLooseEnds)
|
||||
- check: Fix misleading message which printed errors instead of
|
||||
differences (Nick Craig-Wood)
|
||||
- errors: Add WSAECONNREFUSED and more to the list of retriable
|
||||
Windows errors (Nick Craig-Wood)
|
||||
- rcd: Fix incorrect prometheus metrics (Gary Kim)
|
||||
- serve restic: Fix flags so they use environment variables (Nick
|
||||
Craig-Wood)
|
||||
- serve webdav: Fix flags so they use environment variables (Nick
|
||||
Craig-Wood)
|
||||
- sync: Fix –track-renames-strategy modtime (Nick Craig-Wood)
|
||||
- Drive
|
||||
- Fix not being able to delete a directory with a trashed shortcut
|
||||
(Nick Craig-Wood)
|
||||
- Fix creating a directory inside a shortcut (Nick Craig-Wood)
|
||||
- Fix –drive-impersonate with cached root_folder_id (Nick
|
||||
Craig-Wood)
|
||||
- SFTP
|
||||
- Fix SSH key PEM loading (Zac Rubin)
|
||||
- Swift
|
||||
- Speed up deletes by not retrying segment container deletes (Nick
|
||||
Craig-Wood)
|
||||
- Tardigrade
|
||||
- Upgrade to uplink v1.1.1 (Caleb Case)
|
||||
- WebDAV
|
||||
- Fix free/used display for rclone about/df for certain backends
|
||||
(Nick Craig-Wood)
|
||||
|
||||
|
||||
v1.52.1 - 2020-06-10
|
||||
|
||||
See commits
|
||||
|
||||
- Bug Fixes
|
||||
- lib/file: Fix SetSparse on Windows 7 which fixes downloads of
|
||||
files > 250MB (Nick Craig-Wood)
|
||||
- build
|
||||
- Update go.mod to go1.14 to enable -mod=vendor build (Nick
|
||||
Craig-Wood)
|
||||
- Remove quicktest from Dockerfile (Nick Craig-Wood)
|
||||
- Build Docker images with GitHub actions (Matteo Pietro
|
||||
Dazzi)
|
||||
- Update Docker build workflows (Nick Craig-Wood)
|
||||
- Set user_allow_other in /etc/fuse.conf in the Docker image
|
||||
(Nick Craig-Wood)
|
||||
- Fix xgo build after go1.14 go.mod update (Nick Craig-Wood)
|
||||
- docs
|
||||
- Add link to source and modified time to footer of every page
|
||||
(Nick Craig-Wood)
|
||||
- Remove manually set dates and use git dates instead (Nick
|
||||
Craig-Wood)
|
||||
- Minor tense, punctuation, brevity and positivity changes for
|
||||
the home page (edwardxml)
|
||||
- Remove leading slash in page reference in footer when
|
||||
present (Nick Craig-Wood)
|
||||
- Note commands which need obscured input in the docs (Nick
|
||||
Craig-Wood)
|
||||
- obscure: Write more help as we are referencing it elsewhere
|
||||
(Nick Craig-Wood)
|
||||
- VFS
|
||||
- Fix OS vs Unix path confusion - fixes ChangeNotify on Windows
|
||||
(Nick Craig-Wood)
|
||||
- Drive
|
||||
- Fix missing items when listing using –fast-list / ListR (Nick
|
||||
Craig-Wood)
|
||||
- Putio
|
||||
- Fix panic on Object.Open (Cenk Alti)
|
||||
- S3
|
||||
- Fix upload of single files into buckets without create
|
||||
permission (Nick Craig-Wood)
|
||||
- Fix –header-upload (Nick Craig-Wood)
|
||||
- Tardigrade
|
||||
- Fix listing bug by upgrading to v1.0.7
|
||||
- Set UserAgent to rclone (Caleb Case)
|
||||
|
||||
|
||||
v1.52.0 - 2020-05-27
|
||||
|
||||
Special thanks to Martin Michlmayr for proof reading and correcting all
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
SHELL = bash
|
||||
# Branch we are working on
|
||||
BRANCH := $(or $(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
|
||||
BRANCH := $(or $(APPVEYOR_REPO_BRANCH),$(TRAVIS_BRANCH),$(BUILD_SOURCEBRANCHNAME),$(lastword $(subst /, ,$(GITHUB_REF))),$(shell git rev-parse --abbrev-ref HEAD))
|
||||
# Tag of the current commit, if any. If this is not "" then we are building a release
|
||||
RELEASE_TAG := $(shell git tag -l --points-at HEAD)
|
||||
# Version of last release (may not be on this branch)
|
||||
@@ -27,7 +27,7 @@ ifndef RELEASE_TAG
|
||||
TAG := $(TAG)-beta
|
||||
endif
|
||||
GO_VERSION := $(shell go version)
|
||||
GO_FILES := $(shell go list ./... )
|
||||
GO_FILES := $(shell go list ./... | grep -v /vendor/ )
|
||||
ifdef BETA_SUBDIR
|
||||
BETA_SUBDIR := /$(BETA_SUBDIR)
|
||||
endif
|
||||
@@ -61,10 +61,6 @@ vars:
|
||||
@echo GO_VERSION="'$(GO_VERSION)'"
|
||||
@echo BETA_URL="'$(BETA_URL)'"
|
||||
|
||||
btest:
|
||||
@echo "[$(TAG)]($(BETA_URL)) on branch [$(BRANCH)](https://github.com/rclone/rclone/tree/$(BRANCH)) (uploaded in 15-30 mins)" | xclip -r -sel clip
|
||||
@echo "Copied markdown of beta release to clip board"
|
||||
|
||||
version:
|
||||
@echo '$(TAG)'
|
||||
|
||||
@@ -90,43 +86,35 @@ check: rclone
|
||||
build_dep:
|
||||
go run bin/get-github-release.go -extract golangci-lint golangci/golangci-lint 'golangci-lint-.*\.tar\.gz'
|
||||
|
||||
# Get the release dependencies we only install on linux
|
||||
release_dep_linux:
|
||||
# Get the release dependencies
|
||||
release_dep:
|
||||
go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*_Linux_x86_64.tar.gz'
|
||||
go run bin/get-github-release.go -extract github-release aktau/github-release 'linux-amd64-github-release.tar.bz2'
|
||||
|
||||
# Get the release dependencies we only install on Windows
|
||||
release_dep_windows:
|
||||
GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
|
||||
|
||||
# Update dependencies
|
||||
showupdates:
|
||||
@echo "*** Direct dependencies that could be updated ***"
|
||||
@GO111MODULE=on go list -u -f '{{if (and (not (or .Main .Indirect)) .Update)}}{{.Path}}: {{.Version}} -> {{.Update.Version}}{{end}}' -m all 2> /dev/null
|
||||
|
||||
# Update direct and indirect dependencies and test dependencies
|
||||
update:
|
||||
GO111MODULE=on go get -u -t ./...
|
||||
-#GO111MODULE=on go get -d $(go list -m -f '{{if not (or .Main .Indirect)}}{{.Path}}{{end}}' all)
|
||||
GO111MODULE=on go get -u ./...
|
||||
GO111MODULE=on go mod tidy
|
||||
GO111MODULE=on go mod vendor
|
||||
|
||||
# Tidy the module dependencies
|
||||
tidy:
|
||||
GO111MODULE=on go mod tidy
|
||||
GO111MODULE=on go mod vendor
|
||||
|
||||
doc: rclone.1 MANUAL.html MANUAL.txt rcdocs commanddocs
|
||||
|
||||
rclone.1: MANUAL.md
|
||||
pandoc -s --from markdown-smart --to man MANUAL.md -o rclone.1
|
||||
pandoc -s --from markdown --to man MANUAL.md -o rclone.1
|
||||
|
||||
MANUAL.md: bin/make_manual.py docs/content/*.md commanddocs backenddocs
|
||||
./bin/make_manual.py
|
||||
|
||||
MANUAL.html: MANUAL.md
|
||||
pandoc -s --from markdown-smart --to html MANUAL.md -o MANUAL.html
|
||||
pandoc -s --from markdown --to html MANUAL.md -o MANUAL.html
|
||||
|
||||
MANUAL.txt: MANUAL.md
|
||||
pandoc -s --from markdown-smart --to plain MANUAL.md -o MANUAL.txt
|
||||
pandoc -s --from markdown --to plain MANUAL.md -o MANUAL.txt
|
||||
|
||||
commanddocs: rclone
|
||||
XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs docs/content/
|
||||
@@ -195,7 +183,14 @@ log_since_last_release:
|
||||
compile_all:
|
||||
go run bin/cross-compile.go -compile-only $(BUILDTAGS) $(TAG)
|
||||
|
||||
ci_upload:
|
||||
appveyor_upload:
|
||||
rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
|
||||
ifndef BRANCH_PATH
|
||||
rclone --config bin/travis.rclone.conf -v copy --include '*beta-latest*' --include version.txt build/ $(BETA_UPLOAD_ROOT)
|
||||
endif
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
circleci_upload:
|
||||
sudo chown -R $$USER build
|
||||
find build -type l -delete
|
||||
gzip -r9v build
|
||||
@@ -205,7 +200,10 @@ ifndef BRANCH_PATH
|
||||
endif
|
||||
@echo Beta release ready at $(BETA_URL)/testbuilds
|
||||
|
||||
ci_beta:
|
||||
travis_beta:
|
||||
ifeq (linux,$(filter linux,$(subst Linux,linux,$(TRAVIS_OS_NAME) $(AGENT_OS))))
|
||||
go run bin/get-github-release.go -extract nfpm goreleaser/nfpm 'nfpm_.*\.tar.gz'
|
||||
endif
|
||||
git log $(LAST_TAG).. > /tmp/git-log.txt
|
||||
go run bin/cross-compile.go -release beta-latest -git-log /tmp/git-log.txt $(BUILD_FLAGS) $(BUILDTAGS) $(TAG)
|
||||
rclone --config bin/travis.rclone.conf -v copy --exclude '*beta-latest*' build/ $(BETA_UPLOAD)
|
||||
@@ -214,7 +212,7 @@ ifndef BRANCH_PATH
|
||||
endif
|
||||
@echo Beta release ready at $(BETA_URL)
|
||||
|
||||
# Fetch the binary builds from GitHub actions
|
||||
# Fetch the binary builds from travis and appveyor
|
||||
fetch_binaries:
|
||||
rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/
|
||||
|
||||
|
||||
+13
-13
@@ -33,7 +33,7 @@ This file describes how to make the various kinds of releases
|
||||
* make startdev
|
||||
* # announce with forum post, twitter post, patreon post
|
||||
|
||||
Early in the next release cycle update the dependencies
|
||||
Early in the next release cycle update the vendored dependencies
|
||||
|
||||
* Review any pinned packages in go.mod and remove if possible
|
||||
* make update
|
||||
@@ -53,6 +53,7 @@ Can be fixed with
|
||||
|
||||
* GO111MODULE=on go get -u github.com/russross/blackfriday@v1.5.2
|
||||
* GO111MODULE=on go mod tidy
|
||||
* GO111MODULE=on go mod vendor
|
||||
|
||||
|
||||
## Making a point release
|
||||
@@ -62,14 +63,14 @@ If rclone needs a point release due to some horrendous bug:
|
||||
First make the release branch. If this is a second point release then
|
||||
this will be done already.
|
||||
|
||||
* BASE_TAG=v1.XX # eg v1.52
|
||||
* NEW_TAG=${BASE_TAG}.Y # eg v1.52.1
|
||||
* echo $BASE_TAG $NEW_TAG # v1.52 v1.52.1
|
||||
* git branch ${BASE_TAG} ${BASE_TAG}-stable
|
||||
* BASE_TAG=v1.XX # eg v1.49
|
||||
* NEW_TAG=${BASE_TAG}.Y # eg v1.49.1
|
||||
* echo $BASE_TAG $NEW_TAG # v1.49 v1.49.1
|
||||
* git branch ${BASE_TAG} ${BASE_TAG}-fixes
|
||||
|
||||
Now
|
||||
|
||||
* git co ${BASE_TAG}-stable
|
||||
* git co ${BASE_TAG}-fixes
|
||||
* git cherry-pick any fixes
|
||||
* Test (see above)
|
||||
* make NEXT_VERSION=${NEW_TAG} tag
|
||||
@@ -78,7 +79,7 @@ Now
|
||||
* git commit -a -v -m "Version ${NEW_TAG}"
|
||||
* git tag -d ${NEW_TAG}
|
||||
* git tag -s -m "Version ${NEW_TAG}" ${NEW_TAG}
|
||||
* git push --tags -u origin ${BASE_TAG}-stable
|
||||
* git push --tags -u origin ${BASE_TAG}-fixes
|
||||
* Wait for builds to complete
|
||||
* make BRANCH_PATH= TAG=${NEW_TAG} fetch_binaries
|
||||
* make TAG=${NEW_TAG} tarball
|
||||
@@ -91,21 +92,20 @@ Now
|
||||
* git co master
|
||||
* make VERSION=${NEW_TAG} startdev
|
||||
* # cherry pick the changes to the changelog and VERSION
|
||||
* git checkout ${BASE_TAG}-stable VERSION docs/content/changelog.md
|
||||
* git checkout ${BASE_TAG}-fixes VERSION docs/content/changelog.md
|
||||
* git commit --amend
|
||||
* git push
|
||||
* Announce!
|
||||
|
||||
## Making a manual build of docker
|
||||
|
||||
The rclone docker image should autobuild on via GitHub actions. If it doesn't
|
||||
The rclone docker image should autobuild on docker hub. If it doesn't
|
||||
or needs to be updated then rebuild like this.
|
||||
|
||||
```
|
||||
docker pull golang
|
||||
docker build --rm --ulimit memlock=67108864 -t rclone/rclone:1.52.0 -t rclone/rclone:1.52 -t rclone/rclone:1 -t rclone/rclone:latest .
|
||||
docker push rclone/rclone:1.52.0
|
||||
docker push rclone/rclone:1.52
|
||||
docker build -t rclone/rclone:1.49.1 -t rclone/rclone:1.49 -t rclone/rclone:1 -t rclone/rclone:latest .
|
||||
docker push rclone/rclone:1.49.1
|
||||
docker push rclone/rclone:1.49
|
||||
docker push rclone/rclone:1
|
||||
docker push rclone/rclone:latest
|
||||
```
|
||||
@@ -514,6 +514,10 @@ func (f *Fs) listAll(dirID string, title string, directoriesOnly bool, filesOnly
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -661,7 +665,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
return nil, err
|
||||
}
|
||||
// If not create it
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -692,7 +696,13 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -714,6 +724,10 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
|
||||
// create the destination directory if necessary
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcLeaf, srcDirectoryID, err := srcObj.fs.dirCache.FindPath(ctx, srcObj.remote, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -783,24 +797,54 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err = srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, dstRemote, true)
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
dstLeaf, dstDirectoryID, err := f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src parent
|
||||
_, srcDirectoryID, err := srcFs.dirCache.FindPath(ctx, srcRemote, false)
|
||||
findPath = srcRemote
|
||||
var srcDirectoryID string
|
||||
if srcRemote == "" {
|
||||
srcDirectoryID, err = srcFs.dirCache.RootParentID()
|
||||
} else {
|
||||
_, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -846,6 +890,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -989,7 +1037,7 @@ func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
if o.info != nil {
|
||||
return nil
|
||||
}
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return fs.ErrorObjectNotFound
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Package azureblob provides an interface to the Microsoft Azure blob object storage system
|
||||
|
||||
// +build !plan9,!solaris,go1.13
|
||||
// +build !plan9,!solaris
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// +build !plan9,!solaris,go1.13
|
||||
// +build !plan9,!solaris
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Test AzureBlob filesystem interface
|
||||
|
||||
// +build !plan9,!solaris,go1.13
|
||||
// +build !plan9,!solaris
|
||||
|
||||
package azureblob
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Build for azureblob for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
// +build plan9 solaris !go1.13
|
||||
// +build plan9 solaris
|
||||
|
||||
package azureblob
|
||||
@@ -337,11 +337,3 @@ type CopyFileRequest struct {
|
||||
Info map[string]string `json:"fileInfo,omitempty"` // This field stores the metadata that will be stored with the file. (REPLACE only)
|
||||
DestBucketID string `json:"destinationBucketId,omitempty"` // The destination ID of the bucket if set, if not the source bucket will be used
|
||||
}
|
||||
|
||||
// CopyPartRequest is the request for b2_copy_part - the response is UploadPartResponse
|
||||
type CopyPartRequest struct {
|
||||
SourceID string `json:"sourceFileId"` // The ID of the source file being copied.
|
||||
LargeFileID string `json:"largeFileId"` // The ID of the large file the part will belong to, as returned by b2_start_large_file.
|
||||
PartNumber int64 `json:"partNumber"` // Which part this is (starting from 1)
|
||||
Range string `json:"range,omitempty"` // The range of bytes to copy. If not provided, the whole source file will be copied.
|
||||
}
|
||||
+84
-128
@@ -33,7 +33,6 @@ import (
|
||||
"github.com/rclone/rclone/lib/bucket"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/pool"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -55,9 +54,6 @@ const (
|
||||
minChunkSize = 5 * fs.MebiByte
|
||||
defaultChunkSize = 96 * fs.MebiByte
|
||||
defaultUploadCutoff = 200 * fs.MebiByte
|
||||
largeFileCopyCutoff = 4 * fs.GibiByte // 5E9 is the max
|
||||
memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
|
||||
memoryPoolUseMmap = false
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -117,16 +113,6 @@ Files above this size will be uploaded in chunks of "--b2-chunk-size".
|
||||
This value should be set no larger than 4.657GiB (== 5GB).`,
|
||||
Default: defaultUploadCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "copy_cutoff",
|
||||
Help: `Cutoff for switching to multipart copy
|
||||
|
||||
Any files larger than this that need to be server side copied will be
|
||||
copied in chunks of this size.
|
||||
|
||||
The minimum is 0 and the maximum is 4.6GB.`,
|
||||
Default: largeFileCopyCutoff,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "chunk_size",
|
||||
Help: `Upload chunk size. Must fit in memory.
|
||||
@@ -164,18 +150,6 @@ The duration before the download authorization token will expire.
|
||||
The minimum value is 1 second. The maximum value is one week.`,
|
||||
Default: fs.Duration(7 * 24 * time.Hour),
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "memory_pool_flush_time",
|
||||
Default: memoryPoolFlushTime,
|
||||
Advanced: true,
|
||||
Help: `How often internal memory buffer pools will be flushed.
|
||||
Uploads which requires additional buffers (f.e multipart) will use memory pool for allocations.
|
||||
This option controls how often unused buffers will be removed from the pool.`,
|
||||
}, {
|
||||
Name: "memory_pool_use_mmap",
|
||||
Default: memoryPoolUseMmap,
|
||||
Advanced: true,
|
||||
Help: `Whether to use mmap buffers in internal memory pool.`,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -199,13 +173,10 @@ type Options struct {
|
||||
Versions bool `config:"versions"`
|
||||
HardDelete bool `config:"hard_delete"`
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
DisableCheckSum bool `config:"disable_checksum"`
|
||||
DownloadURL string `config:"download_url"`
|
||||
DownloadAuthorizationDuration fs.Duration `config:"download_auth_duration"`
|
||||
MemoryPoolFlushTime fs.Duration `config:"memory_pool_flush_time"`
|
||||
MemoryPoolUseMmap bool `config:"memory_pool_use_mmap"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
@@ -228,8 +199,7 @@ type Fs struct {
|
||||
uploads map[string][]*api.GetUploadURLResponse // Upload URLs by buckedID
|
||||
authMu sync.Mutex // lock for authorizing the account
|
||||
pacer *fs.Pacer // To pace and retry the API calls
|
||||
uploadToken *pacer.TokenDispenser // control concurrency
|
||||
pool *pool.Pool // memory pool
|
||||
bufferTokens chan []byte // control concurrency of multipart uploads
|
||||
}
|
||||
|
||||
// Object describes a b2 object
|
||||
@@ -365,6 +335,7 @@ func (f *Fs) setUploadChunkSize(cs fs.SizeSuffix) (old fs.SizeSuffix, err error)
|
||||
err = checkUploadChunkSize(cs)
|
||||
if err == nil {
|
||||
old, f.opt.ChunkSize = f.opt.ChunkSize, cs
|
||||
f.fillBufferTokens() // reset the buffer tokens
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -425,13 +396,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
_bucketType: make(map[string]string, 1),
|
||||
uploads: make(map[string][]*api.GetUploadURLResponse),
|
||||
pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
uploadToken: pacer.NewTokenDispenser(fs.Config.Transfers),
|
||||
pool: pool.New(
|
||||
time.Duration(opt.MemoryPoolFlushTime),
|
||||
int(opt.ChunkSize),
|
||||
fs.Config.Transfers,
|
||||
opt.MemoryPoolUseMmap,
|
||||
),
|
||||
}
|
||||
f.setRoot(root)
|
||||
f.features = (&fs.Features{
|
||||
@@ -446,6 +410,7 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
f.srv.SetHeader(testModeHeader, testMode)
|
||||
fs.Debugf(f, "Setting test header \"%s: %s\"", testModeHeader, testMode)
|
||||
}
|
||||
f.fillBufferTokens()
|
||||
err = f.authorizeAccount(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to authorize account")
|
||||
@@ -568,25 +533,32 @@ func (f *Fs) clearUploadURL(bucketID string) {
|
||||
f.uploadMu.Unlock()
|
||||
}
|
||||
|
||||
// getBuf gets a buffer of f.opt.ChunkSize and an upload token
|
||||
//
|
||||
// If noBuf is set then it just gets an upload token
|
||||
func (f *Fs) getBuf(noBuf bool) (buf []byte) {
|
||||
f.uploadToken.Get()
|
||||
if !noBuf {
|
||||
buf = f.pool.Get()
|
||||
// Fill up (or reset) the buffer tokens
|
||||
func (f *Fs) fillBufferTokens() {
|
||||
f.bufferTokens = make(chan []byte, fs.Config.Transfers)
|
||||
for i := 0; i < fs.Config.Transfers; i++ {
|
||||
f.bufferTokens <- nil
|
||||
}
|
||||
}
|
||||
|
||||
// getUploadBlock gets a block from the pool of size chunkSize
|
||||
func (f *Fs) getUploadBlock() []byte {
|
||||
buf := <-f.bufferTokens
|
||||
if buf == nil {
|
||||
buf = make([]byte, f.opt.ChunkSize)
|
||||
}
|
||||
// fs.Debugf(f, "Getting upload block %p", buf)
|
||||
return buf
|
||||
}
|
||||
|
||||
// putBuf returns a buffer to the memory pool and an upload token
|
||||
//
|
||||
// If noBuf is set then it just returns the upload token
|
||||
func (f *Fs) putBuf(buf []byte, noBuf bool) {
|
||||
if !noBuf {
|
||||
f.pool.Put(buf)
|
||||
// putUploadBlock returns a block to the pool of size chunkSize
|
||||
func (f *Fs) putUploadBlock(buf []byte) {
|
||||
buf = buf[:cap(buf)]
|
||||
if len(buf) != int(f.opt.ChunkSize) {
|
||||
panic("bad blocksize returned to pool")
|
||||
}
|
||||
f.uploadToken.Put()
|
||||
// fs.Debugf(f, "Returning upload block %p", buf)
|
||||
f.bufferTokens <- buf
|
||||
}
|
||||
|
||||
// Return an Object from a path
|
||||
@@ -1233,63 +1205,6 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
return f.purge(ctx, f.rootBucket, f.rootDirectory, true)
|
||||
}
|
||||
|
||||
// copy does a server side copy from dstObj <- srcObj
|
||||
//
|
||||
// If newInfo is nil then the metadata will be copied otherwise it
|
||||
// will be replaced with newInfo
|
||||
func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *api.File) (err error) {
|
||||
if srcObj.size >= int64(f.opt.CopyCutoff) {
|
||||
if newInfo == nil {
|
||||
newInfo, err = srcObj.getMetaData(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
up, err := f.newLargeUpload(ctx, dstObj, nil, srcObj, f.opt.CopyCutoff, true, newInfo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return up.Upload(ctx)
|
||||
}
|
||||
|
||||
dstBucket, dstPath := dstObj.split()
|
||||
err = f.makeBucket(ctx, dstBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
destBucketID, err := f.getBucketID(ctx, dstBucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_copy_file",
|
||||
}
|
||||
var request = api.CopyFileRequest{
|
||||
SourceID: srcObj.id,
|
||||
Name: f.opt.Enc.FromStandardPath(dstPath),
|
||||
DestBucketID: destBucketID,
|
||||
}
|
||||
if newInfo == nil {
|
||||
request.MetadataDirective = "COPY"
|
||||
} else {
|
||||
request.MetadataDirective = "REPLACE"
|
||||
request.ContentType = newInfo.ContentType
|
||||
request.Info = newInfo.Info
|
||||
}
|
||||
var response api.FileInfo
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return dstObj.decodeMetaDataFileInfo(&response)
|
||||
}
|
||||
|
||||
// Copy src to this remote using server side copy operations.
|
||||
//
|
||||
// This is stored with the remote path given
|
||||
@@ -1300,21 +1215,47 @@ func (f *Fs) copy(ctx context.Context, dstObj *Object, srcObj *Object, newInfo *
|
||||
//
|
||||
// If it isn't possible then return fs.ErrorCantCopy
|
||||
func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
|
||||
dstBucket, dstPath := f.split(remote)
|
||||
err := f.makeBucket(ctx, dstBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcObj, ok := src.(*Object)
|
||||
if !ok {
|
||||
fs.Debugf(src, "Can't copy - not same remote type")
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
// Temporary Object under construction
|
||||
dstObj := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
err := f.copy(ctx, dstObj, srcObj, nil)
|
||||
destBucketID, err := f.getBucketID(ctx, dstBucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return dstObj, nil
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_copy_file",
|
||||
}
|
||||
var request = api.CopyFileRequest{
|
||||
SourceID: srcObj.id,
|
||||
Name: f.opt.Enc.FromStandardPath(dstPath),
|
||||
MetadataDirective: "COPY",
|
||||
DestBucketID: destBucketID,
|
||||
}
|
||||
var response api.FileInfo
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
return f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
o := &Object{
|
||||
fs: f,
|
||||
remote: remote,
|
||||
}
|
||||
err = o.decodeMetaDataFileInfo(&response)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return o, nil
|
||||
}
|
||||
|
||||
// Hashes returns the supported hash sets.
|
||||
@@ -1357,7 +1298,7 @@ func (f *Fs) getDownloadAuthorization(ctx context.Context, bucket, remote string
|
||||
}
|
||||
|
||||
// PublicLink returns a link for downloading without account
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
bucket, bucketPath := f.split(remote)
|
||||
var RootURL string
|
||||
if f.opt.DownloadURL == "" {
|
||||
@@ -1585,10 +1526,28 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, bucketPath := o.split()
|
||||
info.Info[timeKey] = timeString(modTime)
|
||||
|
||||
// Copy to the same name, overwriting the metadata only
|
||||
return o.fs.copy(ctx, o, o, info)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_copy_file",
|
||||
}
|
||||
var request = api.CopyFileRequest{
|
||||
SourceID: o.id,
|
||||
Name: o.fs.opt.Enc.FromStandardPath(bucketPath), // copy to same name
|
||||
MetadataDirective: "REPLACE",
|
||||
ContentType: info.ContentType,
|
||||
Info: info.Info,
|
||||
}
|
||||
var response api.FileInfo
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
resp, err := o.fs.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
return o.fs.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return o.decodeMetaDataFileInfo(&response)
|
||||
}
|
||||
|
||||
// Storable returns if this object is storable
|
||||
@@ -1764,8 +1723,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
if size == -1 {
|
||||
// Check if the file is large enough for a chunked upload (needs to be at least two chunks)
|
||||
buf := o.fs.getBuf(false)
|
||||
|
||||
buf := o.fs.getUploadBlock()
|
||||
n, err := io.ReadFull(in, buf)
|
||||
if err == nil {
|
||||
bufReader := bufio.NewReader(in)
|
||||
@@ -1775,24 +1733,22 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
if err == nil {
|
||||
fs.Debugf(o, "File is big enough for chunked streaming")
|
||||
up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
|
||||
up, err := o.fs.newLargeUpload(ctx, o, in, src)
|
||||
if err != nil {
|
||||
o.fs.putBuf(buf, false)
|
||||
o.fs.putUploadBlock(buf)
|
||||
return err
|
||||
}
|
||||
// NB Stream returns the buffer and token
|
||||
return up.Stream(ctx, buf)
|
||||
} else if err == io.EOF || err == io.ErrUnexpectedEOF {
|
||||
fs.Debugf(o, "File has %d bytes, which makes only one chunk. Using direct upload.", n)
|
||||
defer o.fs.putBuf(buf, false)
|
||||
defer o.fs.putUploadBlock(buf)
|
||||
size = int64(n)
|
||||
in = bytes.NewReader(buf[:n])
|
||||
} else {
|
||||
o.fs.putBuf(buf, false)
|
||||
return err
|
||||
}
|
||||
} else if size > int64(o.fs.opt.UploadCutoff) {
|
||||
up, err := o.fs.newLargeUpload(ctx, o, in, src, o.fs.opt.ChunkSize, false, nil)
|
||||
up, err := o.fs.newLargeUpload(ctx, o, in, src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+152
-204
@@ -20,9 +20,7 @@ import (
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type hashAppendingReader struct {
|
||||
@@ -70,26 +68,20 @@ func newHashAppendingReader(in io.Reader, h gohash.Hash) *hashAppendingReader {
|
||||
|
||||
// largeUpload is used to control the upload of large files which need chunking
|
||||
type largeUpload struct {
|
||||
f *Fs // parent Fs
|
||||
o *Object // object being uploaded
|
||||
doCopy bool // doing copy rather than upload
|
||||
what string // text name of operation for logs
|
||||
in io.Reader // read the data from here
|
||||
wrap accounting.WrapFn // account parts being transferred
|
||||
id string // ID of the file being uploaded
|
||||
size int64 // total size
|
||||
parts int64 // calculated number of parts, if known
|
||||
sha1s []string // slice of SHA1s for each part
|
||||
uploadMu sync.Mutex // lock for upload variable
|
||||
uploads []*api.GetUploadPartURLResponse // result of get upload URL calls
|
||||
chunkSize int64 // chunk size to use
|
||||
src *Object // if copying, object we are reading from
|
||||
f *Fs // parent Fs
|
||||
o *Object // object being uploaded
|
||||
in io.Reader // read the data from here
|
||||
wrap accounting.WrapFn // account parts being transferred
|
||||
id string // ID of the file being uploaded
|
||||
size int64 // total size
|
||||
parts int64 // calculated number of parts, if known
|
||||
sha1s []string // slice of SHA1s for each part
|
||||
uploadMu sync.Mutex // lock for upload variable
|
||||
uploads []*api.GetUploadPartURLResponse // result of get upload URL calls
|
||||
}
|
||||
|
||||
// newLargeUpload starts an upload of object o from in with metadata in src
|
||||
//
|
||||
// If newInfo is set then metadata from that will be used instead of reading it from src
|
||||
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
|
||||
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo) (up *largeUpload, err error) {
|
||||
remote := o.remote
|
||||
size := src.Size()
|
||||
parts := int64(0)
|
||||
@@ -97,8 +89,8 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
|
||||
if size == -1 {
|
||||
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
|
||||
} else {
|
||||
parts = size / int64(chunkSize)
|
||||
if size%int64(chunkSize) != 0 {
|
||||
parts = size / int64(o.fs.opt.ChunkSize)
|
||||
if size%int64(o.fs.opt.ChunkSize) != 0 {
|
||||
parts++
|
||||
}
|
||||
if parts > maxParts {
|
||||
@@ -107,6 +99,7 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
|
||||
sha1SliceSize = parts
|
||||
}
|
||||
|
||||
modTime := src.ModTime(ctx)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_start_large_file",
|
||||
@@ -117,24 +110,18 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
|
||||
return nil, err
|
||||
}
|
||||
var request = api.StartLargeFileRequest{
|
||||
BucketID: bucketID,
|
||||
Name: f.opt.Enc.FromStandardPath(bucketPath),
|
||||
}
|
||||
if newInfo == nil {
|
||||
modTime := src.ModTime(ctx)
|
||||
request.ContentType = fs.MimeType(ctx, src)
|
||||
request.Info = map[string]string{
|
||||
BucketID: bucketID,
|
||||
Name: f.opt.Enc.FromStandardPath(bucketPath),
|
||||
ContentType: fs.MimeType(ctx, src),
|
||||
Info: map[string]string{
|
||||
timeKey: timeString(modTime),
|
||||
},
|
||||
}
|
||||
// Set the SHA1 if known
|
||||
if !o.fs.opt.DisableCheckSum {
|
||||
if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
|
||||
request.Info[sha1Key] = calculatedSha1
|
||||
}
|
||||
// Set the SHA1 if known
|
||||
if !o.fs.opt.DisableCheckSum || doCopy {
|
||||
if calculatedSha1, err := src.Hash(ctx, hash.SHA1); err == nil && calculatedSha1 != "" {
|
||||
request.Info[sha1Key] = calculatedSha1
|
||||
}
|
||||
}
|
||||
} else {
|
||||
request.ContentType = newInfo.ContentType
|
||||
request.Info = newInfo.Info
|
||||
}
|
||||
var response api.StartLargeFileResponse
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -144,24 +131,18 @@ func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
up = &largeUpload{
|
||||
f: f,
|
||||
o: o,
|
||||
doCopy: doCopy,
|
||||
what: "upload",
|
||||
id: response.ID,
|
||||
size: size,
|
||||
parts: parts,
|
||||
sha1s: make([]string, sha1SliceSize),
|
||||
chunkSize: int64(chunkSize),
|
||||
}
|
||||
// unwrap the accounting from the input, we use wrap to put it
|
||||
// back on after the buffering
|
||||
if doCopy {
|
||||
up.what = "copy"
|
||||
up.src = src.(*Object)
|
||||
} else {
|
||||
up.in, up.wrap = accounting.UnWrap(in)
|
||||
in, wrap := accounting.UnWrap(in)
|
||||
up = &largeUpload{
|
||||
f: f,
|
||||
o: o,
|
||||
in: in,
|
||||
wrap: wrap,
|
||||
id: response.ID,
|
||||
size: size,
|
||||
parts: parts,
|
||||
sha1s: make([]string, sha1SliceSize),
|
||||
}
|
||||
return up, nil
|
||||
}
|
||||
@@ -275,41 +256,9 @@ func (up *largeUpload) transferChunk(ctx context.Context, part int64, body []byt
|
||||
return err
|
||||
}
|
||||
|
||||
// Copy a chunk
|
||||
func (up *largeUpload) copyChunk(ctx context.Context, part int64, partSize int64) error {
|
||||
err := up.f.pacer.Call(func() (bool, error) {
|
||||
fs.Debugf(up.o, "Copying chunk %d length %d", part, partSize)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_copy_part",
|
||||
}
|
||||
offset := (part - 1) * up.chunkSize // where we are in the source file
|
||||
var request = api.CopyPartRequest{
|
||||
SourceID: up.src.id,
|
||||
LargeFileID: up.id,
|
||||
PartNumber: part,
|
||||
Range: fmt.Sprintf("bytes=%d-%d", offset, offset+partSize-1),
|
||||
}
|
||||
var response api.UploadPartResponse
|
||||
resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
retry, err := up.f.shouldRetry(ctx, resp, err)
|
||||
if err != nil {
|
||||
fs.Debugf(up.o, "Error copying chunk %d (retry=%v): %v: %#v", part, retry, err, err)
|
||||
}
|
||||
up.sha1s[part-1] = response.SHA1
|
||||
return retry, err
|
||||
})
|
||||
if err != nil {
|
||||
fs.Debugf(up.o, "Error copying chunk %d: %v", part, err)
|
||||
} else {
|
||||
fs.Debugf(up.o, "Done copying chunk %d", part)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// finish closes off the large upload
|
||||
func (up *largeUpload) finish(ctx context.Context) error {
|
||||
fs.Debugf(up.o, "Finishing large file %s with %d parts", up.what, up.parts)
|
||||
fs.Debugf(up.o, "Finishing large file upload with %d parts", up.parts)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_finish_large_file",
|
||||
@@ -331,7 +280,6 @@ func (up *largeUpload) finish(ctx context.Context) error {
|
||||
|
||||
// cancel aborts the large upload
|
||||
func (up *largeUpload) cancel(ctx context.Context) error {
|
||||
fs.Debugf(up.o, "Cancelling large file %s", up.what)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/b2_cancel_large_file",
|
||||
@@ -344,139 +292,139 @@ func (up *largeUpload) cancel(ctx context.Context) error {
|
||||
resp, err := up.f.srv.CallJSON(ctx, &opts, &request, &response)
|
||||
return up.f.shouldRetry(ctx, resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
fs.Errorf(up.o, "Failed to cancel large file %s: %v", up.what, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (up *largeUpload) managedTransferChunk(ctx context.Context, wg *sync.WaitGroup, errs chan error, part int64, buf []byte) {
|
||||
wg.Add(1)
|
||||
go func(part int64, buf []byte) {
|
||||
defer wg.Done()
|
||||
defer up.f.putUploadBlock(buf)
|
||||
err := up.transferChunk(ctx, part, buf)
|
||||
if err != nil {
|
||||
select {
|
||||
case errs <- err:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}(part, buf)
|
||||
}
|
||||
|
||||
func (up *largeUpload) finishOrCancelOnError(ctx context.Context, err error, errs chan error) error {
|
||||
if err == nil {
|
||||
select {
|
||||
case err = <-errs:
|
||||
default:
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
fs.Debugf(up.o, "Cancelling large file upload due to error: %v", err)
|
||||
cancelErr := up.cancel(ctx)
|
||||
if cancelErr != nil {
|
||||
fs.Errorf(up.o, "Failed to cancel large file upload: %v", cancelErr)
|
||||
}
|
||||
return err
|
||||
}
|
||||
return up.finish(ctx)
|
||||
}
|
||||
|
||||
// Stream uploads the chunks from the input, starting with a required initial
|
||||
// chunk. Assumes the file size is unknown and will upload until the input
|
||||
// reaches EOF.
|
||||
//
|
||||
// Note that initialUploadBlock must be returned to f.putBuf()
|
||||
func (up *largeUpload) Stream(ctx context.Context, initialUploadBlock []byte) (err error) {
|
||||
defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
|
||||
fs.Debugf(up.o, "Starting streaming of large file (id %q)", up.id)
|
||||
var (
|
||||
g, gCtx = errgroup.WithContext(ctx)
|
||||
hasMoreParts = true
|
||||
)
|
||||
errs := make(chan error, 1)
|
||||
hasMoreParts := true
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Transfer initial chunk
|
||||
up.size = int64(len(initialUploadBlock))
|
||||
g.Go(func() error {
|
||||
for part := int64(1); hasMoreParts; part++ {
|
||||
// Get a block of memory from the pool and token which limits concurrency.
|
||||
var buf []byte
|
||||
if part == 1 {
|
||||
buf = initialUploadBlock
|
||||
} else {
|
||||
buf = up.f.getBuf(false)
|
||||
}
|
||||
up.managedTransferChunk(ctx, &wg, errs, 1, initialUploadBlock)
|
||||
|
||||
// Fail fast, in case an errgroup managed function returns an error
|
||||
// gCtx is cancelled. There is no point in uploading all the other parts.
|
||||
if gCtx.Err() != nil {
|
||||
up.f.putBuf(buf, false)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read the chunk
|
||||
var n int
|
||||
if part == 1 {
|
||||
n = len(buf)
|
||||
} else {
|
||||
n, err = io.ReadFull(up.in, buf)
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
|
||||
buf = buf[:n]
|
||||
hasMoreParts = false
|
||||
} else if err == io.EOF {
|
||||
fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
|
||||
up.f.putBuf(buf, false)
|
||||
return nil
|
||||
} else if err != nil {
|
||||
// other kinds of errors indicate failure
|
||||
up.f.putBuf(buf, false)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Keep stats up to date
|
||||
up.parts = part
|
||||
up.size += int64(n)
|
||||
if part > maxParts {
|
||||
up.f.putBuf(buf, false)
|
||||
return errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
|
||||
}
|
||||
|
||||
part := part // for the closure
|
||||
g.Go(func() (err error) {
|
||||
defer up.f.putBuf(buf, false)
|
||||
return up.transferChunk(gCtx, part, buf)
|
||||
})
|
||||
outer:
|
||||
for part := int64(2); hasMoreParts; part++ {
|
||||
// Check any errors
|
||||
select {
|
||||
case err = <-errs:
|
||||
break outer
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
})
|
||||
err = g.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
// Get a block of memory
|
||||
buf := up.f.getUploadBlock()
|
||||
|
||||
// Read the chunk
|
||||
var n int
|
||||
n, err = io.ReadFull(up.in, buf)
|
||||
if err == io.ErrUnexpectedEOF {
|
||||
fs.Debugf(up.o, "Read less than a full chunk, making this the last one.")
|
||||
buf = buf[:n]
|
||||
hasMoreParts = false
|
||||
err = nil
|
||||
} else if err == io.EOF {
|
||||
fs.Debugf(up.o, "Could not read any more bytes, previous chunk was the last.")
|
||||
up.f.putUploadBlock(buf)
|
||||
err = nil
|
||||
break outer
|
||||
} else if err != nil {
|
||||
// other kinds of errors indicate failure
|
||||
up.f.putUploadBlock(buf)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Keep stats up to date
|
||||
up.parts = part
|
||||
up.size += int64(n)
|
||||
if part > maxParts {
|
||||
err = errors.Errorf("%q too big (%d bytes so far) makes too many parts %d > %d - increase --b2-chunk-size", up.o, up.size, up.parts, maxParts)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
up.managedTransferChunk(ctx, &wg, errs, part, buf)
|
||||
}
|
||||
wg.Wait()
|
||||
up.sha1s = up.sha1s[:up.parts]
|
||||
return up.finish(ctx)
|
||||
|
||||
return up.finishOrCancelOnError(ctx, err, errs)
|
||||
}
|
||||
|
||||
// Upload uploads the chunks from the input
|
||||
func (up *largeUpload) Upload(ctx context.Context) (err error) {
|
||||
defer atexit.OnError(&err, func() { _ = up.cancel(ctx) })()
|
||||
fs.Debugf(up.o, "Starting %s of large file in %d chunks (id %q)", up.what, up.parts, up.id)
|
||||
var (
|
||||
g, gCtx = errgroup.WithContext(ctx)
|
||||
remaining = up.size
|
||||
)
|
||||
g.Go(func() error {
|
||||
for part := int64(1); part <= up.parts; part++ {
|
||||
// Get a block of memory from the pool and token which limits concurrency.
|
||||
buf := up.f.getBuf(up.doCopy)
|
||||
|
||||
// Fail fast, in case an errgroup managed function returns an error
|
||||
// gCtx is cancelled. There is no point in uploading all the other parts.
|
||||
if gCtx.Err() != nil {
|
||||
up.f.putBuf(buf, up.doCopy)
|
||||
return nil
|
||||
}
|
||||
|
||||
reqSize := remaining
|
||||
if reqSize >= up.chunkSize {
|
||||
reqSize = up.chunkSize
|
||||
}
|
||||
|
||||
if !up.doCopy {
|
||||
// Read the chunk
|
||||
buf = buf[:reqSize]
|
||||
_, err = io.ReadFull(up.in, buf)
|
||||
if err != nil {
|
||||
up.f.putBuf(buf, up.doCopy)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
part := part // for the closure
|
||||
g.Go(func() (err error) {
|
||||
defer up.f.putBuf(buf, up.doCopy)
|
||||
if !up.doCopy {
|
||||
err = up.transferChunk(gCtx, part, buf)
|
||||
} else {
|
||||
err = up.copyChunk(gCtx, part, reqSize)
|
||||
}
|
||||
return err
|
||||
})
|
||||
remaining -= reqSize
|
||||
func (up *largeUpload) Upload(ctx context.Context) error {
|
||||
fs.Debugf(up.o, "Starting upload of large file in %d chunks (id %q)", up.parts, up.id)
|
||||
remaining := up.size
|
||||
errs := make(chan error, 1)
|
||||
var wg sync.WaitGroup
|
||||
var err error
|
||||
outer:
|
||||
for part := int64(1); part <= up.parts; part++ {
|
||||
// Check any errors
|
||||
select {
|
||||
case err = <-errs:
|
||||
break outer
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
})
|
||||
err = g.Wait()
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
reqSize := remaining
|
||||
if reqSize >= int64(up.f.opt.ChunkSize) {
|
||||
reqSize = int64(up.f.opt.ChunkSize)
|
||||
}
|
||||
|
||||
// Get a block of memory
|
||||
buf := up.f.getUploadBlock()[:reqSize]
|
||||
|
||||
// Read the chunk
|
||||
_, err = io.ReadFull(up.in, buf)
|
||||
if err != nil {
|
||||
up.f.putUploadBlock(buf)
|
||||
break outer
|
||||
}
|
||||
|
||||
// Transfer the chunk
|
||||
up.managedTransferChunk(ctx, &wg, errs, part, buf)
|
||||
remaining -= reqSize
|
||||
}
|
||||
return up.finish(ctx)
|
||||
wg.Wait()
|
||||
|
||||
return up.finishOrCancelOnError(ctx, err, errs)
|
||||
}
|
||||
+72
-71
@@ -26,7 +26,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
"github.com/rclone/rclone/lib/jwtutil"
|
||||
|
||||
"github.com/youmark/pkcs8"
|
||||
@@ -113,7 +112,7 @@ func init() {
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "box_config_file",
|
||||
Help: "Box App config.json location\nLeave blank normally." + env.ShellExpandHelp,
|
||||
Help: "Box App config.json location\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "box_sub_type",
|
||||
Default: "user",
|
||||
@@ -154,7 +153,6 @@ func init() {
|
||||
}
|
||||
|
||||
func refreshJWTToken(jsonFile string, boxSubType string, name string, m configmap.Mapper) error {
|
||||
jsonFile = env.ShellExpand(jsonFile)
|
||||
boxConfig, err := getBoxConfig(jsonFile)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
@@ -329,7 +327,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
|
||||
// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -617,6 +615,10 @@ OUTER:
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -657,7 +659,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -713,7 +715,13 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -738,6 +746,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -944,14 +956,64 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, directoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do the move
|
||||
_, err = f.move(ctx, "/folders/", srcID, dstLeaf, dstDirectoryID)
|
||||
_, err = f.move(ctx, "/folders/", srcID, leaf, directoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -960,7 +1022,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
|
||||
id, err := f.dirCache.FindDir(ctx, remote, false)
|
||||
var opts rest.Opts
|
||||
if err == nil {
|
||||
@@ -999,66 +1061,6 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
return info.SharedLink.URL, err
|
||||
}
|
||||
|
||||
// deletePermanently permenently deletes a trashed file
|
||||
func (f *Fs) deletePermanently(ctx context.Context, itemType, id string) error {
|
||||
opts := rest.Opts{
|
||||
Method: "DELETE",
|
||||
NoResponse: true,
|
||||
}
|
||||
if itemType == api.ItemTypeFile {
|
||||
opts.Path = "/files/" + id + "/trash"
|
||||
} else {
|
||||
opts.Path = "/folders/" + id + "/trash"
|
||||
}
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.Call(ctx, &opts)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
}
|
||||
|
||||
// CleanUp empties the trash
|
||||
func (f *Fs) CleanUp(ctx context.Context) (err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: "/folders/trash/items",
|
||||
Parameters: url.Values{
|
||||
"fields": []string{"type", "id"},
|
||||
},
|
||||
}
|
||||
opts.Parameters.Set("limit", strconv.Itoa(listChunks))
|
||||
offset := 0
|
||||
for {
|
||||
opts.Parameters.Set("offset", strconv.Itoa(offset))
|
||||
|
||||
var result api.FolderItems
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't list trash")
|
||||
}
|
||||
for i := range result.Entries {
|
||||
item := &result.Entries[i]
|
||||
if item.Type == api.ItemTypeFolder || item.Type == api.ItemTypeFile {
|
||||
err := f.deletePermanently(ctx, item.Type, item.ID)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "failed to delete file")
|
||||
}
|
||||
} else {
|
||||
fs.Debugf(f, "Ignoring %q - unknown type %q", item.Name, item.Type)
|
||||
continue
|
||||
}
|
||||
}
|
||||
offset += result.Limit
|
||||
if offset >= result.TotalCount {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DirCacheFlush resets the directory cache - used in testing as an
|
||||
// optional interface
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
@@ -1266,7 +1268,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
remote := o.Remote()
|
||||
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1301,7 +1303,6 @@ var (
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.CleanUpper = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
)
|
||||
@@ -19,7 +19,6 @@ import (
|
||||
"github.com/rclone/rclone/backend/box/api"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/accounting"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/rest"
|
||||
)
|
||||
|
||||
@@ -183,13 +182,15 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, leaf, direct
|
||||
fs.Debugf(o, "Multipart upload session started for %d parts of size %v", session.TotalParts, fs.SizeSuffix(chunkSize))
|
||||
|
||||
// Cancel the session if something went wrong
|
||||
defer atexit.OnError(&err, func() {
|
||||
fs.Debugf(o, "Cancelling multipart upload: %v", err)
|
||||
cancelErr := o.abortUpload(ctx, session.ID)
|
||||
if cancelErr != nil {
|
||||
fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
|
||||
defer func() {
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Cancelling multipart upload: %v", err)
|
||||
cancelErr := o.abortUpload(ctx, session.ID)
|
||||
if cancelErr != nil {
|
||||
fs.Logf(o, "Failed to cancel multipart upload: %v", err)
|
||||
}
|
||||
}
|
||||
})()
|
||||
}()
|
||||
|
||||
// unwrap the accounting from the input, we use wrap to put it
|
||||
// back on after the buffering
|
||||
|
||||
Vendored
-14
@@ -1829,19 +1829,6 @@ func (f *Fs) isRootInPath(p string) bool {
|
||||
return strings.HasPrefix(p, f.Root()+"/")
|
||||
}
|
||||
|
||||
// MergeDirs merges the contents of all the directories passed
|
||||
// in into the first one and rmdirs the other directories.
|
||||
func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
do := f.Fs.Features().MergeDirs
|
||||
if do == nil {
|
||||
return errors.New("MergeDirs not supported")
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
_ = f.cache.RemoveDir(dir.Remote())
|
||||
}
|
||||
return do(ctx, dirs)
|
||||
}
|
||||
|
||||
// DirCacheFlush flushes the dir cache
|
||||
func (f *Fs) DirCacheFlush() {
|
||||
_ = f.cache.RemoveDir("")
|
||||
@@ -1939,5 +1926,4 @@ var (
|
||||
_ fs.UserInfoer = (*Fs)(nil)
|
||||
_ fs.Disconnecter = (*Fs)(nil)
|
||||
_ fs.Commander = (*Fs)(nil)
|
||||
_ fs.MergeDirser = (*Fs)(nil)
|
||||
)
|
||||
Vendored
+1
-1
@@ -18,7 +18,7 @@ func TestIntegration(t *testing.T) {
|
||||
fstests.Run(t, &fstests.Opt{
|
||||
RemoteName: "TestCache:",
|
||||
NilObject: (*cache.Object)(nil),
|
||||
UnimplementableFsMethods: []string{"PublicLink", "OpenWriterAt"},
|
||||
UnimplementableFsMethods: []string{"PublicLink", "MergeDirs", "OpenWriterAt"},
|
||||
UnimplementableObjectMethods: []string{"MimeType", "ID", "GetTier", "SetTier"},
|
||||
SkipInvalidUTF8: true, // invalid UTF-8 confuses the cache
|
||||
})
|
||||
|
||||
+3
-19
@@ -72,20 +72,6 @@ NB If filename_encryption is "off" then this option will do nothing.`,
|
||||
Name: "password2",
|
||||
Help: "Password or pass phrase for salt. Optional but recommended.\nShould be different to the previous password.",
|
||||
IsPassword: true,
|
||||
}, {
|
||||
Name: "server_side_across_configs",
|
||||
Default: false,
|
||||
Help: `Allow server side operations (eg copy) to work across different crypt configs.
|
||||
|
||||
Normally this option is not what you want, but if you have two crypts
|
||||
pointing to the same backend you can use it.
|
||||
|
||||
This can be used, for example, to change file name encryption type
|
||||
without re-uploading all the data. Just make two crypt backends
|
||||
pointing to two different directories with the single changed
|
||||
parameter and use rclone move to move the files between the crypt
|
||||
remotes.`,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "show_mapping",
|
||||
Help: `For all files listed show how the names encrypt.
|
||||
@@ -195,7 +181,6 @@ func NewFs(name, rpath string, m configmap.Mapper) (fs.Fs, error) {
|
||||
CanHaveEmptyDirectories: true,
|
||||
SetTier: true,
|
||||
GetTier: true,
|
||||
ServerSideAcrossConfigs: opt.ServerSideAcrossConfigs,
|
||||
}).Fill(f).Mask(wrappedFs).WrapsFs(f, wrappedFs)
|
||||
|
||||
return f, err
|
||||
@@ -208,7 +193,6 @@ type Options struct {
|
||||
DirectoryNameEncryption bool `config:"directory_name_encryption"`
|
||||
Password string `config:"password"`
|
||||
Password2 string `config:"password2"`
|
||||
ServerSideAcrossConfigs bool `config:"server_side_across_configs"`
|
||||
ShowMapping bool `config:"show_mapping"`
|
||||
}
|
||||
|
||||
@@ -672,7 +656,7 @@ func (f *Fs) DirCacheFlush() {
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
|
||||
do := f.Fs.Features().PublicLink
|
||||
if do == nil {
|
||||
return "", errors.New("PublicLink not supported")
|
||||
@@ -680,9 +664,9 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
o, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
// assume it is a directory
|
||||
return do(ctx, f.cipher.EncryptDirName(remote), expire, unlink)
|
||||
return do(ctx, f.cipher.EncryptDirName(remote))
|
||||
}
|
||||
return do(ctx, o.(*Object).Object.Remote(), expire, unlink)
|
||||
return do(ctx, o.(*Object).Object.Remote())
|
||||
}
|
||||
|
||||
// ChangeNotify calls the passed function with a path
|
||||
|
||||
+87
-28
@@ -18,6 +18,7 @@ import (
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -40,7 +41,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/lib/dircache"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"github.com/rclone/rclone/lib/readers"
|
||||
@@ -57,7 +57,6 @@ const (
|
||||
rcloneEncryptedClientSecret = "eX8GpZTVx3vxMWVkuuBdDWmAUE6rGhTwVrvG9GhllYccSdj2-mvHVg"
|
||||
driveFolderType = "application/vnd.google-apps.folder"
|
||||
shortcutMimeType = "application/vnd.google-apps.shortcut"
|
||||
shortcutMimeTypeDangling = "application/vnd.google-apps.shortcut.dangling" // synthetic mime type for internal use
|
||||
timeFormatIn = time.RFC3339
|
||||
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
defaultMinSleep = fs.Duration(100 * time.Millisecond)
|
||||
@@ -230,7 +229,7 @@ in with the ID of the root folder.
|
||||
`,
|
||||
}, {
|
||||
Name: "service_account_file",
|
||||
Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
|
||||
Help: "Service Account Credentials JSON file path \nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
}, {
|
||||
Name: "service_account_credentials",
|
||||
Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
@@ -1009,7 +1008,7 @@ func createOAuthClient(opt *Options, name string, m configmap.Mapper) (*http.Cli
|
||||
|
||||
// try loading service account credentials from env variable, then from a file
|
||||
if len(opt.ServiceAccountCredentials) == 0 && opt.ServiceAccountFile != "" {
|
||||
loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
|
||||
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error opening service account credentials file")
|
||||
}
|
||||
@@ -1128,7 +1127,7 @@ func NewFs(name, path string, m configmap.Mapper) (fs.Fs, error) {
|
||||
// using impersonate which they shouldn't have done. It is possible
|
||||
// someone is using impersonate and root_folder_id in which case this
|
||||
// breaks their workflow. There isn't an easy way around that.
|
||||
if opt.RootFolderID != "" && opt.RootFolderID != "appDataFolder" && opt.Impersonate != "" {
|
||||
if opt.RootFolderID != "" && opt.Impersonate != "" {
|
||||
fs.Logf(f, "Ignoring cached root_folder_id when using --drive-impersonate")
|
||||
opt.RootFolderID = ""
|
||||
}
|
||||
@@ -1357,10 +1356,6 @@ func (f *Fs) newObjectWithExportInfo(
|
||||
// and not from a listing. This is unlikely.
|
||||
fs.Debugf(remote, "Ignoring shortcut as skip shortcuts is set")
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
case info.MimeType == shortcutMimeTypeDangling:
|
||||
// Pretend a dangling shortcut is a regular object
|
||||
// It will error if used, but appear in listings so it can be deleted
|
||||
return f.newRegularObject(remote, info), nil
|
||||
case info.Md5Checksum != "" || info.Size > 0:
|
||||
// If item has MD5 sum or a length it is a file stored on drive
|
||||
return f.newRegularObject(remote, info), nil
|
||||
@@ -1594,6 +1589,10 @@ func (f *Fs) findImportFormat(mimeType string) string {
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1815,6 +1814,10 @@ func (f *Fs) listRRunner(ctx context.Context, wg *sync.WaitGroup, in chan listRE
|
||||
// Don't implement this unless you have a more efficient way
|
||||
// of listing recursively that doing a directory traversal.
|
||||
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1985,12 +1988,6 @@ func (f *Fs) resolveShortcut(item *drive.File) (newItem *drive.File, err error)
|
||||
}
|
||||
newItem, err = f.getFile(item.ShortcutDetails.TargetId, f.fileFields)
|
||||
if err != nil {
|
||||
if gerr, ok := errors.Cause(err).(*googleapi.Error); ok && gerr.Code == 404 {
|
||||
// 404 means dangling shortcut, so just return the shortcut with the mime type mangled
|
||||
fs.Logf(nil, "Dangling shortcut %q detected", item.Name)
|
||||
item.MimeType = shortcutMimeTypeDangling
|
||||
return item, nil
|
||||
}
|
||||
return nil, errors.Wrap(err, "failed to resolve shortcut")
|
||||
}
|
||||
// make sure we use the Name, Parents and Trashed from the original item
|
||||
@@ -2029,7 +2026,7 @@ func (f *Fs) itemToDirEntry(remote string, item *drive.File) (entry fs.DirEntry,
|
||||
//
|
||||
// Used to create new objects
|
||||
func (f *Fs) createFileInfo(ctx context.Context, remote string, modTime time.Time) (*drive.File, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -2192,7 +2189,13 @@ func (f *Fs) MergeDirs(ctx context.Context, dirs []fs.Directory) error {
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2365,11 +2368,11 @@ func (f *Fs) Purge(ctx context.Context) error {
|
||||
if f.opt.TrashedOnly {
|
||||
return errors.New("Can't purge with --drive-trashed-only. Use delete if you want to selectively delete files")
|
||||
}
|
||||
rootID, err := f.dirCache.RootID(ctx, false)
|
||||
err := f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = f.delete(ctx, shortcutID(rootID), f.opt.UseTrash)
|
||||
err = f.delete(ctx, shortcutID(f.dirCache.RootID()), f.opt.UseTrash)
|
||||
f.dirCache.ResetRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -2504,7 +2507,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
id, err := f.dirCache.FindDir(ctx, remote, false)
|
||||
if err == nil {
|
||||
fs.Debugf(f, "attempting to share directory '%s'", remote)
|
||||
@@ -2553,19 +2556,77 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = srcLeaf
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, dstDirectoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dstDirectoryID = actualID(dstDirectoryID)
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src parent
|
||||
var srcDirectoryID string
|
||||
if srcRemote == "" {
|
||||
srcDirectoryID, err = srcFs.dirCache.RootParentID()
|
||||
} else {
|
||||
_, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, srcRemote, false)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
srcDirectoryID = actualID(srcDirectoryID)
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Do the move
|
||||
patch := drive.File{
|
||||
Name: dstLeaf,
|
||||
Name: leaf,
|
||||
}
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.svc.Files.Update(shortcutID(srcID), &patch).
|
||||
@@ -2817,10 +2878,11 @@ func (f *Fs) makeShortcut(ctx context.Context, srcPath string, dstFs *Fs, dstPat
|
||||
isDir := false
|
||||
if srcPath == "" {
|
||||
// source is root directory
|
||||
srcID, err = f.dirCache.RootID(ctx, false)
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
srcID = f.dirCache.RootID()
|
||||
isDir = true
|
||||
} else if srcObj, err := srcFs.NewObject(ctx, srcPath); err != nil {
|
||||
if err != fs.ErrorNotAFile {
|
||||
@@ -3049,7 +3111,7 @@ func (f *Fs) getRemoteInfo(ctx context.Context, remote string) (info *drive.File
|
||||
// getRemoteInfoWithExport returns a drive.File and the export settings for the remote
|
||||
func (f *Fs) getRemoteInfoWithExport(ctx context.Context, remote string) (
|
||||
info *drive.File, extension, exportName, exportMimeType string, isDocument bool, err error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, "", "", "", false, fs.ErrorObjectNotFound
|
||||
@@ -3234,9 +3296,6 @@ func (o *baseObject) open(ctx context.Context, url string, options ...fs.OpenOpt
|
||||
|
||||
// Open an object for read
|
||||
func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.ReadCloser, err error) {
|
||||
if o.mimeType == shortcutMimeTypeDangling {
|
||||
return nil, errors.New("can't read dangling shortcut")
|
||||
}
|
||||
if o.v2Download {
|
||||
var v2File *drive_v2.File
|
||||
err = o.fs.pacer.Call(func() (bool, error) {
|
||||
|
||||
@@ -274,7 +274,7 @@ func (f *Fs) InternalTestShortcuts(t *testing.T) {
|
||||
const (
|
||||
// from fstest/fstests/fstests.go
|
||||
existingDir = "hello? sausage"
|
||||
existingFile = `hello? sausage/êé/Hello, 世界/ " ' @ < > & ? + ≠/z.txt`
|
||||
existingFile = "file name.txt"
|
||||
existingSubDir = "êé"
|
||||
)
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -782,17 +782,11 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
absPath := f.opt.Enc.FromStandardPath(path.Join(f.slashRoot, remote))
|
||||
fs.Debugf(f, "attempting to share '%s' (absolute path: %s)", remote, absPath)
|
||||
createArg := sharing.CreateSharedLinkWithSettingsArg{
|
||||
Path: absPath,
|
||||
// FIXME this gives settings_error/not_authorized/.. errors
|
||||
// and the expires setting isn't in the documentation so remove
|
||||
// for now.
|
||||
// Settings: &sharing.SharedLinkSettings{
|
||||
// Expires: time.Now().Add(time.Duration(expire)).UTC().Round(time.Second),
|
||||
// },
|
||||
}
|
||||
var linkRes sharing.IsSharedLinkMetadata
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
|
||||
@@ -148,6 +148,11 @@ func (f *Fs) listFolders(ctx context.Context, directoryID int) (foldersList *Fol
|
||||
}
|
||||
|
||||
func (f *Fs) listDir(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -264,7 +264,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// NewObject finds the Object at remote. If it can't be found
|
||||
// it returns the error ErrorObjectNotFound.
|
||||
func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -334,7 +334,7 @@ func (f *Fs) putUnchecked(ctx context.Context, in io.Reader, remote string, size
|
||||
return nil, err
|
||||
}
|
||||
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -389,7 +389,13 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
//
|
||||
// Shouldn't return an error if it already exists
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -397,6 +403,11 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
//
|
||||
// Return an error if it doesn't exist or isn't empty
|
||||
func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
err := f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+3
-66
@@ -8,8 +8,6 @@ import (
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -52,19 +50,8 @@ func init() {
|
||||
IsPassword: true,
|
||||
Required: true,
|
||||
}, {
|
||||
Name: "tls",
|
||||
Help: `Use FTPS over TLS (Implicit)
|
||||
When using implicit FTP over TLS the client will connect using TLS
|
||||
right from the start, which in turn breaks the compatibility with
|
||||
non-TLS-aware servers. This is usually served over port 990 rather
|
||||
than port 21. Cannot be used in combination with explicit FTP.`,
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "explicit_tls",
|
||||
Help: `Use FTP over TLS (Explicit)
|
||||
When using explicit FTP over TLS the client explicitly request
|
||||
security from the server in order to upgrade a plain text connection
|
||||
to an encrypted one. Cannot be used in combination with implicit FTP.`,
|
||||
Name: "tls",
|
||||
Help: "Use FTP over TLS (Implicit)",
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "concurrency",
|
||||
@@ -103,7 +90,6 @@ type Options struct {
|
||||
Pass string `config:"pass"`
|
||||
Port string `config:"port"`
|
||||
TLS bool `config:"tls"`
|
||||
ExplicitTLS bool `config:"explicit_tls"`
|
||||
Concurrency int `config:"concurrency"`
|
||||
SkipVerifyTLSCert bool `config:"no_check_certificate"`
|
||||
DisableEPSV bool `config:"disable_epsv"`
|
||||
@@ -162,69 +148,20 @@ func (f *Fs) Features() *fs.Features {
|
||||
return f.features
|
||||
}
|
||||
|
||||
// Enable debugging output
|
||||
type debugLog struct {
|
||||
mu sync.Mutex
|
||||
auth bool
|
||||
}
|
||||
|
||||
// Write writes len(p) bytes from p to the underlying data stream. It returns
|
||||
// the number of bytes written from p (0 <= n <= len(p)) and any error
|
||||
// encountered that caused the write to stop early. Write must return a non-nil
|
||||
// error if it returns n < len(p). Write must not modify the slice data, even
|
||||
// temporarily.
|
||||
//
|
||||
// Implementations must not retain p.
|
||||
//
|
||||
// This writes debug info to the log
|
||||
func (dl *debugLog) Write(p []byte) (n int, err error) {
|
||||
dl.mu.Lock()
|
||||
defer dl.mu.Unlock()
|
||||
_, file, _, ok := runtime.Caller(1)
|
||||
direction := "FTP Rx"
|
||||
if ok && strings.Contains(file, "multi") {
|
||||
direction = "FTP Tx"
|
||||
}
|
||||
lines := strings.Split(string(p), "\r\n")
|
||||
if lines[len(lines)-1] == "" {
|
||||
lines = lines[:len(lines)-1]
|
||||
}
|
||||
for _, line := range lines {
|
||||
if !dl.auth && strings.HasPrefix(line, "PASS") {
|
||||
fs.Debugf(direction, "PASS *****")
|
||||
continue
|
||||
}
|
||||
fs.Debugf(direction, "%q", line)
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
// Open a new connection to the FTP server.
|
||||
func (f *Fs) ftpConnection() (*ftp.ServerConn, error) {
|
||||
fs.Debugf(f, "Connecting to FTP server")
|
||||
ftpConfig := []ftp.DialOption{ftp.DialWithTimeout(fs.Config.ConnectTimeout)}
|
||||
if f.opt.TLS && f.opt.ExplicitTLS {
|
||||
fs.Errorf(f, "Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
|
||||
return nil, errors.New("Implicit TLS and explicit TLS are mutually incompatible. Please revise your config")
|
||||
} else if f.opt.TLS {
|
||||
if f.opt.TLS {
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: f.opt.Host,
|
||||
InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
|
||||
}
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithTLS(tlsConfig))
|
||||
} else if f.opt.ExplicitTLS {
|
||||
tlsConfig := &tls.Config{
|
||||
ServerName: f.opt.Host,
|
||||
InsecureSkipVerify: f.opt.SkipVerifyTLSCert,
|
||||
}
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithExplicitTLS(tlsConfig))
|
||||
}
|
||||
if f.opt.DisableEPSV {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDisabledEPSV(true))
|
||||
}
|
||||
if fs.Config.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 {
|
||||
ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: fs.Config.Dump&fs.DumpAuth != 0}))
|
||||
}
|
||||
c, err := ftp.Dial(f.dialAddr, ftpConfig...)
|
||||
if err != nil {
|
||||
fs.Errorf(f, "Error while Dialing %s: %s", f.dialAddr, err)
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -37,7 +38,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/lib/bucket"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/env"
|
||||
"github.com/rclone/rclone/lib/oauthutil"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
"golang.org/x/oauth2"
|
||||
@@ -79,8 +79,7 @@ func init() {
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
saFile, _ := m.Get("service_account_file")
|
||||
saCreds, _ := m.Get("service_account_credentials")
|
||||
anonymous, _ := m.Get("anonymous")
|
||||
if saFile != "" || saCreds != "" || anonymous == "true" {
|
||||
if saFile != "" || saCreds != "" {
|
||||
return
|
||||
}
|
||||
err := oauthutil.Config("google cloud storage", name, m, storageConfig, nil)
|
||||
@@ -99,15 +98,11 @@ func init() {
|
||||
Help: "Project number.\nOptional - needed only for list/create/delete buckets - see your developer console.",
|
||||
}, {
|
||||
Name: "service_account_file",
|
||||
Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login." + env.ShellExpandHelp,
|
||||
Help: "Service Account Credentials JSON file path\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
}, {
|
||||
Name: "service_account_credentials",
|
||||
Help: "Service Account Credentials JSON blob\nLeave blank normally.\nNeeded only if you want use SA instead of interactive login.",
|
||||
Hide: fs.OptionHideBoth,
|
||||
}, {
|
||||
Name: "anonymous",
|
||||
Help: "Access public buckets and objects without credentials\nSet to 'true' if you just want to download files and don't configure credentials.",
|
||||
Default: false,
|
||||
}, {
|
||||
Name: "object_acl",
|
||||
Help: "Access Control List for new objects.",
|
||||
@@ -270,7 +265,6 @@ type Options struct {
|
||||
ProjectNumber string `config:"project_number"`
|
||||
ServiceAccountFile string `config:"service_account_file"`
|
||||
ServiceAccountCredentials string `config:"service_account_credentials"`
|
||||
Anonymous bool `config:"anonymous"`
|
||||
ObjectACL string `config:"object_acl"`
|
||||
BucketACL string `config:"bucket_acl"`
|
||||
BucketPolicyOnly bool `config:"bucket_policy_only"`
|
||||
@@ -411,15 +405,13 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
|
||||
// try loading service account credentials from env variable, then from a file
|
||||
if opt.ServiceAccountCredentials == "" && opt.ServiceAccountFile != "" {
|
||||
loadedCreds, err := ioutil.ReadFile(env.ShellExpand(opt.ServiceAccountFile))
|
||||
loadedCreds, err := ioutil.ReadFile(os.ExpandEnv(opt.ServiceAccountFile))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "error opening service account credentials file")
|
||||
}
|
||||
opt.ServiceAccountCredentials = string(loadedCreds)
|
||||
}
|
||||
if opt.Anonymous {
|
||||
oAuthClient = &http.Client{}
|
||||
} else if opt.ServiceAccountCredentials != "" {
|
||||
if opt.ServiceAccountCredentials != "" {
|
||||
oAuthClient, err = getServiceAccountClient([]byte(opt.ServiceAccountCredentials))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed configuring Google Cloud Storage Service Account")
|
||||
|
||||
@@ -30,7 +30,7 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
albums.add(a1)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"one": []*api.Album{a1},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"1": a1,
|
||||
@@ -39,7 +39,7 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
"one": a1,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one"},
|
||||
"": []string{"one"},
|
||||
}, albums.path)
|
||||
|
||||
a2 := &api.Album{
|
||||
@@ -49,8 +49,8 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
albums.add(a2)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"1": a1,
|
||||
@@ -61,7 +61,7 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
"two": a2,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two"},
|
||||
"": []string{"one", "two"},
|
||||
}, albums.path)
|
||||
|
||||
// Add a duplicate
|
||||
@@ -72,8 +72,8 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
albums.add(a2a)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"1": a1,
|
||||
@@ -86,7 +86,7 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
"two {2a}": a2a,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two {2}", "two {2a}"},
|
||||
"": []string{"one", "two {2}", "two {2a}"},
|
||||
}, albums.path)
|
||||
|
||||
// Add a sub directory
|
||||
@@ -97,9 +97,9 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
albums.add(a1sub)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"1": a1,
|
||||
@@ -114,8 +114,8 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
"two {2a}": a2a,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two {2}", "two {2a}"},
|
||||
"one": {"sub"},
|
||||
"": []string{"one", "two {2}", "two {2a}"},
|
||||
"one": []string{"sub"},
|
||||
}, albums.path)
|
||||
|
||||
// Add a weird path
|
||||
@@ -126,10 +126,10 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
albums.add(a0)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"{0}": {a0},
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"{0}": []*api.Album{a0},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"0": a0,
|
||||
@@ -146,8 +146,8 @@ func TestAlbumsAdd(t *testing.T) {
|
||||
"two {2a}": a2a,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two {2}", "two {2a}", "{0}"},
|
||||
"one": {"sub"},
|
||||
"": []string{"one", "two {2}", "two {2a}", "{0}"},
|
||||
"one": []string{"sub"},
|
||||
}, albums.path)
|
||||
}
|
||||
|
||||
@@ -181,9 +181,9 @@ func TestAlbumsDel(t *testing.T) {
|
||||
albums.add(a1sub)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"1": a1,
|
||||
@@ -198,16 +198,16 @@ func TestAlbumsDel(t *testing.T) {
|
||||
"two {2a}": a2a,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two {2}", "two {2a}"},
|
||||
"one": {"sub"},
|
||||
"": []string{"one", "two {2}", "two {2a}"},
|
||||
"one": []string{"sub"},
|
||||
}, albums.path)
|
||||
|
||||
albums.del(a1)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"2": a2,
|
||||
@@ -220,16 +220,16 @@ func TestAlbumsDel(t *testing.T) {
|
||||
"two {2a}": a2a,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two {2}", "two {2a}"},
|
||||
"one": {"sub"},
|
||||
"": []string{"one", "two {2}", "two {2a}"},
|
||||
"one": []string{"sub"},
|
||||
}, albums.path)
|
||||
|
||||
albums.del(a2)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"2a": a2a,
|
||||
@@ -240,16 +240,16 @@ func TestAlbumsDel(t *testing.T) {
|
||||
"two {2a}": a2a,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one", "two {2a}"},
|
||||
"one": {"sub"},
|
||||
"": []string{"one", "two {2a}"},
|
||||
"one": []string{"sub"},
|
||||
}, albums.path)
|
||||
|
||||
albums.del(a2a)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{
|
||||
"1sub": a1sub,
|
||||
@@ -258,16 +258,16 @@ func TestAlbumsDel(t *testing.T) {
|
||||
"one/sub": a1sub,
|
||||
}, albums.byTitle)
|
||||
assert.Equal(t, map[string][]string{
|
||||
"": {"one"},
|
||||
"one": {"sub"},
|
||||
"": []string{"one"},
|
||||
"one": []string{"sub"},
|
||||
}, albums.path)
|
||||
|
||||
albums.del(a1sub)
|
||||
|
||||
assert.Equal(t, map[string][]*api.Album{
|
||||
"one": {a1},
|
||||
"two": {a2, a2a},
|
||||
"one/sub": {a1sub},
|
||||
"one": []*api.Album{a1},
|
||||
"two": []*api.Album{a2, a2a},
|
||||
"one/sub": []*api.Album{a1sub},
|
||||
}, albums.dupes)
|
||||
assert.Equal(t, map[string]*api.Album{}, albums.byID)
|
||||
assert.Equal(t, map[string]*api.Album{}, albums.byTitle)
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -27,7 +26,6 @@ import (
|
||||
"github.com/rclone/rclone/fs/config"
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/configstruct"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
@@ -41,33 +39,26 @@ import (
|
||||
|
||||
// Globals
|
||||
const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
defaultDevice = "Jotta"
|
||||
defaultMountpoint = "Archive"
|
||||
rootURL = "https://jfs.jottacloud.com/jfs/"
|
||||
apiURL = "https://api.jottacloud.com/"
|
||||
baseURL = "https://www.jottacloud.com/"
|
||||
defaultTokenURL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
|
||||
cachePrefix = "rclone-jcmd5-"
|
||||
configDevice = "device"
|
||||
configMountpoint = "mountpoint"
|
||||
configTokenURL = "tokenURL"
|
||||
configClientID = "client_id"
|
||||
configClientSecret = "client_secret"
|
||||
configVersion = 1
|
||||
|
||||
v1tokenURL = "https://api.jottacloud.com/auth/v1/token"
|
||||
v1registerURL = "https://api.jottacloud.com/auth/v1/register"
|
||||
v1ClientID = "nibfk8biu12ju7hpqomr8b1e40"
|
||||
v1EncryptedClientSecret = "Vp8eAv7eVElMnQwN-kgU9cbhgApNDaMqWdlDi5qFydlQoji4JBxrGMF2"
|
||||
v1configVersion = 0
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
defaultDevice = "Jotta"
|
||||
defaultMountpoint = "Archive"
|
||||
rootURL = "https://www.jottacloud.com/jfs/"
|
||||
apiURL = "https://api.jottacloud.com/"
|
||||
baseURL = "https://www.jottacloud.com/"
|
||||
defaultTokenURL = "https://id.jottacloud.com/auth/realms/jottacloud/protocol/openid-connect/token"
|
||||
cachePrefix = "rclone-jcmd5-"
|
||||
configDevice = "device"
|
||||
configMountpoint = "mountpoint"
|
||||
configTokenURL = "tokenURL"
|
||||
configVersion = 1
|
||||
)
|
||||
|
||||
var (
|
||||
// Description of how to auth for this app for a personal account
|
||||
oauthConfig = &oauth2.Config{
|
||||
ClientID: "jottacli",
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: defaultTokenURL,
|
||||
TokenURL: defaultTokenURL,
|
||||
@@ -92,7 +83,7 @@ func init() {
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to parse config version - corrupted config")
|
||||
}
|
||||
refresh = (ver != configVersion) && (ver != v1configVersion)
|
||||
refresh = ver != configVersion
|
||||
}
|
||||
|
||||
if refresh {
|
||||
@@ -107,12 +98,42 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Printf("Use legacy authentification?.\nThis is only required for certain whitelabel versions of Jottacloud and not recommended for normal users.\n")
|
||||
if config.Confirm(false) {
|
||||
v1config(ctx, name, m)
|
||||
} else {
|
||||
v2config(ctx, name, m)
|
||||
clientConfig := *fs.Config
|
||||
clientConfig.UserAgent = "JottaCli 0.6.18626 windows-amd64"
|
||||
srv := rest.NewClient(fshttp.NewClient(&clientConfig))
|
||||
|
||||
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
|
||||
fmt.Printf("Login Token> ")
|
||||
loginToken := config.ReadLine()
|
||||
|
||||
token, err := doAuth(ctx, srv, loginToken, m)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get oauth token: %s", err)
|
||||
}
|
||||
err = oauthutil.PutToken(name, m, &token, true)
|
||||
if err != nil {
|
||||
log.Fatalf("Error while saving token: %s", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
|
||||
if config.Confirm(false) {
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
}
|
||||
|
||||
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
|
||||
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
|
||||
|
||||
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to setup mountpoint: %s", err)
|
||||
}
|
||||
m.Set(configDevice, device)
|
||||
m.Set(configMountpoint, mountpoint)
|
||||
}
|
||||
|
||||
m.Set("configVersion", strconv.Itoa(configVersion))
|
||||
},
|
||||
Options: []fs.Option{{
|
||||
Name: "md5_memory_limit",
|
||||
@@ -129,6 +150,11 @@ func init() {
|
||||
Help: "Delete files permanently rather than putting them into the trash.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "unlink",
|
||||
Help: "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "upload_resume_limit",
|
||||
Help: "Files bigger than this can be resumed if the upload fail's.",
|
||||
@@ -155,6 +181,7 @@ type Options struct {
|
||||
MD5MemoryThreshold fs.SizeSuffix `config:"md5_memory_limit"`
|
||||
TrashedOnly bool `config:"trashed_only"`
|
||||
HardDelete bool `config:"hard_delete"`
|
||||
Unlink bool `config:"unlink"`
|
||||
UploadThreshold fs.SizeSuffix `config:"upload_resume_limit"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
@@ -230,181 +257,8 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
return fserrors.ShouldRetry(err) || fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
|
||||
}
|
||||
|
||||
// v1config configure a jottacloud backend using legacy authentification
|
||||
func v1config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
srv := rest.NewClient(fshttp.NewClient(fs.Config))
|
||||
|
||||
fmt.Printf("\nDo you want to create a machine specific API key?\n\nRclone has it's own Jottacloud API KEY which works fine as long as one only uses rclone on a single machine. When you want to use rclone with this account on more than one machine it's recommended to create a machine specific API key. These keys can NOT be shared between machines.\n\n")
|
||||
if config.Confirm(false) {
|
||||
deviceRegistration, err := registerDevice(ctx, srv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to register device: %v", err)
|
||||
}
|
||||
|
||||
m.Set(configClientID, deviceRegistration.ClientID)
|
||||
m.Set(configClientSecret, obscure.MustObscure(deviceRegistration.ClientSecret))
|
||||
fs.Debugf(nil, "Got clientID '%s' and clientSecret '%s'", deviceRegistration.ClientID, deviceRegistration.ClientSecret)
|
||||
}
|
||||
|
||||
clientID, ok := m.Get(configClientID)
|
||||
if !ok {
|
||||
clientID = v1ClientID
|
||||
}
|
||||
clientSecret, ok := m.Get(configClientSecret)
|
||||
if !ok {
|
||||
clientSecret = v1EncryptedClientSecret
|
||||
}
|
||||
oauthConfig.ClientID = clientID
|
||||
oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
|
||||
|
||||
oauthConfig.Endpoint.AuthURL = v1tokenURL
|
||||
oauthConfig.Endpoint.TokenURL = v1tokenURL
|
||||
|
||||
fmt.Printf("Username> ")
|
||||
username := config.ReadLine()
|
||||
password := config.GetPassword("Your Jottacloud password is only required during setup and will not be stored.")
|
||||
|
||||
token, err := doAuthV1(ctx, srv, username, password)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get oauth token: %s", err)
|
||||
}
|
||||
err = oauthutil.PutToken(name, m, &token, true)
|
||||
if err != nil {
|
||||
log.Fatalf("Error while saving token: %s", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
|
||||
if config.Confirm(false) {
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
}
|
||||
|
||||
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
|
||||
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
|
||||
|
||||
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to setup mountpoint: %s", err)
|
||||
}
|
||||
m.Set(configDevice, device)
|
||||
m.Set(configMountpoint, mountpoint)
|
||||
}
|
||||
|
||||
m.Set("configVersion", strconv.Itoa(v1configVersion))
|
||||
}
|
||||
|
||||
// registerDevice register a new device for use with the jottacloud API
|
||||
func registerDevice(ctx context.Context, srv *rest.Client) (reg *api.DeviceRegistrationResponse, err error) {
|
||||
// random generator to generate random device names
|
||||
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
randonDeviceNamePartLength := 21
|
||||
randomDeviceNamePart := make([]byte, randonDeviceNamePartLength)
|
||||
charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
for i := range randomDeviceNamePart {
|
||||
randomDeviceNamePart[i] = charset[seededRand.Intn(len(charset))]
|
||||
}
|
||||
randomDeviceName := "rclone-" + string(randomDeviceNamePart)
|
||||
fs.Debugf(nil, "Trying to register device '%s'", randomDeviceName)
|
||||
|
||||
values := url.Values{}
|
||||
values.Set("device_id", randomDeviceName)
|
||||
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
RootURL: v1registerURL,
|
||||
ContentType: "application/x-www-form-urlencoded",
|
||||
ExtraHeaders: map[string]string{"Authorization": "Bearer c2xrZmpoYWRsZmFramhkc2xma2phaHNkbGZramhhc2xkZmtqaGFzZGxrZmpobGtq"},
|
||||
Parameters: values,
|
||||
}
|
||||
|
||||
var deviceRegistration *api.DeviceRegistrationResponse
|
||||
_, err = srv.CallJSON(ctx, &opts, nil, &deviceRegistration)
|
||||
return deviceRegistration, err
|
||||
}
|
||||
|
||||
// doAuthV1 runs the actual token request for V1 authentification
|
||||
func doAuthV1(ctx context.Context, srv *rest.Client, username, password string) (token oauth2.Token, err error) {
|
||||
// prepare out token request with username and password
|
||||
values := url.Values{}
|
||||
values.Set("grant_type", "PASSWORD")
|
||||
values.Set("password", password)
|
||||
values.Set("username", username)
|
||||
values.Set("client_id", oauthConfig.ClientID)
|
||||
values.Set("client_secret", oauthConfig.ClientSecret)
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
RootURL: oauthConfig.Endpoint.AuthURL,
|
||||
ContentType: "application/x-www-form-urlencoded",
|
||||
Parameters: values,
|
||||
}
|
||||
|
||||
// do the first request
|
||||
var jsonToken api.TokenJSON
|
||||
resp, err := srv.CallJSON(ctx, &opts, nil, &jsonToken)
|
||||
if err != nil {
|
||||
// if 2fa is enabled the first request is expected to fail. We will do another request with the 2fa code as an additional http header
|
||||
if resp != nil {
|
||||
if resp.Header.Get("X-JottaCloud-OTP") == "required; SMS" {
|
||||
fmt.Printf("This account uses 2 factor authentication you will receive a verification code via SMS.\n")
|
||||
fmt.Printf("Enter verification code> ")
|
||||
authCode := config.ReadLine()
|
||||
|
||||
authCode = strings.Replace(authCode, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
|
||||
opts.ExtraHeaders = make(map[string]string)
|
||||
opts.ExtraHeaders["X-Jottacloud-Otp"] = authCode
|
||||
resp, err = srv.CallJSON(ctx, &opts, nil, &jsonToken)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
token.AccessToken = jsonToken.AccessToken
|
||||
token.RefreshToken = jsonToken.RefreshToken
|
||||
token.TokenType = jsonToken.TokenType
|
||||
token.Expiry = time.Now().Add(time.Duration(jsonToken.ExpiresIn) * time.Second)
|
||||
return token, err
|
||||
}
|
||||
|
||||
// v2config configure a jottacloud backend using the modern JottaCli token based authentification
|
||||
func v2config(ctx context.Context, name string, m configmap.Mapper) {
|
||||
srv := rest.NewClient(fshttp.NewClient(fs.Config))
|
||||
|
||||
fmt.Printf("Generate a personal login token here: https://www.jottacloud.com/web/secure\n")
|
||||
fmt.Printf("Login Token> ")
|
||||
loginToken := config.ReadLine()
|
||||
|
||||
token, err := doAuthV2(ctx, srv, loginToken, m)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to get oauth token: %s", err)
|
||||
}
|
||||
err = oauthutil.PutToken(name, m, &token, true)
|
||||
if err != nil {
|
||||
log.Fatalf("Error while saving token: %s", err)
|
||||
}
|
||||
|
||||
fmt.Printf("\nDo you want to use a non standard device/mountpoint e.g. for accessing files uploaded using the official Jottacloud client?\n\n")
|
||||
if config.Confirm(false) {
|
||||
oauthConfig.ClientID = "jottacli"
|
||||
oAuthClient, _, err := oauthutil.NewClient(name, m, oauthConfig)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to load oAuthClient: %s", err)
|
||||
}
|
||||
|
||||
srv = rest.NewClient(oAuthClient).SetRoot(rootURL)
|
||||
apiSrv := rest.NewClient(oAuthClient).SetRoot(apiURL)
|
||||
device, mountpoint, err := setupMountpoint(ctx, srv, apiSrv)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to setup mountpoint: %s", err)
|
||||
}
|
||||
m.Set(configDevice, device)
|
||||
m.Set(configMountpoint, mountpoint)
|
||||
}
|
||||
|
||||
m.Set("configVersion", strconv.Itoa(configVersion))
|
||||
}
|
||||
|
||||
// doAuthV2 runs the actual token request for V2 authentification
|
||||
func doAuthV2(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
|
||||
// doAuth runs the actual token request
|
||||
func doAuth(ctx context.Context, srv *rest.Client, loginTokenBase64 string, m configmap.Mapper) (token oauth2.Token, err error) {
|
||||
loginTokenBytes, err := base64.RawURLEncoding.DecodeString(loginTokenBase64)
|
||||
if err != nil {
|
||||
return token, err
|
||||
@@ -615,29 +469,6 @@ func (f *Fs) filePath(file string) string {
|
||||
return urlPathEscape(f.filePathRaw(file))
|
||||
}
|
||||
|
||||
// Jottacloud requires the grant_type 'refresh_token' string
|
||||
// to be uppercase and throws a 400 Bad Request if we use the
|
||||
// lower case used by the oauth2 module
|
||||
//
|
||||
// This filter catches all refresh requests, reads the body,
|
||||
// changes the case and then sends it on
|
||||
func grantTypeFilter(req *http.Request) {
|
||||
if v1tokenURL == req.URL.String() {
|
||||
// read the entire body
|
||||
refreshBody, err := ioutil.ReadAll(req.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = req.Body.Close()
|
||||
|
||||
// make the refresh token upper case
|
||||
refreshBody = []byte(strings.Replace(string(refreshBody), "grant_type=refresh_token", "grant_type=REFRESH_TOKEN", 1))
|
||||
|
||||
// set the new ReadCloser (with a dummy Close())
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader(refreshBody))
|
||||
}
|
||||
}
|
||||
|
||||
// NewFs constructs an Fs from the path, container:path
|
||||
func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
ctx := context.TODO()
|
||||
@@ -649,55 +480,28 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
}
|
||||
|
||||
// Check config version
|
||||
var ver int
|
||||
version, ok := m.Get("configVersion")
|
||||
if ok {
|
||||
ver, err = strconv.Atoi(version)
|
||||
var ok bool
|
||||
var version string
|
||||
if version, ok = m.Get("configVersion"); ok {
|
||||
ver, err := strconv.Atoi(version)
|
||||
if err != nil {
|
||||
return nil, errors.New("Failed to parse config version")
|
||||
}
|
||||
ok = (ver == configVersion) || (ver == v1configVersion)
|
||||
ok = ver == configVersion
|
||||
}
|
||||
if !ok {
|
||||
return nil, errors.New("Outdated config - please reconfigure this backend")
|
||||
}
|
||||
|
||||
baseClient := fshttp.NewClient(fs.Config)
|
||||
|
||||
if ver == configVersion {
|
||||
oauthConfig.ClientID = "jottacli"
|
||||
// if custom endpoints are set use them else stick with defaults
|
||||
if tokenURL, ok := m.Get(configTokenURL); ok {
|
||||
oauthConfig.Endpoint.TokenURL = tokenURL
|
||||
// jottacloud is weird. we need to use the tokenURL as authURL
|
||||
oauthConfig.Endpoint.AuthURL = tokenURL
|
||||
}
|
||||
} else if ver == v1configVersion {
|
||||
clientID, ok := m.Get(configClientID)
|
||||
if !ok {
|
||||
clientID = v1ClientID
|
||||
}
|
||||
clientSecret, ok := m.Get(configClientSecret)
|
||||
if !ok {
|
||||
clientSecret = v1EncryptedClientSecret
|
||||
}
|
||||
oauthConfig.ClientID = clientID
|
||||
oauthConfig.ClientSecret = obscure.MustReveal(clientSecret)
|
||||
|
||||
oauthConfig.Endpoint.TokenURL = v1tokenURL
|
||||
oauthConfig.Endpoint.AuthURL = v1tokenURL
|
||||
|
||||
// add the request filter to fix token refresh
|
||||
if do, ok := baseClient.Transport.(interface {
|
||||
SetRequestFilter(f func(req *http.Request))
|
||||
}); ok {
|
||||
do.SetRequestFilter(grantTypeFilter)
|
||||
} else {
|
||||
fs.Debugf(name+":", "Couldn't add request filter - uploads will fail")
|
||||
}
|
||||
// if custom endpoints are set use them else stick with defaults
|
||||
if tokenURL, ok := m.Get(configTokenURL); ok {
|
||||
oauthConfig.Endpoint.TokenURL = tokenURL
|
||||
// jottacloud is weird. we need to use the tokenURL as authURL
|
||||
oauthConfig.Endpoint.AuthURL = tokenURL
|
||||
}
|
||||
|
||||
// Create OAuth Client
|
||||
baseClient := fshttp.NewClient(fs.Config)
|
||||
oAuthClient, ts, err := oauthutil.NewClientWithBaseClient(name, m, oauthConfig, baseClient)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Failed to configure Jottacloud oauth client")
|
||||
@@ -1087,8 +891,7 @@ func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *ap
|
||||
var resp *http.Response
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
|
||||
retry, _ := shouldRetry(resp, err)
|
||||
return (retry && resp.StatusCode != 500), err
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1192,18 +995,6 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
|
||||
_, err = f.copyOrMove(ctx, "mvDir", path.Join(f.endpointURL, f.opt.Enc.FromStandardPath(srcPath))+"/", dstRemote)
|
||||
|
||||
// surprise! jottacloud fucked up dirmove - the api spits out an error but
|
||||
// dir gets moved regardless
|
||||
if apiErr, ok := err.(*api.Error); ok {
|
||||
if apiErr.StatusCode == 500 {
|
||||
_, err := f.NewObject(ctx, dstRemote)
|
||||
if err == fs.ErrorNotAFile {
|
||||
log.Printf("FIXME: ignoring DirMove error - move succeeded anyway\n")
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "couldn't move directory")
|
||||
}
|
||||
@@ -1211,14 +1002,14 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
opts := rest.Opts{
|
||||
Method: "GET",
|
||||
Path: f.filePath(remote),
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
|
||||
if unlink {
|
||||
if f.opt.Unlink {
|
||||
opts.Parameters.Set("mode", "disableShare")
|
||||
} else {
|
||||
opts.Parameters.Set("mode", "enableShare")
|
||||
@@ -1238,12 +1029,12 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if unlink {
|
||||
if f.opt.Unlink {
|
||||
return "", errors.Wrap(err, "couldn't remove public link")
|
||||
}
|
||||
return "", errors.Wrap(err, "couldn't create public link")
|
||||
}
|
||||
if unlink {
|
||||
if f.opt.Unlink {
|
||||
if result.PublicSharePath != "" {
|
||||
return "", errors.Errorf("couldn't remove public link - %q", result.PublicSharePath)
|
||||
}
|
||||
|
||||
@@ -603,7 +603,7 @@ func createLink(c *koofrclient.KoofrClient, mountID string, path string) (*link,
|
||||
}
|
||||
|
||||
// PublicLink creates a public link to the remote path
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
|
||||
linkData, err := createLink(f.client, f.mountID, f.fullPath(remote))
|
||||
if err != nil {
|
||||
return "", translateErrorsDir(err)
|
||||
|
||||
+61
-84
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -89,24 +90,7 @@ are being uploaded and aborts with a message which starts "can't copy
|
||||
|
||||
However on some file systems this modification time check may fail (eg
|
||||
[Glusterfs #2206](https://github.com/rclone/rclone/issues/2206)) so this
|
||||
check can be disabled with this flag.
|
||||
|
||||
If this flag is set, rclone will use its best efforts to transfer a
|
||||
file which is being updated. If the file is only having things
|
||||
appended to it (eg a log) then rclone will transfer the log file with
|
||||
the size it had the first time rclone saw it.
|
||||
|
||||
If the file is being modified throughout (not just appended to) then
|
||||
the transfer may fail with a hash check failure.
|
||||
|
||||
In detail, once the file has had stat() called on it for the first
|
||||
time we:
|
||||
|
||||
- Only transfer the size that stat gave
|
||||
- Only checksum the size that stat gave
|
||||
- Don't update the stat info for the file
|
||||
|
||||
`,
|
||||
check can be disabled with this flag.`,
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
@@ -182,22 +166,20 @@ type Fs struct {
|
||||
warned map[string]struct{} // whether we have warned about this string
|
||||
|
||||
// do os.Lstat or os.Stat
|
||||
lstat func(name string) (os.FileInfo, error)
|
||||
objectMetaMu sync.RWMutex // global lock for Object metadata
|
||||
lstat func(name string) (os.FileInfo, error)
|
||||
objectHashesMu sync.Mutex // global lock for Object.hashes
|
||||
}
|
||||
|
||||
// Object represents a local filesystem object
|
||||
type Object struct {
|
||||
fs *Fs // The Fs this object is part of
|
||||
remote string // The remote path (encoded path)
|
||||
path string // The local path (OS path)
|
||||
// When using these items the fs.objectMetaMu must be held
|
||||
size int64 // file metadata - always present
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
hashes map[hash.Type]string // Hashes
|
||||
// these are read only and don't need the mutex held
|
||||
translatedLink bool // Is this object a translated link
|
||||
fs *Fs // The Fs this object is part of
|
||||
remote string // The remote path (encoded path)
|
||||
path string // The local path (OS path)
|
||||
size int64 // file metadata - always present
|
||||
mode os.FileMode
|
||||
modTime time.Time
|
||||
hashes map[hash.Type]string // Hashes
|
||||
translatedLink bool // Is this object a translated link
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
@@ -232,7 +214,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
CaseInsensitive: f.caseInsensitive(),
|
||||
CanHaveEmptyDirectories: true,
|
||||
IsLocal: true,
|
||||
SlowHash: true,
|
||||
}).Fill(f)
|
||||
if opt.FollowSymlinks {
|
||||
f.lstat = os.Stat
|
||||
@@ -634,9 +615,6 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
|
||||
// Temporary Object under construction
|
||||
dstObj := f.newObject(remote)
|
||||
dstObj.fs.objectMetaMu.RLock()
|
||||
dstObjMode := dstObj.mode
|
||||
dstObj.fs.objectMetaMu.RUnlock()
|
||||
|
||||
// Check it is a file if it exists
|
||||
err := dstObj.lstat()
|
||||
@@ -644,7 +622,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
} else if !dstObj.fs.isRegular(dstObjMode) {
|
||||
} else if !dstObj.fs.isRegular(dstObj.mode) {
|
||||
// It isn't a file
|
||||
return nil, errors.New("can't move file onto non-file")
|
||||
}
|
||||
@@ -798,10 +776,8 @@ func (o *Object) Remote() string {
|
||||
// Hash returns the requested hash of a file as a lowercase hex string
|
||||
func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
// Check that the underlying file hasn't changed
|
||||
o.fs.objectMetaMu.RLock()
|
||||
oldtime := o.modTime
|
||||
oldsize := o.size
|
||||
o.fs.objectMetaMu.RUnlock()
|
||||
err := o.lstat()
|
||||
var changed bool
|
||||
if err != nil {
|
||||
@@ -813,16 +789,15 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
return "", errors.Wrap(err, "hash: failed to stat")
|
||||
}
|
||||
} else {
|
||||
o.fs.objectMetaMu.RLock()
|
||||
changed = !o.modTime.Equal(oldtime) || oldsize != o.size
|
||||
o.fs.objectMetaMu.RUnlock()
|
||||
}
|
||||
|
||||
o.fs.objectMetaMu.RLock()
|
||||
o.fs.objectHashesMu.Lock()
|
||||
hashes := o.hashes
|
||||
hashValue, hashFound := o.hashes[r]
|
||||
o.fs.objectMetaMu.RUnlock()
|
||||
o.fs.objectHashesMu.Unlock()
|
||||
|
||||
if changed || !hashFound {
|
||||
if changed || hashes == nil || !hashFound {
|
||||
var in io.ReadCloser
|
||||
|
||||
if !o.translatedLink {
|
||||
@@ -834,14 +809,9 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
} else {
|
||||
in, err = o.openTranslatedLink(0, -1)
|
||||
}
|
||||
// If not checking for updates, only read size given
|
||||
if o.fs.opt.NoCheckUpdated {
|
||||
in = readers.NewLimitedReadCloser(in, o.size)
|
||||
}
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "hash: failed to open")
|
||||
}
|
||||
var hashes map[hash.Type]string
|
||||
hashes, err = hash.StreamTypes(in, hash.NewHashSet(r))
|
||||
closeErr := in.Close()
|
||||
if err != nil {
|
||||
@@ -851,28 +821,24 @@ func (o *Object) Hash(ctx context.Context, r hash.Type) (string, error) {
|
||||
return "", errors.Wrap(closeErr, "hash: failed to close")
|
||||
}
|
||||
hashValue = hashes[r]
|
||||
o.fs.objectMetaMu.Lock()
|
||||
o.fs.objectHashesMu.Lock()
|
||||
if o.hashes == nil {
|
||||
o.hashes = hashes
|
||||
} else {
|
||||
o.hashes[r] = hashValue
|
||||
}
|
||||
o.fs.objectMetaMu.Unlock()
|
||||
o.fs.objectHashesMu.Unlock()
|
||||
}
|
||||
return hashValue, nil
|
||||
}
|
||||
|
||||
// Size returns the size of an object in bytes
|
||||
func (o *Object) Size() int64 {
|
||||
o.fs.objectMetaMu.RLock()
|
||||
defer o.fs.objectMetaMu.RUnlock()
|
||||
return o.size
|
||||
}
|
||||
|
||||
// ModTime returns the modification time of the object
|
||||
func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
o.fs.objectMetaMu.RLock()
|
||||
defer o.fs.objectMetaMu.RUnlock()
|
||||
return o.modTime
|
||||
}
|
||||
|
||||
@@ -893,9 +859,7 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
|
||||
|
||||
// Storable returns a boolean showing if this object is storable
|
||||
func (o *Object) Storable() bool {
|
||||
o.fs.objectMetaMu.RLock()
|
||||
mode := o.mode
|
||||
o.fs.objectMetaMu.RUnlock()
|
||||
if mode&os.ModeSymlink != 0 && !o.fs.opt.TranslateSymlinks {
|
||||
if !o.fs.opt.SkipSymlinks {
|
||||
fs.Logf(o, "Can't follow symlink without -L/--copy-links")
|
||||
@@ -928,15 +892,11 @@ func (file *localOpenFile) Read(p []byte) (n int, err error) {
|
||||
if err != nil {
|
||||
return 0, errors.Wrap(err, "can't read status of source file while transferring")
|
||||
}
|
||||
file.o.fs.objectMetaMu.RLock()
|
||||
oldtime := file.o.modTime
|
||||
oldsize := file.o.size
|
||||
file.o.fs.objectMetaMu.RUnlock()
|
||||
if oldsize != fi.Size() {
|
||||
return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", oldsize, fi.Size()))
|
||||
if file.o.size != fi.Size() {
|
||||
return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (size changed from %d to %d)", file.o.size, fi.Size()))
|
||||
}
|
||||
if !oldtime.Equal(fi.ModTime()) {
|
||||
return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", oldtime, fi.ModTime()))
|
||||
if !file.o.modTime.Equal(fi.ModTime()) {
|
||||
return 0, fserrors.NoLowLevelRetryError(errors.Errorf("can't copy - source file is being updated (mod time changed from %v to %v)", file.o.modTime, fi.ModTime()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -953,9 +913,9 @@ func (file *localOpenFile) Close() (err error) {
|
||||
err = file.in.Close()
|
||||
if err == nil {
|
||||
if file.hash.Size() == file.o.Size() {
|
||||
file.o.fs.objectMetaMu.Lock()
|
||||
file.o.fs.objectHashesMu.Lock()
|
||||
file.o.hashes = file.hash.Sums()
|
||||
file.o.fs.objectMetaMu.Unlock()
|
||||
file.o.fs.objectHashesMu.Unlock()
|
||||
}
|
||||
}
|
||||
return err
|
||||
@@ -980,7 +940,7 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
case *fs.SeekOption:
|
||||
offset = x.Offset
|
||||
case *fs.RangeOption:
|
||||
offset, limit = x.Decode(o.Size())
|
||||
offset, limit = x.Decode(o.size)
|
||||
case *fs.HashesOption:
|
||||
if x.Hashes.Count() > 0 {
|
||||
hasher, err = hash.NewMultiHasherTypes(x.Hashes)
|
||||
@@ -995,13 +955,6 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
|
||||
}
|
||||
}
|
||||
|
||||
// If not checking updated then limit to current size. This means if
|
||||
// file is being extended, readers will read a o.Size() bytes rather
|
||||
// than the new size making for a consistent upload.
|
||||
if limit < 0 && o.fs.opt.NoCheckUpdated {
|
||||
limit = o.size
|
||||
}
|
||||
|
||||
// Handle a translated link
|
||||
if o.translatedLink {
|
||||
return o.openTranslatedLink(offset, limit)
|
||||
@@ -1138,9 +1091,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
|
||||
// All successful so update the hashes
|
||||
if hasher != nil {
|
||||
o.fs.objectMetaMu.Lock()
|
||||
o.fs.objectHashesMu.Lock()
|
||||
o.hashes = hasher.Sums()
|
||||
o.fs.objectMetaMu.Unlock()
|
||||
o.fs.objectHashesMu.Unlock()
|
||||
}
|
||||
|
||||
// Set the mtime
|
||||
@@ -1198,15 +1151,17 @@ func (f *Fs) OpenWriterAt(ctx context.Context, remote string, size int64) (fs.Wr
|
||||
|
||||
// setMetadata sets the file info from the os.FileInfo passed in
|
||||
func (o *Object) setMetadata(info os.FileInfo) {
|
||||
// if not checking updated then don't update the stat
|
||||
if o.fs.opt.NoCheckUpdated && !o.modTime.IsZero() {
|
||||
return
|
||||
// Don't overwrite the info if we don't need to
|
||||
// this avoids upsetting the race detector
|
||||
if o.size != info.Size() {
|
||||
o.size = info.Size()
|
||||
}
|
||||
if !o.modTime.Equal(info.ModTime()) {
|
||||
o.modTime = info.ModTime()
|
||||
}
|
||||
if o.mode != info.Mode() {
|
||||
o.mode = info.Mode()
|
||||
}
|
||||
o.fs.objectMetaMu.Lock()
|
||||
o.size = info.Size()
|
||||
o.modTime = info.ModTime()
|
||||
o.mode = info.Mode()
|
||||
o.fs.objectMetaMu.Unlock()
|
||||
}
|
||||
|
||||
// Stat an Object into info
|
||||
@@ -1238,7 +1193,7 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
|
||||
|
||||
if !noUNC {
|
||||
// Convert to UNC
|
||||
s = file.UNCPath(s)
|
||||
s = uncPath(s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
@@ -1252,6 +1207,28 @@ func cleanRootPath(s string, noUNC bool, enc encoder.MultiEncoder) string {
|
||||
return s
|
||||
}
|
||||
|
||||
// Pattern to match a windows absolute path: "c:\" and similar
|
||||
var isAbsWinDrive = regexp.MustCompile(`^[a-zA-Z]\:\\`)
|
||||
|
||||
// uncPath converts an absolute Windows path
|
||||
// to a UNC long path.
|
||||
func uncPath(l string) string {
|
||||
// If prefix is "\\", we already have a UNC path or server.
|
||||
if strings.HasPrefix(l, `\\`) {
|
||||
// If already long path, just keep it
|
||||
if strings.HasPrefix(l, `\\?\`) {
|
||||
return l
|
||||
}
|
||||
|
||||
// Trim "\\" from path and add UNC prefix.
|
||||
return `\\?\UNC\` + strings.TrimPrefix(l, `\\`)
|
||||
}
|
||||
if isAbsWinDrive.MatchString(l) {
|
||||
return `\\?\` + l
|
||||
}
|
||||
return l
|
||||
}
|
||||
|
||||
// Check the interfaces are satisfied
|
||||
var (
|
||||
_ fs.Fs = &Fs{}
|
||||
|
||||
@@ -5,6 +5,49 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var uncTestPaths = []string{
|
||||
`C:\Ba*d\P|a?t<h>\Windows\Folder`,
|
||||
`C:\Windows\Folder`,
|
||||
`\\?\C:\Windows\Folder`,
|
||||
`\\?\UNC\server\share\Desktop`,
|
||||
`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`\\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`\\server\share\Desktop`,
|
||||
`\\?\UNC\\share\folder\Desktop`,
|
||||
`\\server\share`,
|
||||
}
|
||||
|
||||
var uncTestPathsResults = []string{
|
||||
`\\?\C:\Ba*d\P|a?t<h>\Windows\Folder`,
|
||||
`\\?\C:\Windows\Folder`,
|
||||
`\\?\C:\Windows\Folder`,
|
||||
`\\?\UNC\server\share\Desktop`,
|
||||
`\\?\unC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`\\?\UNC\server\share\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`\\?\C:\Desktop\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`\\?\C:\AbsoluteToRoot\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path\Very Long path`,
|
||||
`\\?\UNC\server\share\Desktop`,
|
||||
`\\?\UNC\\share\folder\Desktop`,
|
||||
`\\?\UNC\server\share`,
|
||||
}
|
||||
|
||||
// Test that UNC paths are converted.
|
||||
func TestUncPaths(t *testing.T) {
|
||||
for i, p := range uncTestPaths {
|
||||
unc := uncPath(p)
|
||||
if unc != uncTestPathsResults[i] {
|
||||
t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
|
||||
}
|
||||
// Test we don't add more.
|
||||
unc = uncPath(unc)
|
||||
if unc != uncTestPathsResults[i] {
|
||||
t.Fatalf("UNC test path\nInput:%s\nOutput:%s\nExpected:%s", p, unc, uncTestPathsResults[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test Windows character replacements
|
||||
var testsWindows = [][2]string{
|
||||
{`c:\temp`, `c:\temp`},
|
||||
|
||||
@@ -1450,7 +1450,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
// fs.Debugf(f, ">>> PublicLink %q", remote)
|
||||
|
||||
token, err := f.accessToken()
|
||||
|
||||
@@ -836,7 +836,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
root, err := f.findRoot(false)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "PublicLink failed to find root node")
|
||||
|
||||
+103
-34
@@ -498,13 +498,13 @@ func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.It
|
||||
var dirCacheFoundRoot bool
|
||||
var rootNormalizedID string
|
||||
if f.dirCache != nil {
|
||||
rootNormalizedID, err = f.dirCache.RootID(ctx, false)
|
||||
dirCacheRootIDExists := err == nil
|
||||
var dirCacheRootIDExists bool
|
||||
rootNormalizedID, dirCacheRootIDExists = f.dirCache.Get("")
|
||||
if f.root == "" {
|
||||
// if f.root == "", it means f.root is the absolute root of the drive
|
||||
// and its ID should have been found in NewFs
|
||||
dirCacheFoundRoot = dirCacheRootIDExists
|
||||
} else if _, err := f.dirCache.RootParentID(ctx, false); err == nil {
|
||||
} else if _, err := f.dirCache.RootParentID(); err == nil {
|
||||
// if root is in a folder, it must have a parent folder, and
|
||||
// if dirCache has found root in NewFs, the parent folder's ID
|
||||
// should be present.
|
||||
@@ -813,6 +813,10 @@ OUTER:
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -860,7 +864,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, leaf, directoryID, err
|
||||
}
|
||||
@@ -891,7 +895,13 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -914,6 +924,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1107,10 +1121,9 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
id, dstDriveID, _ := parseNormalizedID(directoryID)
|
||||
_, srcObjDriveID, _ := parseNormalizedID(srcObj.id)
|
||||
|
||||
if f.canonicalDriveID(dstDriveID) != srcObj.fs.canonicalDriveID(srcObjDriveID) {
|
||||
if dstDriveID != srcObjDriveID {
|
||||
// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
|
||||
// "Items cannot be moved between Drives using this request."
|
||||
fs.Debugf(f, "Can't move files between drives (%q != %q)", dstDriveID, srcObjDriveID)
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
|
||||
@@ -1160,22 +1173,70 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, dstDirectoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
parsedDstDirID, dstDriveID, _ := parseNormalizedID(dstDirectoryID)
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, srcDriveID, _ := parseNormalizedID(srcID)
|
||||
|
||||
if f.canonicalDriveID(dstDriveID) != srcFs.canonicalDriveID(srcDriveID) {
|
||||
if dstDriveID != srcDriveID {
|
||||
// https://docs.microsoft.com/en-us/graph/api/driveitem-move?view=graph-rest-1.0
|
||||
// "Items cannot be moved between Drives using this request."
|
||||
fs.Debugf(f, "Can't move directories between drives (%q != %q)", dstDriveID, srcDriveID)
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Get timestamps of src so they can be preserved
|
||||
srcInfo, _, err := srcFs.readMetaDataForPathRelativeToID(ctx, srcID, "")
|
||||
if err != nil {
|
||||
@@ -1185,7 +1246,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
// Do the move
|
||||
opts := newOptsCall(srcID, "PATCH", "")
|
||||
move := api.MoveItemRequest{
|
||||
Name: f.opt.Enc.FromStandardName(dstLeaf),
|
||||
Name: f.opt.Enc.FromStandardName(leaf),
|
||||
ParentReference: &api.ItemReference{
|
||||
DriveID: dstDriveID,
|
||||
ID: parsedDstDirID,
|
||||
@@ -1250,7 +1311,7 @@ func (f *Fs) Hashes() hash.Set {
|
||||
}
|
||||
|
||||
// PublicLink returns a link for downloading without account.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
info, _, err := f.readMetaDataForPath(ctx, f.rootPath(remote))
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1627,22 +1688,41 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, size int64,
|
||||
return nil, errors.New("unknown-sized upload not supported")
|
||||
}
|
||||
|
||||
// Create upload session
|
||||
fs.Debugf(o, "Starting multipart upload")
|
||||
session, err := o.createUploadSession(ctx, modTime)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
uploadURL := session.UploadURL
|
||||
uploadURLChan := make(chan string, 1)
|
||||
gracefulCancel := func() {
|
||||
uploadURL, ok := <-uploadURLChan
|
||||
// Reading from uploadURLChan blocks the atexit process until
|
||||
// we are able to use uploadURL to cancel the upload
|
||||
if !ok { // createUploadSession failed - no need to cancel upload
|
||||
return
|
||||
}
|
||||
|
||||
// Cancel the session if something went wrong
|
||||
defer atexit.OnError(&err, func() {
|
||||
fs.Debugf(o, "Cancelling multipart upload: %v", err)
|
||||
fs.Debugf(o, "Cancelling multipart upload")
|
||||
cancelErr := o.cancelUploadSession(ctx, uploadURL)
|
||||
if cancelErr != nil {
|
||||
fs.Logf(o, "Failed to cancel multipart upload: %v", cancelErr)
|
||||
}
|
||||
})()
|
||||
}
|
||||
cancelFuncHandle := atexit.Register(gracefulCancel)
|
||||
|
||||
// Create upload session
|
||||
fs.Debugf(o, "Starting multipart upload")
|
||||
session, err := o.createUploadSession(ctx, modTime)
|
||||
if err != nil {
|
||||
close(uploadURLChan)
|
||||
atexit.Unregister(cancelFuncHandle)
|
||||
return nil, err
|
||||
}
|
||||
uploadURL := session.UploadURL
|
||||
uploadURLChan <- uploadURL
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
fs.Debugf(o, "Error encountered during upload: %v", err)
|
||||
gracefulCancel()
|
||||
}
|
||||
atexit.Unregister(cancelFuncHandle)
|
||||
}()
|
||||
|
||||
// Upload the chunks
|
||||
remaining := size
|
||||
@@ -1789,17 +1869,6 @@ func parseNormalizedID(ID string) (string, string, string) {
|
||||
return ID, "", ""
|
||||
}
|
||||
|
||||
// Returns the canonical form of the driveID
|
||||
func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
|
||||
if driveID == "" {
|
||||
canonicalDriveID = f.opt.DriveID
|
||||
} else {
|
||||
canonicalDriveID = driveID
|
||||
}
|
||||
canonicalDriveID = strings.ToLower(canonicalDriveID)
|
||||
return canonicalDriveID
|
||||
}
|
||||
|
||||
// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
|
||||
// returns a relative path for `target` based on `base` and a boolean `true`.
|
||||
// Otherwise returns "", false.
|
||||
|
||||
@@ -280,7 +280,13 @@ func errorHandler(resp *http.Response) error {
|
||||
// Mkdir creates the folder if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
// fs.Debugf(nil, "Mkdir(\"%s\")", dir)
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -306,6 +312,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -473,8 +483,58 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err = srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, directoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -486,9 +546,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
moveFolderData := moveCopyFolder{
|
||||
SessionID: f.session.SessionID,
|
||||
FolderID: srcID,
|
||||
DstFolderID: dstDirectoryID,
|
||||
DstFolderID: directoryID,
|
||||
Move: "true",
|
||||
NewFolderName: dstLeaf,
|
||||
NewFolderName: leaf,
|
||||
}
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
@@ -560,7 +620,7 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) {
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, leaf, directoryID, err
|
||||
}
|
||||
@@ -743,6 +803,10 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
// fs.Debugf(nil, "List(%v)", dir)
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -1015,7 +1079,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
}
|
||||
|
||||
func (o *Object) readMetaData(ctx context.Context) (err error) {
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return fs.ErrorObjectNotFound
|
||||
|
||||
@@ -152,14 +152,6 @@ type ChecksumFileResult struct {
|
||||
Metadata Item `json:"metadata"`
|
||||
}
|
||||
|
||||
// PubLinkResult is returned from /getfilepublink and /getfolderpublink
|
||||
type PubLinkResult struct {
|
||||
Error
|
||||
LinkID int `json:"linkid"`
|
||||
Link string `json:"link"`
|
||||
LinkCode string `json:"code"`
|
||||
}
|
||||
|
||||
// UserInfo is returned from /userinfo
|
||||
type UserInfo struct {
|
||||
Error
|
||||
|
||||
+78
-107
@@ -42,7 +42,7 @@ const (
|
||||
minSleep = 10 * time.Millisecond
|
||||
maxSleep = 2 * time.Second
|
||||
decayConstant = 2 // bigger for slower decay, exponential
|
||||
defaultHostname = "api.pcloud.com"
|
||||
rootURL = "https://api.pcloud.com"
|
||||
)
|
||||
|
||||
// Globals
|
||||
@@ -51,8 +51,8 @@ var (
|
||||
oauthConfig = &oauth2.Config{
|
||||
Scopes: nil,
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: "https://my.pcloud.com/oauth2/authorize",
|
||||
// TokenURL: "https://api.pcloud.com/oauth2_token", set by updateTokenURL
|
||||
AuthURL: "https://my.pcloud.com/oauth2/authorize",
|
||||
TokenURL: "https://api.pcloud.com/oauth2_token",
|
||||
},
|
||||
ClientID: rcloneClientID,
|
||||
ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret),
|
||||
@@ -60,45 +60,17 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
// Update the TokenURL with the actual hostname
|
||||
func updateTokenURL(oauthConfig *oauth2.Config, hostname string) {
|
||||
oauthConfig.Endpoint.TokenURL = "https://" + hostname + "/oauth2_token"
|
||||
}
|
||||
|
||||
// Register with Fs
|
||||
func init() {
|
||||
updateTokenURL(oauthConfig, defaultHostname)
|
||||
fs.Register(&fs.RegInfo{
|
||||
Name: "pcloud",
|
||||
Description: "Pcloud",
|
||||
NewFs: NewFs,
|
||||
Config: func(name string, m configmap.Mapper) {
|
||||
optc := new(Options)
|
||||
err := configstruct.Set(m, optc)
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to read config: %v", err)
|
||||
}
|
||||
updateTokenURL(oauthConfig, optc.Hostname)
|
||||
checkAuth := func(oauthConfig *oauth2.Config, auth *oauthutil.AuthResult) error {
|
||||
if auth == nil || auth.Form == nil {
|
||||
return errors.New("form not found in response")
|
||||
}
|
||||
hostname := auth.Form.Get("hostname")
|
||||
if hostname == "" {
|
||||
hostname = defaultHostname
|
||||
}
|
||||
// Save the hostname in the config
|
||||
m.Set("hostname", hostname)
|
||||
// Update the token URL
|
||||
updateTokenURL(oauthConfig, hostname)
|
||||
fs.Debugf(nil, "pcloud: got hostname %q", hostname)
|
||||
return nil
|
||||
}
|
||||
opt := oauthutil.Options{
|
||||
CheckAuth: checkAuth,
|
||||
StateBlankOK: true, // pCloud seems to drop the state parameter now - see #4210
|
||||
}
|
||||
err = oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
|
||||
err := oauthutil.Config("pcloud", name, m, oauthConfig, &opt)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to configure token: %v", err)
|
||||
}
|
||||
@@ -124,13 +96,6 @@ func init() {
|
||||
Help: "Fill in for rclone to use a non root folder as its starting point.",
|
||||
Default: "d0",
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "hostname",
|
||||
Help: `Hostname to connect to.
|
||||
|
||||
This is normally set when rclone initially does the oauth connection.`,
|
||||
Default: defaultHostname,
|
||||
Advanced: true,
|
||||
}},
|
||||
})
|
||||
}
|
||||
@@ -139,7 +104,6 @@ This is normally set when rclone initially does the oauth connection.`,
|
||||
type Options struct {
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
RootFolderID string `config:"root_folder_id"`
|
||||
Hostname string `config:"hostname"`
|
||||
}
|
||||
|
||||
// Fs represents a remote pcloud
|
||||
@@ -234,7 +198,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.Item, err error) {
|
||||
// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -289,13 +253,12 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "failed to configure Pcloud")
|
||||
}
|
||||
updateTokenURL(oauthConfig, opt.Hostname)
|
||||
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
opt: *opt,
|
||||
srv: rest.NewClient(oAuthClient).SetRoot("https://" + opt.Hostname),
|
||||
srv: rest.NewClient(oAuthClient).SetRoot(rootURL),
|
||||
pacer: fs.NewPacer(pacer.NewDefault(pacer.MinSleep(minSleep), pacer.MaxSleep(maxSleep), pacer.DecayConstant(decayConstant))),
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
@@ -492,6 +455,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -532,7 +499,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -563,7 +530,13 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -575,6 +548,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -682,7 +659,7 @@ func (f *Fs) Purge(ctx context.Context) error {
|
||||
|
||||
// CleanUp empties the trash
|
||||
func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
rootID, err := f.dirCache.RootID(ctx, false)
|
||||
err := f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -691,7 +668,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
|
||||
Path: "/trash_clear",
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
opts.Parameters.Set("folderid", dirIDtoNumber(rootID))
|
||||
opts.Parameters.Set("folderid", dirIDtoNumber(f.dirCache.RootID()))
|
||||
var resp *http.Response
|
||||
var result api.Error
|
||||
return f.pacer.Call(func() (bool, error) {
|
||||
@@ -764,8 +741,58 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, directoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -777,8 +804,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
opts.Parameters.Set("folderid", dirIDtoNumber(srcID))
|
||||
opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(dstLeaf))
|
||||
opts.Parameters.Set("tofolderid", dirIDtoNumber(dstDirectoryID))
|
||||
opts.Parameters.Set("toname", f.opt.Enc.FromStandardName(leaf))
|
||||
opts.Parameters.Set("tofolderid", dirIDtoNumber(directoryID))
|
||||
var resp *http.Response
|
||||
var result api.ItemResult
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
@@ -800,61 +827,6 @@ func (f *Fs) DirCacheFlush() {
|
||||
f.dirCache.ResetRoot()
|
||||
}
|
||||
|
||||
func (f *Fs) linkDir(ctx context.Context, dirID string, expire fs.Duration) (string, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/getfolderpublink",
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
var result api.PubLinkResult
|
||||
opts.Parameters.Set("folderid", dirIDtoNumber(dirID))
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return result.Link, err
|
||||
}
|
||||
|
||||
func (f *Fs) linkFile(ctx context.Context, path string, expire fs.Duration) (string, error) {
|
||||
opts := rest.Opts{
|
||||
Method: "POST",
|
||||
Path: "/getfilepublink",
|
||||
Parameters: url.Values{},
|
||||
}
|
||||
var result api.PubLinkResult
|
||||
opts.Parameters.Set("path", path)
|
||||
err := f.pacer.Call(func() (bool, error) {
|
||||
resp, err := f.srv.CallJSON(ctx, &opts, nil, &result)
|
||||
err = result.Error.Update(err)
|
||||
return shouldRetry(resp, err)
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return result.Link, nil
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
err := f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
dirID, err := f.dirCache.FindDir(ctx, remote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return f.linkFile(ctx, remote, expire)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return f.linkDir(ctx, dirID, expire)
|
||||
}
|
||||
|
||||
// About gets quota information
|
||||
func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
|
||||
opts := rest.Opts{
|
||||
@@ -1084,7 +1056,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
remote := o.Remote()
|
||||
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1189,7 +1161,6 @@ var (
|
||||
_ fs.Mover = (*Fs)(nil)
|
||||
_ fs.DirMover = (*Fs)(nil)
|
||||
_ fs.DirCacheFlusher = (*Fs)(nil)
|
||||
_ fs.PublicLinker = (*Fs)(nil)
|
||||
_ fs.Abouter = (*Fs)(nil)
|
||||
_ fs.Object = (*Object)(nil)
|
||||
_ fs.IDer = (*Object)(nil)
|
||||
|
||||
@@ -183,7 +183,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
|
||||
// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -450,6 +450,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -489,7 +493,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -540,7 +544,13 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -552,6 +562,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -731,14 +745,75 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, directoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Find ID of src parent, not creating subdirs
|
||||
var srcLeaf, srcDirectoryID string
|
||||
findPath = srcRemote
|
||||
if srcRemote == "" {
|
||||
findPath = srcFs.root
|
||||
}
|
||||
srcLeaf, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do the move
|
||||
err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
err = f.move(ctx, false, srcID, srcLeaf, leaf, srcDirectoryID, directoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -747,7 +822,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
|
||||
_, err := f.dirCache.FindDir(ctx, remote, false)
|
||||
if err == nil {
|
||||
return "", fs.ErrorCantShareDirectories
|
||||
@@ -915,7 +990,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
size := src.Size()
|
||||
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+68
-8
@@ -197,6 +197,10 @@ func (f *Fs) FindLeaf(ctx context.Context, pathID, leaf string) (pathIDOut strin
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -256,7 +260,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
// defer log.Trace(f, "src=%+v", src)("o=%+v, err=%v", &o, &err)
|
||||
size := src.Size()
|
||||
remote := src.Remote()
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -454,7 +458,13 @@ func (f *Fs) makeUploadPatchRequest(ctx context.Context, location string, in io.
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) (err error) {
|
||||
// defer log.Trace(f, "dir=%v", dir)("err=%v", &err)
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -518,11 +528,12 @@ func (f *Fs) Purge(ctx context.Context) (err error) {
|
||||
if f.root == "" {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
rootIDs, err := f.dirCache.RootID(ctx, false)
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID := atoi(rootIDs)
|
||||
|
||||
rootID := atoi(f.dirCache.RootID())
|
||||
// Let putio delete the filesystem tree
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
// fs.Debugf(f, "deleting file: %d", rootID)
|
||||
@@ -548,7 +559,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (o fs.Objec
|
||||
if !ok {
|
||||
return nil, fs.ErrorCantCopy
|
||||
}
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -587,7 +598,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (o fs.Objec
|
||||
if !ok {
|
||||
return nil, fs.ErrorCantMove
|
||||
}
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -625,8 +636,57 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
if !ok {
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err = srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, dstDirectoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, dstDirectoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -635,7 +695,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
params := url.Values{}
|
||||
params.Set("file_id", srcID)
|
||||
params.Set("parent_id", dstDirectoryID)
|
||||
params.Set("name", f.opt.Enc.FromStandardName(dstLeaf))
|
||||
params.Set("name", f.opt.Enc.FromStandardName(leaf))
|
||||
req, err := f.client.NewRequest(ctx, "POST", "/v2/files/move", strings.NewReader(params.Encode()))
|
||||
if err != nil {
|
||||
return false, err
|
||||
|
||||
@@ -125,7 +125,7 @@ func (o *Object) setMetadataFromEntry(info putio.File) error {
|
||||
// Reads the entry for a file from putio
|
||||
func (o *Object) readEntry(ctx context.Context) (f *putio.File, err error) {
|
||||
// defer log.Trace(o, "")("f=%+v, err=%v", f, &err)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, o.remote, false)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, o.remote, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
|
||||
@@ -356,7 +356,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
WriteMimeType: true,
|
||||
BucketBased: true,
|
||||
BucketBasedRootOK: true,
|
||||
SlowModTime: true,
|
||||
}).Fill(f)
|
||||
|
||||
if f.rootBucket != "" && f.rootDirectory != "" {
|
||||
|
||||
@@ -15,7 +15,6 @@ import (
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
qs "github.com/yunify/qingstor-sdk-go/v3/service"
|
||||
)
|
||||
|
||||
@@ -347,15 +346,12 @@ func (mu *multiUploader) multiPartUpload(firstBuf io.ReadSeeker) (err error) {
|
||||
if err = mu.initiate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Cancel the session if something went wrong
|
||||
defer atexit.OnError(&err, func() {
|
||||
fs.Debugf(mu, "Cancelling multipart upload: %v", err)
|
||||
cancelErr := mu.abort()
|
||||
if cancelErr != nil {
|
||||
fs.Logf(mu, "Failed to cancel multipart upload: %v", cancelErr)
|
||||
defer func() {
|
||||
// Abort the transfer if returning an error
|
||||
if err != nil {
|
||||
_ = mu.abort()
|
||||
}
|
||||
})()
|
||||
}()
|
||||
|
||||
ch := make(chan chunk, mu.cfg.concurrency)
|
||||
for i := 0; i < mu.cfg.concurrency; i++ {
|
||||
|
||||
+41
-249
@@ -40,7 +40,6 @@ import (
|
||||
"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
|
||||
"github.com/aws/aws-sdk-go/aws/defaults"
|
||||
"github.com/aws/aws-sdk-go/aws/ec2metadata"
|
||||
"github.com/aws/aws-sdk-go/aws/endpoints"
|
||||
"github.com/aws/aws-sdk-go/aws/request"
|
||||
"github.com/aws/aws-sdk-go/aws/session"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
@@ -53,9 +52,7 @@ import (
|
||||
"github.com/rclone/rclone/fs/fserrors"
|
||||
"github.com/rclone/rclone/fs/fshttp"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/rclone/rclone/fs/walk"
|
||||
"github.com/rclone/rclone/lib/atexit"
|
||||
"github.com/rclone/rclone/lib/bucket"
|
||||
"github.com/rclone/rclone/lib/encoder"
|
||||
"github.com/rclone/rclone/lib/pacer"
|
||||
@@ -72,7 +69,6 @@ func init() {
|
||||
Name: "s3",
|
||||
Description: "Amazon S3 Compliant Storage Provider (AWS, Alibaba, Ceph, Digital Ocean, Dreamhost, IBM COS, Minio, etc)",
|
||||
NewFs: NewFs,
|
||||
CommandHelp: commandHelp,
|
||||
Options: []fs.Option{{
|
||||
Name: fs.ConfigProvider,
|
||||
Help: "Choose your S3 provider.",
|
||||
@@ -100,9 +96,6 @@ func init() {
|
||||
}, {
|
||||
Value: "Netease",
|
||||
Help: "Netease Object Storage (NOS)",
|
||||
}, {
|
||||
Value: "Scaleway",
|
||||
Help: "Scaleway Object Storage",
|
||||
}, {
|
||||
Value: "StackPath",
|
||||
Help: "StackPath Object Storage",
|
||||
@@ -183,21 +176,10 @@ func init() {
|
||||
Value: "sa-east-1",
|
||||
Help: "South America (Sao Paulo) Region\nNeeds location constraint sa-east-1.",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.",
|
||||
Provider: "Scaleway",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "nl-ams",
|
||||
Help: "Amsterdam, The Netherlands",
|
||||
}, {
|
||||
Value: "fr-par",
|
||||
Help: "Paris, France",
|
||||
}},
|
||||
}, {
|
||||
Name: "region",
|
||||
Help: "Region to connect to.\nLeave blank if you are using an S3 clone and you don't have a region.",
|
||||
Provider: "!AWS,Alibaba,Scaleway",
|
||||
Provider: "!AWS,Alibaba",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Use this if unsure. Will use v4 signatures and an empty region.",
|
||||
@@ -379,17 +361,6 @@ func init() {
|
||||
Value: "oss-me-east-1.aliyuncs.com",
|
||||
Help: "Middle East 1 (Dubai)",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for Scaleway Object Storage.",
|
||||
Provider: "Scaleway",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "s3.nl-ams.scw.cloud",
|
||||
Help: "Amsterdam Endpoint",
|
||||
}, {
|
||||
Value: "s3.fr-par.scw.cloud",
|
||||
Help: "Paris Endpoint",
|
||||
}},
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for StackPath Object Storage.",
|
||||
@@ -407,7 +378,7 @@ func init() {
|
||||
}, {
|
||||
Name: "endpoint",
|
||||
Help: "Endpoint for S3 API.\nRequired when using an S3 clone.",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,StackPath",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "objects-us-east-1.dream.io",
|
||||
Help: "Dream Objects endpoint",
|
||||
@@ -594,7 +565,7 @@ func init() {
|
||||
}, {
|
||||
Name: "location_constraint",
|
||||
Help: "Location constraint - must be set to match the Region.\nLeave blank if not sure. Used when creating buckets only.",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,Scaleway,StackPath",
|
||||
Provider: "!AWS,IBMCOS,Alibaba,StackPath",
|
||||
}, {
|
||||
Name: "acl",
|
||||
Help: `Canned ACL used when creating buckets and storing or copying objects.
|
||||
@@ -770,21 +741,6 @@ isn't set then "acl" is used instead.`,
|
||||
Value: "STANDARD_IA",
|
||||
Help: "Infrequent access storage mode.",
|
||||
}},
|
||||
}, {
|
||||
// Mapping from here: https://www.scaleway.com/en/docs/object-storage-glacier/#-Scaleway-Storage-Classes
|
||||
Name: "storage_class",
|
||||
Help: "The storage class to use when storing new objects in S3.",
|
||||
Provider: "Scaleway",
|
||||
Examples: []fs.OptionExample{{
|
||||
Value: "",
|
||||
Help: "Default",
|
||||
}, {
|
||||
Value: "STANDARD",
|
||||
Help: "The Standard class for any upload; suitable for on-demand content like streaming or CDN.",
|
||||
}, {
|
||||
Value: "GLACIER",
|
||||
Help: "Archived storage; prices are lower, but it needs to be restored first to be accessed.",
|
||||
}},
|
||||
}, {
|
||||
Name: "upload_cutoff",
|
||||
Help: `Cutoff for switching to chunked upload
|
||||
@@ -818,21 +774,6 @@ file you can stream upload is 48GB. If you wish to stream upload
|
||||
larger files then you will need to increase chunk_size.`,
|
||||
Default: minChunkSize,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "max_upload_parts",
|
||||
Help: `Maximum number of parts in a multipart upload.
|
||||
|
||||
This option defines the maximum number of multipart chunks to use
|
||||
when doing a multipart upload.
|
||||
|
||||
This can be useful if a service does not support the AWS S3
|
||||
specification of 10,000 chunks.
|
||||
|
||||
Rclone will automatically increase the chunk size when uploading a
|
||||
large file of a known size to stay below this number of chunks limit.
|
||||
`,
|
||||
Default: maxUploadParts,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: "copy_cutoff",
|
||||
Help: `Cutoff for switching to multipart copy
|
||||
@@ -969,7 +910,6 @@ const (
|
||||
|
||||
memoryPoolFlushTime = fs.Duration(time.Minute) // flush the cached buffers after this long
|
||||
memoryPoolUseMmap = false
|
||||
maxExpireDuration = fs.Duration(7 * 24 * time.Hour) // max expiry is 1 week
|
||||
)
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
@@ -992,7 +932,6 @@ type Options struct {
|
||||
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
|
||||
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
|
||||
ChunkSize fs.SizeSuffix `config:"chunk_size"`
|
||||
MaxUploadParts int64 `config:"max_upload_parts"`
|
||||
DisableChecksum bool `config:"disable_checksum"`
|
||||
SessionToken string `config:"session_token"`
|
||||
UploadConcurrency int `config:"upload_concurrency"`
|
||||
@@ -1188,23 +1127,21 @@ func s3Connection(opt *Options) (*s3.S3, *session.Session, error) {
|
||||
return nil, nil, errors.New("secret_access_key not found")
|
||||
}
|
||||
|
||||
if opt.Region == "" && opt.Endpoint == "" {
|
||||
opt.Endpoint = "https://s3.amazonaws.com/"
|
||||
}
|
||||
if opt.Region == "" {
|
||||
opt.Region = "us-east-1"
|
||||
}
|
||||
if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.Provider == "Scaleway" || opt.UseAccelerateEndpoint {
|
||||
if opt.Provider == "AWS" || opt.Provider == "Alibaba" || opt.Provider == "Netease" || opt.UseAccelerateEndpoint {
|
||||
opt.ForcePathStyle = false
|
||||
}
|
||||
if opt.Provider == "Scaleway" && opt.MaxUploadParts > 1000 {
|
||||
opt.MaxUploadParts = 1000
|
||||
}
|
||||
awsConfig := aws.NewConfig().
|
||||
WithMaxRetries(0). // Rely on rclone's retry logic
|
||||
WithCredentials(cred).
|
||||
WithHTTPClient(fshttp.NewClient(fs.Config)).
|
||||
WithS3ForcePathStyle(opt.ForcePathStyle).
|
||||
WithS3UseAccelerate(opt.UseAccelerateEndpoint).
|
||||
WithS3UsEast1RegionalEndpoint(endpoints.RegionalS3UsEast1Endpoint)
|
||||
|
||||
WithS3UseAccelerate(opt.UseAccelerateEndpoint)
|
||||
if opt.Region != "" {
|
||||
awsConfig.WithRegion(opt.Region)
|
||||
}
|
||||
@@ -1331,7 +1268,6 @@ func NewFs(name, root string, m configmap.Mapper) (fs.Fs, error) {
|
||||
BucketBasedRootOK: true,
|
||||
SetTier: true,
|
||||
GetTier: true,
|
||||
SlowModTime: true,
|
||||
}).Fill(f)
|
||||
if f.rootBucket != "" && f.rootDirectory != "" {
|
||||
// Check to see if the object exists
|
||||
@@ -1904,19 +1840,21 @@ func (f *Fs) copyMultipart(ctx context.Context, req *s3.CopyObjectInput, dstBuck
|
||||
}
|
||||
uid := cout.UploadId
|
||||
|
||||
defer atexit.OnError(&err, func() {
|
||||
// Try to abort the upload, but ignore the error.
|
||||
fs.Debugf(nil, "Cancelling multipart copy")
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstPath,
|
||||
UploadId: uid,
|
||||
RequestPayer: req.RequestPayer,
|
||||
defer func() {
|
||||
if err != nil {
|
||||
// We can try to abort the upload, but ignore the error.
|
||||
fs.Debugf(nil, "Cancelling multipart copy")
|
||||
_ = f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
|
||||
Bucket: &dstBucket,
|
||||
Key: &dstPath,
|
||||
UploadId: uid,
|
||||
RequestPayer: req.RequestPayer,
|
||||
})
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
})()
|
||||
}
|
||||
}()
|
||||
|
||||
partSize := int64(f.opt.CopyCutoff)
|
||||
numParts := (srcSize-1)/partSize + 1
|
||||
@@ -2022,147 +1960,6 @@ func (f *Fs) getMemoryPool(size int64) *pool.Pool {
|
||||
)
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
if strings.HasSuffix(remote, "/") {
|
||||
return "", fs.ErrorCantShareDirectories
|
||||
}
|
||||
if _, err := f.NewObject(ctx, remote); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if expire > maxExpireDuration {
|
||||
fs.Logf(f, "Public Link: Reducing expiry to %v as %v is greater than the max time allowed", maxExpireDuration, expire)
|
||||
expire = maxExpireDuration
|
||||
}
|
||||
bucket, bucketPath := f.split(remote)
|
||||
httpReq, _ := f.c.GetObjectRequest(&s3.GetObjectInput{
|
||||
Bucket: &bucket,
|
||||
Key: &bucketPath,
|
||||
})
|
||||
|
||||
return httpReq.Presign(time.Duration(expire))
|
||||
}
|
||||
|
||||
var commandHelp = []fs.CommandHelp{{
|
||||
Name: "restore",
|
||||
Short: "Restore objects from GLACIER to normal storage",
|
||||
Long: `This command can be used to restore one or more objects from GLACIER
|
||||
to normal storage.
|
||||
|
||||
Usage Examples:
|
||||
|
||||
rclone backend restore s3:bucket/path/to/object [-o priority=PRIORITY] [-o lifetime=DAYS]
|
||||
rclone backend restore s3:bucket/path/to/directory [-o priority=PRIORITY] [-o lifetime=DAYS]
|
||||
rclone backend restore s3:bucket [-o priority=PRIORITY] [-o lifetime=DAYS]
|
||||
|
||||
This flag also obeys the filters. Test first with -i/--interactive or --dry-run flags
|
||||
|
||||
rclone -i backend restore --include "*.txt" s3:bucket/path -o priority=Standard
|
||||
|
||||
All the objects shown will be marked for restore, then
|
||||
|
||||
rclone backend restore --include "*.txt" s3:bucket/path -o priority=Standard
|
||||
|
||||
It returns a list of status dictionaries with Remote and Status
|
||||
keys. The Status will be OK if it was successfull or an error message
|
||||
if not.
|
||||
|
||||
[
|
||||
{
|
||||
"Status": "OK",
|
||||
"Path": "test.txt"
|
||||
},
|
||||
{
|
||||
"Status": "OK",
|
||||
"Path": "test/file4.txt"
|
||||
}
|
||||
]
|
||||
|
||||
`,
|
||||
Opts: map[string]string{
|
||||
"priority": "Priority of restore: Standard|Expedited|Bulk",
|
||||
"lifetime": "Lifetime of the active copy in days",
|
||||
"description": "The optional description for the job.",
|
||||
},
|
||||
}}
|
||||
|
||||
// Command the backend to run a named command
|
||||
//
|
||||
// The command run is name
|
||||
// args may be used to read arguments from
|
||||
// opts may be used to read optional arguments from
|
||||
//
|
||||
// The result should be capable of being JSON encoded
|
||||
// If it is a string or a []string it will be shown to the user
|
||||
// otherwise it will be JSON encoded and shown to the user like that
|
||||
func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[string]string) (out interface{}, err error) {
|
||||
switch name {
|
||||
case "restore":
|
||||
req := s3.RestoreObjectInput{
|
||||
//Bucket: &f.rootBucket,
|
||||
//Key: &encodedDirectory,
|
||||
RestoreRequest: &s3.RestoreRequest{},
|
||||
}
|
||||
if lifetime := opt["lifetime"]; lifetime != "" {
|
||||
ilifetime, err := strconv.ParseInt(lifetime, 10, 64)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "bad lifetime")
|
||||
}
|
||||
req.RestoreRequest.Days = &ilifetime
|
||||
}
|
||||
if priority := opt["priority"]; priority != "" {
|
||||
req.RestoreRequest.GlacierJobParameters = &s3.GlacierJobParameters{
|
||||
Tier: &priority,
|
||||
}
|
||||
}
|
||||
if description := opt["description"]; description != "" {
|
||||
req.RestoreRequest.Description = &description
|
||||
}
|
||||
type status struct {
|
||||
Status string
|
||||
Remote string
|
||||
}
|
||||
var (
|
||||
outMu sync.Mutex
|
||||
out = []status{}
|
||||
)
|
||||
err = operations.ListFn(ctx, f, func(obj fs.Object) {
|
||||
// Remember this is run --checkers times concurrently
|
||||
o, ok := obj.(*Object)
|
||||
st := status{Status: "OK", Remote: obj.Remote()}
|
||||
defer func() {
|
||||
outMu.Lock()
|
||||
out = append(out, st)
|
||||
outMu.Unlock()
|
||||
}()
|
||||
if operations.SkipDestructive(ctx, obj, "restore") {
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
st.Status = "Not an S3 object"
|
||||
return
|
||||
}
|
||||
bucket, bucketPath := o.split()
|
||||
reqCopy := req
|
||||
reqCopy.Bucket = &bucket
|
||||
reqCopy.Key = &bucketPath
|
||||
err = f.pacer.Call(func() (bool, error) {
|
||||
_, err = f.c.RestoreObject(&reqCopy)
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if err != nil {
|
||||
st.Status = err.Error()
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return out, err
|
||||
}
|
||||
return out, nil
|
||||
default:
|
||||
return nil, fs.ErrorCommandNotFound
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------
|
||||
|
||||
// Fs returns the parent Fs
|
||||
@@ -2387,13 +2184,6 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
}
|
||||
tokens := pacer.NewTokenDispenser(concurrency)
|
||||
|
||||
uploadParts := f.opt.MaxUploadParts
|
||||
if uploadParts < 1 {
|
||||
uploadParts = 1
|
||||
} else if uploadParts > maxUploadParts {
|
||||
uploadParts = maxUploadParts
|
||||
}
|
||||
|
||||
// calculate size of parts
|
||||
partSize := int(f.opt.ChunkSize)
|
||||
|
||||
@@ -2403,13 +2193,13 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
if size == -1 {
|
||||
warnStreamUpload.Do(func() {
|
||||
fs.Logf(f, "Streaming uploads using chunk size %v will have maximum file size of %v",
|
||||
f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
|
||||
f.opt.ChunkSize, fs.SizeSuffix(partSize*maxUploadParts))
|
||||
})
|
||||
} else {
|
||||
// Adjust partSize until the number of parts is small enough.
|
||||
if size/int64(partSize) >= uploadParts {
|
||||
if size/int64(partSize) >= maxUploadParts {
|
||||
// Calculate partition size rounded up to the nearest MB
|
||||
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
|
||||
partSize = int((((size / maxUploadParts) >> 20) + 1) << 20)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2428,24 +2218,27 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
|
||||
}
|
||||
uid := cout.UploadId
|
||||
|
||||
defer atexit.OnError(&err, func() {
|
||||
defer func() {
|
||||
if o.fs.opt.LeavePartsOnError {
|
||||
return
|
||||
}
|
||||
fs.Debugf(o, "Cancelling multipart upload")
|
||||
errCancel := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
|
||||
Bucket: req.Bucket,
|
||||
Key: req.Key,
|
||||
UploadId: uid,
|
||||
RequestPayer: req.RequestPayer,
|
||||
if err != nil {
|
||||
// We can try to abort the upload, but ignore the error.
|
||||
fs.Debugf(o, "Cancelling multipart upload")
|
||||
errCancel := f.pacer.Call(func() (bool, error) {
|
||||
_, err := f.c.AbortMultipartUploadWithContext(context.Background(), &s3.AbortMultipartUploadInput{
|
||||
Bucket: req.Bucket,
|
||||
Key: req.Key,
|
||||
UploadId: uid,
|
||||
RequestPayer: req.RequestPayer,
|
||||
})
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
return f.shouldRetry(err)
|
||||
})
|
||||
if errCancel != nil {
|
||||
fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
|
||||
if errCancel != nil {
|
||||
fs.Debugf(o, "Failed to cancel multipart upload: %v", errCancel)
|
||||
}
|
||||
}
|
||||
})()
|
||||
}()
|
||||
|
||||
var (
|
||||
g, gCtx = errgroup.WithContext(ctx)
|
||||
@@ -2780,7 +2573,6 @@ var (
|
||||
_ fs.Copier = &Fs{}
|
||||
_ fs.PutStreamer = &Fs{}
|
||||
_ fs.ListRer = &Fs{}
|
||||
_ fs.Commander = &Fs{}
|
||||
_ fs.Object = &Object{}
|
||||
_ fs.MimeTyper = &Object{}
|
||||
_ fs.GetTierer = &Object{}
|
||||
|
||||
@@ -972,7 +972,7 @@ func (f *Fs) UserInfo(ctx context.Context) (map[string]string, error) {
|
||||
// ==================== Optional Interface fs.PublicLinker ====================
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
|
||||
libraryName, filePath := f.splitPath(remote)
|
||||
if libraryName == "" {
|
||||
// We cannot share the whole seafile server, we need at least a library
|
||||
|
||||
+10
-23
@@ -74,7 +74,7 @@ func init() {
|
||||
Help: "Raw PEM-encoded private key, If specified, will override key_file parameter.",
|
||||
}, {
|
||||
Name: "key_file",
|
||||
Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent." + env.ShellExpandHelp,
|
||||
Help: "Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.",
|
||||
}, {
|
||||
Name: "key_file_pass",
|
||||
Help: `The passphrase to decrypt the PEM-encoded private key file.
|
||||
@@ -193,7 +193,6 @@ type Options struct {
|
||||
type Fs struct {
|
||||
name string
|
||||
root string
|
||||
absRoot string
|
||||
opt Options // parsed options
|
||||
m configmap.Mapper // config
|
||||
features *fs.Features // optional features
|
||||
@@ -492,7 +491,6 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
|
||||
f := &Fs{
|
||||
name: name,
|
||||
root: root,
|
||||
absRoot: root,
|
||||
opt: *opt,
|
||||
m: m,
|
||||
config: sshConfig,
|
||||
@@ -502,27 +500,17 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
|
||||
}
|
||||
f.features = (&fs.Features{
|
||||
CanHaveEmptyDirectories: true,
|
||||
SlowHash: true,
|
||||
}).Fill(f)
|
||||
// Make a connection and pool it to return errors early
|
||||
c, err := f.getSftpConnection()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "NewFs")
|
||||
}
|
||||
cwd, err := c.sftpClient.Getwd()
|
||||
f.putSftpConnection(&c, nil)
|
||||
if err != nil {
|
||||
fs.Debugf(f, "Failed to read current directory - using relative paths: %v", err)
|
||||
} else if !path.IsAbs(f.root) {
|
||||
f.absRoot = path.Join(cwd, f.root)
|
||||
fs.Debugf(f, "Using absolute root directory %q", f.absRoot)
|
||||
}
|
||||
if root != "" {
|
||||
// Check to see if the root actually an existing file
|
||||
oldAbsRoot := f.absRoot
|
||||
remote := path.Base(root)
|
||||
f.root = path.Dir(root)
|
||||
f.absRoot = path.Dir(f.absRoot)
|
||||
if f.root == "." {
|
||||
f.root = ""
|
||||
}
|
||||
@@ -531,7 +519,6 @@ func NewFsWithConnection(ctx context.Context, name string, root string, m config
|
||||
if err == fs.ErrorObjectNotFound || errors.Cause(err) == fs.ErrorNotAFile {
|
||||
// File doesn't exist so return old f
|
||||
f.root = root
|
||||
f.absRoot = oldAbsRoot
|
||||
return f, nil
|
||||
}
|
||||
return nil, err
|
||||
@@ -614,7 +601,7 @@ func (f *Fs) dirExists(dir string) (bool, error) {
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
root := path.Join(f.absRoot, dir)
|
||||
root := path.Join(f.root, dir)
|
||||
ok, err := f.dirExists(root)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "List failed")
|
||||
@@ -695,7 +682,7 @@ func (f *Fs) PutStream(ctx context.Context, in io.Reader, src fs.ObjectInfo, opt
|
||||
// directories above that
|
||||
func (f *Fs) mkParentDir(remote string) error {
|
||||
parent := path.Dir(remote)
|
||||
return f.mkdir(path.Join(f.absRoot, parent))
|
||||
return f.mkdir(path.Join(f.root, parent))
|
||||
}
|
||||
|
||||
// mkdir makes the directory and parents using native paths
|
||||
@@ -731,7 +718,7 @@ func (f *Fs) mkdir(dirPath string) error {
|
||||
|
||||
// Mkdir makes the root directory of the Fs object
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
root := path.Join(f.absRoot, dir)
|
||||
root := path.Join(f.root, dir)
|
||||
return f.mkdir(root)
|
||||
}
|
||||
|
||||
@@ -747,7 +734,7 @@ func (f *Fs) Rmdir(ctx context.Context, dir string) error {
|
||||
return fs.ErrorDirectoryNotEmpty
|
||||
}
|
||||
// Remove the directory
|
||||
root := path.Join(f.absRoot, dir)
|
||||
root := path.Join(f.root, dir)
|
||||
c, err := f.getSftpConnection()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "Rmdir")
|
||||
@@ -774,7 +761,7 @@ func (f *Fs) Move(ctx context.Context, src fs.Object, remote string) (fs.Object,
|
||||
}
|
||||
err = c.sftpClient.Rename(
|
||||
srcObj.path(),
|
||||
path.Join(f.absRoot, remote),
|
||||
path.Join(f.root, remote),
|
||||
)
|
||||
f.putSftpConnection(&c, err)
|
||||
if err != nil {
|
||||
@@ -801,8 +788,8 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.absRoot, srcRemote)
|
||||
dstPath := path.Join(f.absRoot, dstRemote)
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
// Check if destination exists
|
||||
ok, err := f.dirExists(dstPath)
|
||||
@@ -1088,7 +1075,7 @@ func (o *Object) ModTime(ctx context.Context) time.Time {
|
||||
|
||||
// path returns the native path of the object
|
||||
func (o *Object) path() string {
|
||||
return path.Join(o.fs.absRoot, o.remote)
|
||||
return path.Join(o.fs.root, o.remote)
|
||||
}
|
||||
|
||||
// setMetadata updates the info in the object from the stat result passed in
|
||||
@@ -1104,7 +1091,7 @@ func (f *Fs) stat(remote string) (info os.FileInfo, err error) {
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "stat")
|
||||
}
|
||||
absPath := path.Join(f.absRoot, remote)
|
||||
absPath := path.Join(f.root, remote)
|
||||
info, err = c.sftpClient.Stat(absPath)
|
||||
f.putSftpConnection(&c, err)
|
||||
return info, err
|
||||
|
||||
@@ -350,7 +350,7 @@ func (f *Fs) readMetaDataForID(ctx context.Context, id string, directoriesOnly b
|
||||
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string, directoriesOnly bool, filesOnly bool) (info *api.Item, err error) {
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -700,6 +700,10 @@ func (f *Fs) listAll(ctx context.Context, dirID string, directoriesOnly bool, fi
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -739,7 +743,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -795,7 +799,13 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -807,6 +817,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rootID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1019,14 +1033,75 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, srcDirectoryID, srcLeaf, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, directoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Find ID of src parent, not creating subdirs
|
||||
var srcLeaf, srcDirectoryID string
|
||||
findPath = srcRemote
|
||||
if srcRemote == "" {
|
||||
findPath = srcFs.root
|
||||
}
|
||||
srcLeaf, srcDirectoryID, err = srcFs.dirCache.FindPath(ctx, findPath, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do the move
|
||||
_, err = f.move(ctx, false, srcID, srcLeaf, dstLeaf, srcDirectoryID, dstDirectoryID)
|
||||
_, err = f.move(ctx, false, srcID, srcLeaf, leaf, srcDirectoryID, directoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1327,7 +1402,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
isLargeFile := size < 0 || size > int64(o.fs.opt.UploadCutoff)
|
||||
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -255,7 +255,7 @@ func shouldRetry(resp *http.Response, err error) (bool, error) {
|
||||
// readMetaDataForPath reads the metadata from the path
|
||||
func (f *Fs) readMetaDataForPath(ctx context.Context, path string) (info *api.File, err error) {
|
||||
// defer fs.Trace(f, "path=%q", path)("info=%+v, err=%v", &info, &err)
|
||||
leaf, directoryID, err := f.dirCache.FindPath(ctx, path, false)
|
||||
leaf, directoryID, err := f.dirCache.FindRootAndPath(ctx, path, false)
|
||||
if err != nil {
|
||||
if err == fs.ErrorDirNotFound {
|
||||
return nil, fs.ErrorObjectNotFound
|
||||
@@ -663,6 +663,10 @@ OUTER:
|
||||
// This should return ErrDirNotFound if the directory isn't
|
||||
// found.
|
||||
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
|
||||
err = f.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -705,7 +709,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
|
||||
// Used to create new objects
|
||||
func (f *Fs) createObject(ctx context.Context, remote string, modTime time.Time, size int64) (o *Object, leaf string, directoryID string, err error) {
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err = f.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -761,7 +765,13 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
|
||||
|
||||
// Mkdir creates the container if it doesn't exist
|
||||
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
|
||||
_, err := f.dirCache.FindDir(ctx, dir, true)
|
||||
err := f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if dir != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dir, true)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -797,6 +807,10 @@ func (f *Fs) purgeCheck(ctx context.Context, dir string, check bool) error {
|
||||
return errors.New("can't purge root directory")
|
||||
}
|
||||
dc := f.dirCache
|
||||
err := dc.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
directoryID, err := dc.FindDir(ctx, dir, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -1019,14 +1033,64 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
fs.Debugf(srcFs, "Can't move directory - not same remote type")
|
||||
return fs.ErrorCantDirMove
|
||||
}
|
||||
srcPath := path.Join(srcFs.root, srcRemote)
|
||||
dstPath := path.Join(f.root, dstRemote)
|
||||
|
||||
srcID, _, _, dstDirectoryID, dstLeaf, err := f.dirCache.DirMove(ctx, srcFs.dirCache, srcFs.root, srcRemote, f.root, dstRemote)
|
||||
// Refuse to move to or from the root
|
||||
if srcPath == "" || dstPath == "" {
|
||||
fs.Debugf(src, "DirMove error: Can't move root")
|
||||
return errors.New("can't move root directory")
|
||||
}
|
||||
|
||||
// find the root src directory
|
||||
err := srcFs.dirCache.FindRoot(ctx, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// find the root dst directory
|
||||
if dstRemote != "" {
|
||||
err = f.dirCache.FindRoot(ctx, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if f.dirCache.FoundRoot() {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of dst parent, creating subdirs if necessary
|
||||
var leaf, directoryID string
|
||||
findPath := dstRemote
|
||||
if dstRemote == "" {
|
||||
findPath = f.root
|
||||
}
|
||||
leaf, directoryID, err = f.dirCache.FindPath(ctx, findPath, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check destination does not exist
|
||||
if dstRemote != "" {
|
||||
_, err = f.dirCache.FindDir(ctx, dstRemote, false)
|
||||
if err == fs.ErrorDirNotFound {
|
||||
// OK
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
return fs.ErrorDirExists
|
||||
}
|
||||
}
|
||||
|
||||
// Find ID of src
|
||||
srcID, err := srcFs.dirCache.FindDir(ctx, srcRemote, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Do the move
|
||||
err = f.moveDir(ctx, srcID, dstLeaf, dstDirectoryID)
|
||||
err = f.moveDir(ctx, srcID, leaf, directoryID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -1035,7 +1099,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// PublicLink adds a "readable by anyone with link" permission on the given file or folder.
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (string, error) {
|
||||
obj, err := f.NewObject(ctx, remote)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -1225,7 +1289,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
|
||||
remote := o.Remote()
|
||||
|
||||
// Create the directory for the object if it doesn't exist
|
||||
leaf, directoryID, err := o.fs.dirCache.FindPath(ctx, remote, true)
|
||||
leaf, directoryID, err := o.fs.dirCache.FindRootAndPath(ctx, remote, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+10
-19
@@ -447,7 +447,6 @@ func NewFsWithConnection(opt *Options, name, root string, c *swift.Connection, n
|
||||
WriteMimeType: true,
|
||||
BucketBased: true,
|
||||
BucketBasedRootOK: true,
|
||||
SlowModTime: true,
|
||||
}).Fill(f)
|
||||
if f.rootContainer != "" && f.rootDirectory != "" {
|
||||
// Check to see if the object exists - ignoring directory markers
|
||||
@@ -505,15 +504,7 @@ func (f *Fs) newObjectWithInfo(remote string, info *swift.Object) (fs.Object, er
|
||||
// making sure we read the full metadata for all 0 byte files.
|
||||
// We don't read the metadata for directory marker objects.
|
||||
if info != nil && info.Bytes == 0 && info.ContentType != "application/directory" {
|
||||
err := o.readMetaData() // reads info and headers, returning an error
|
||||
if err == fs.ErrorObjectNotFound {
|
||||
// We have a dangling large object here so just return the original metadata
|
||||
fs.Errorf(o, "dangling large object with no contents")
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
return o, nil
|
||||
}
|
||||
info = nil
|
||||
}
|
||||
if info != nil {
|
||||
// Set info but not headers
|
||||
@@ -545,7 +536,7 @@ type listFn func(remote string, object *swift.Object, isDirectory bool) error
|
||||
// container to the start.
|
||||
//
|
||||
// Set recurse to read sub directories
|
||||
func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn listFn) error {
|
||||
func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer bool, recurse bool, fn listFn) error {
|
||||
if prefix != "" && !strings.HasSuffix(prefix, "/") {
|
||||
prefix += "/"
|
||||
}
|
||||
@@ -579,7 +570,7 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
|
||||
fs.Logf(f, "Odd name received %q", remote)
|
||||
continue
|
||||
}
|
||||
if !includeDirMarkers && remote == prefix {
|
||||
if remote == prefix {
|
||||
// If we have zero length directory markers ending in / then swift
|
||||
// will return them in the listing for the directory which causes
|
||||
// duplicate directories. Ignore them here.
|
||||
@@ -602,8 +593,8 @@ func (f *Fs) listContainerRoot(container, directory, prefix string, addContainer
|
||||
type addEntryFn func(fs.DirEntry) error
|
||||
|
||||
// list the objects into the function supplied
|
||||
func (f *Fs) list(container, directory, prefix string, addContainer bool, recurse bool, includeDirMarkers bool, fn addEntryFn) error {
|
||||
err := f.listContainerRoot(container, directory, prefix, addContainer, recurse, includeDirMarkers, func(remote string, object *swift.Object, isDirectory bool) (err error) {
|
||||
func (f *Fs) list(container, directory, prefix string, addContainer bool, recurse bool, fn addEntryFn) error {
|
||||
err := f.listContainerRoot(container, directory, prefix, addContainer, recurse, func(remote string, object *swift.Object, isDirectory bool) (err error) {
|
||||
if isDirectory {
|
||||
remote = strings.TrimRight(remote, "/")
|
||||
d := fs.NewDir(remote, time.Time{}).SetSize(object.Bytes)
|
||||
@@ -615,7 +606,7 @@ func (f *Fs) list(container, directory, prefix string, addContainer bool, recurs
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if includeDirMarkers || o.Storable() {
|
||||
if o.Storable() {
|
||||
err = fn(o)
|
||||
}
|
||||
}
|
||||
@@ -633,7 +624,7 @@ func (f *Fs) listDir(container, directory, prefix string, addContainer bool) (en
|
||||
return nil, fs.ErrorListBucketRequired
|
||||
}
|
||||
// List the objects
|
||||
err = f.list(container, directory, prefix, addContainer, false, false, func(entry fs.DirEntry) error {
|
||||
err = f.list(container, directory, prefix, addContainer, false, func(entry fs.DirEntry) error {
|
||||
entries = append(entries, entry)
|
||||
return nil
|
||||
})
|
||||
@@ -703,7 +694,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
|
||||
container, directory := f.split(dir)
|
||||
list := walk.NewListRHelper(callback)
|
||||
listR := func(container, directory, prefix string, addContainer bool) error {
|
||||
return f.list(container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error {
|
||||
return f.list(container, directory, prefix, addContainer, true, func(entry fs.DirEntry) error {
|
||||
return list.Add(entry)
|
||||
})
|
||||
}
|
||||
@@ -850,7 +841,7 @@ func (f *Fs) Purge(ctx context.Context) error {
|
||||
go func() {
|
||||
delErr <- operations.DeleteFiles(ctx, toBeDeleted)
|
||||
}()
|
||||
err := f.list(f.rootContainer, f.rootDirectory, f.rootDirectory, f.rootContainer == "", true, true, func(entry fs.DirEntry) error {
|
||||
err := f.list(f.rootContainer, f.rootDirectory, f.rootDirectory, f.rootContainer == "", true, func(entry fs.DirEntry) error {
|
||||
if o, ok := entry.(*Object); ok {
|
||||
toBeDeleted <- o
|
||||
}
|
||||
@@ -1112,7 +1103,7 @@ func min(x, y int64) int64 {
|
||||
// if except is passed in then segments with that prefix won't be deleted
|
||||
func (o *Object) removeSegments(except string) error {
|
||||
segmentsContainer, prefix, err := o.getSegmentsDlo()
|
||||
err = o.fs.listContainerRoot(segmentsContainer, prefix, "", false, true, true, func(remote string, object *swift.Object, isDirectory bool) error {
|
||||
err = o.fs.listContainerRoot(segmentsContainer, prefix, "", false, true, func(remote string, object *swift.Object, isDirectory bool) error {
|
||||
if isDirectory {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -73,6 +73,11 @@ func init() {
|
||||
}, {
|
||||
Name: config.ConfigClientSecret,
|
||||
Help: "Yandex Client Secret\nLeave blank normally.",
|
||||
}, {
|
||||
Name: "unlink",
|
||||
Help: "Remove existing public link to file/folder with link command rather than creating.\nDefault is false, meaning link command will create or retrieve public link.",
|
||||
Default: false,
|
||||
Advanced: true,
|
||||
}, {
|
||||
Name: config.ConfigEncoding,
|
||||
Help: config.ConfigEncodingHelp,
|
||||
@@ -87,8 +92,9 @@ func init() {
|
||||
|
||||
// Options defines the configuration for this backend
|
||||
type Options struct {
|
||||
Token string `config:"token"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
Token string `config:"token"`
|
||||
Unlink bool `config:"unlink"`
|
||||
Enc encoder.MultiEncoder `config:"encoding"`
|
||||
}
|
||||
|
||||
// Fs represents a remote yandex
|
||||
@@ -795,9 +801,9 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
|
||||
}
|
||||
|
||||
// PublicLink generates a public link to the remote path (usually readable by anyone)
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (link string, err error) {
|
||||
func (f *Fs) PublicLink(ctx context.Context, remote string) (link string, err error) {
|
||||
var path string
|
||||
if unlink {
|
||||
if f.opt.Unlink {
|
||||
path = "/resources/unpublish"
|
||||
} else {
|
||||
path = "/resources/publish"
|
||||
@@ -824,7 +830,7 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration,
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
if unlink {
|
||||
if f.opt.Unlink {
|
||||
return "", errors.Wrap(err, "couldn't remove public link")
|
||||
}
|
||||
return "", errors.Wrap(err, "couldn't create public link")
|
||||
|
||||
+1
-116
@@ -5,7 +5,6 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
@@ -21,8 +20,6 @@ import (
|
||||
"sync"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-semver/semver"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -40,8 +37,6 @@ var (
|
||||
)
|
||||
|
||||
// GOOS/GOARCH pairs we build for
|
||||
//
|
||||
// If the GOARCH contains a - it is a synthetic arch with more parameters
|
||||
var osarches = []string{
|
||||
"windows/386",
|
||||
"windows/amd64",
|
||||
@@ -50,18 +45,15 @@ var osarches = []string{
|
||||
"linux/386",
|
||||
"linux/amd64",
|
||||
"linux/arm",
|
||||
"linux/arm-v7",
|
||||
"linux/arm64",
|
||||
"linux/mips",
|
||||
"linux/mipsle",
|
||||
"freebsd/386",
|
||||
"freebsd/amd64",
|
||||
"freebsd/arm",
|
||||
"freebsd/arm-v7",
|
||||
"netbsd/386",
|
||||
"netbsd/amd64",
|
||||
"netbsd/arm",
|
||||
"netbsd/arm-v7",
|
||||
"openbsd/386",
|
||||
"openbsd/amd64",
|
||||
"plan9/386",
|
||||
@@ -74,7 +66,6 @@ var archFlags = map[string][]string{
|
||||
"386": {"GO386=387"},
|
||||
"mips": {"GOMIPS=softfloat"},
|
||||
"mipsle": {"GOMIPS=softfloat"},
|
||||
"arm-v7": {"GOARM=7"},
|
||||
}
|
||||
|
||||
// runEnv - run a shell command with env
|
||||
@@ -177,118 +168,12 @@ func buildDebAndRpm(dir, version, goarch string) []string {
|
||||
return artifacts
|
||||
}
|
||||
|
||||
// generate system object (syso) file to be picked up by a following go build for embedding icon and version info resources into windows executable
|
||||
func buildWindowsResourceSyso(goarch string, versionTag string) string {
|
||||
type M map[string]interface{}
|
||||
version := strings.TrimPrefix(versionTag, "v")
|
||||
semanticVersion := semver.New(version)
|
||||
|
||||
// Build json input to goversioninfo utility
|
||||
bs, err := json.Marshal(M{
|
||||
"FixedFileInfo": M{
|
||||
"FileVersion": M{
|
||||
"Major": semanticVersion.Major,
|
||||
"Minor": semanticVersion.Minor,
|
||||
"Patch": semanticVersion.Patch,
|
||||
},
|
||||
"ProductVersion": M{
|
||||
"Major": semanticVersion.Major,
|
||||
"Minor": semanticVersion.Minor,
|
||||
"Patch": semanticVersion.Patch,
|
||||
},
|
||||
},
|
||||
"StringFileInfo": M{
|
||||
"CompanyName": "https://rclone.org",
|
||||
"ProductName": "Rclone",
|
||||
"FileDescription": "Rsync for cloud storage",
|
||||
"InternalName": "rclone",
|
||||
"OriginalFilename": "rclone.exe",
|
||||
"LegalCopyright": "The Rclone Authors",
|
||||
"FileVersion": version,
|
||||
"ProductVersion": version,
|
||||
},
|
||||
"IconPath": "../graphics/logo/ico/logo_symbol_color.ico",
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed to build version info json: %v", err)
|
||||
return ""
|
||||
}
|
||||
|
||||
// Write json to temporary file that will only be used by the goversioninfo command executed below.
|
||||
jsonPath, err := filepath.Abs("versioninfo_windows_" + goarch + ".json") // Appending goos and goarch as suffix to avoid any race conditions
|
||||
if err != nil {
|
||||
log.Printf("Failed to resolve path: %v", err)
|
||||
return ""
|
||||
}
|
||||
err = ioutil.WriteFile(jsonPath, bs, 0644)
|
||||
if err != nil {
|
||||
log.Printf("Failed to write %s: %v", jsonPath, err)
|
||||
return ""
|
||||
}
|
||||
defer func() {
|
||||
if err := os.Remove(jsonPath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", jsonPath, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Execute goversioninfo utility using the json file as input.
|
||||
// It will produce a system object (syso) file that a following go build should pick up.
|
||||
sysoPath, err := filepath.Abs("../resource_windows_" + goarch + ".syso") // Appending goos and goarch as suffix to avoid any race conditions, and also it is recognized by go build and avoids any builds for other systems considering it
|
||||
if err != nil {
|
||||
log.Printf("Failed to resolve path: %v", err)
|
||||
return ""
|
||||
}
|
||||
args := []string{
|
||||
"goversioninfo",
|
||||
"-o",
|
||||
sysoPath,
|
||||
}
|
||||
if goarch == "amd64" {
|
||||
args = append(args, "-64") // Make the syso a 64-bit coff file
|
||||
}
|
||||
args = append(args, jsonPath)
|
||||
err = runEnv(args, nil)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return sysoPath
|
||||
}
|
||||
|
||||
// delete generated system object (syso) resource file
|
||||
func cleanupResourceSyso(sysoFilePath string) {
|
||||
if sysoFilePath == "" {
|
||||
return
|
||||
}
|
||||
if err := os.Remove(sysoFilePath); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
log.Printf("Warning: Couldn't remove generated %s: %v. Please remove it manually.", sysoFilePath, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Trip a version suffix off the arch if present
|
||||
func stripVersion(goarch string) string {
|
||||
i := strings.Index(goarch, "-")
|
||||
if i < 0 {
|
||||
return goarch
|
||||
}
|
||||
return goarch[:i]
|
||||
}
|
||||
|
||||
// build the binary in dir returning success or failure
|
||||
func compileArch(version, goos, goarch, dir string) bool {
|
||||
log.Printf("Compiling %s/%s", goos, goarch)
|
||||
output := filepath.Join(dir, "rclone")
|
||||
if goos == "windows" {
|
||||
output += ".exe"
|
||||
sysoPath := buildWindowsResourceSyso(goarch, version)
|
||||
if sysoPath == "" {
|
||||
log.Printf("Warning: Windows binaries will not have file information embedded")
|
||||
}
|
||||
defer cleanupResourceSyso(sysoPath)
|
||||
}
|
||||
err := os.MkdirAll(dir, 0777)
|
||||
if err != nil {
|
||||
@@ -305,7 +190,7 @@ func compileArch(version, goos, goarch, dir string) bool {
|
||||
}
|
||||
env := []string{
|
||||
"GOOS=" + goos,
|
||||
"GOARCH=" + stripVersion(goarch),
|
||||
"GOARCH=" + goarch,
|
||||
}
|
||||
if !*cgo {
|
||||
env = append(env, "CGO_ENABLED=0")
|
||||
|
||||
@@ -42,10 +42,10 @@ var (
|
||||
// Globals
|
||||
matchProject = regexp.MustCompile(`^([\w-]+)/([\w-]+)$`)
|
||||
osAliases = map[string][]string{
|
||||
"darwin": {"macos", "osx"},
|
||||
"darwin": []string{"macos", "osx"},
|
||||
}
|
||||
archAliases = map[string][]string{
|
||||
"amd64": {"x86_64"},
|
||||
"amd64": []string{"x86_64"},
|
||||
}
|
||||
)
|
||||
|
||||
@@ -374,13 +374,16 @@ func untar(srcFile, fileName, extractDir string) {
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't open output file: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
err := out.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't close output: %v", err)
|
||||
}
|
||||
}()
|
||||
n, err := io.Copy(out, tarReader)
|
||||
if err != nil {
|
||||
log.Fatalf("Couldn't write output file: %v", err)
|
||||
}
|
||||
if err = out.Close(); err != nil {
|
||||
log.Fatalf("Couldn't close output: %v", err)
|
||||
}
|
||||
log.Printf("Wrote %s (%d bytes) as %q", fileName, n, outPath)
|
||||
}
|
||||
}
|
||||
|
||||
+2
-5
@@ -6,7 +6,6 @@ conversion into man pages etc.
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
docpath = "docs/content"
|
||||
@@ -157,19 +156,17 @@ def read_commands(docpath):
|
||||
if command != "rclone.md":
|
||||
docs.append(read_command(command))
|
||||
return "\n".join(docs)
|
||||
|
||||
|
||||
def main():
|
||||
check_docs(docpath)
|
||||
command_docs = read_commands(docpath).replace("\\", "\\\\") # escape \ so we can use command_docs in re.sub
|
||||
build_date = datetime.utcfromtimestamp(
|
||||
int(os.environ.get('SOURCE_DATE_EPOCH', time.time())))
|
||||
with open(outfile, "w") as out:
|
||||
out.write("""\
|
||||
%% rclone(1) User Manual
|
||||
%% Nick Craig-Wood
|
||||
%% %s
|
||||
|
||||
""" % build_date.strftime("%b %d, %Y"))
|
||||
""" % datetime.now().strftime("%b %d, %Y"))
|
||||
for doc in docs:
|
||||
contents = read_doc(doc)
|
||||
# Substitute the commands into doc.md
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
// This shows the commits not yet in the stable branch
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
)
|
||||
|
||||
// version=$(sed <VERSION -e 's/\.[0-9]+*$//g')
|
||||
// echo "Checking version ${version}"
|
||||
// echo
|
||||
//
|
||||
// git log --oneline ${version}.0..${version}-stable | cut -c11- | sort > /tmp/in-stable
|
||||
// git log --oneline ${version}.0..master | cut -c11- | sort > /tmp/in-master
|
||||
//
|
||||
// comm -23 /tmp/in-master /tmp/in-stable
|
||||
|
||||
var logRe = regexp.MustCompile(`^([0-9a-f]{4,}) (.*)$`)
|
||||
|
||||
// run the test passed in with the -run passed in
|
||||
func readCommits(from, to string) (logMap map[string]string, logs []string) {
|
||||
cmd := exec.Command("git", "log", "--oneline", from+".."+to)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
log.Fatalf("failed to run git log: %v", err)
|
||||
}
|
||||
logMap = map[string]string{}
|
||||
logs = []string{}
|
||||
for _, line := range bytes.Split(out, []byte{'\n'}) {
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
match := logRe.FindSubmatch(line)
|
||||
if match == nil {
|
||||
log.Fatalf("failed to parse line: %q", line)
|
||||
}
|
||||
var hash, logMessage = string(match[1]), string(match[2])
|
||||
logMap[logMessage] = hash
|
||||
logs = append(logs, logMessage)
|
||||
}
|
||||
return logMap, logs
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
args := flag.Args()
|
||||
if len(args) != 0 {
|
||||
log.Fatalf("Syntax: %s", os.Args[0])
|
||||
}
|
||||
versionBytes, err := ioutil.ReadFile("VERSION")
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to read version: %v", err)
|
||||
}
|
||||
i := bytes.LastIndexByte(versionBytes, '.')
|
||||
version := string(versionBytes[:i])
|
||||
log.Printf("Finding commits not in stable %s", version)
|
||||
masterMap, masterLogs := readCommits(version+".0", "master")
|
||||
stableMap, _ := readCommits(version+".0", version+"-stable")
|
||||
for _, logMessage := range masterLogs {
|
||||
// Commit found in stable already
|
||||
if _, found := stableMap[logMessage]; found {
|
||||
continue
|
||||
}
|
||||
hash := masterMap[logMessage]
|
||||
fmt.Printf("%s %s\n", hash, logMessage)
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# defaults
|
||||
buildflags=""
|
||||
binary="test.binary"
|
||||
flags=""
|
||||
iterations="100"
|
||||
logprefix="test.out"
|
||||
|
||||
help="
|
||||
This runs go tests repeatedly logging all the failures to separate
|
||||
files. It is very useful for debugging with printf for tests which
|
||||
don't fail very often.
|
||||
|
||||
Syntax: $0 [flags]
|
||||
|
||||
Note that flags for 'go test' need to be expanded, eg '-test.v' instead
|
||||
of just '-v'. '-race' does not need to be expanded.
|
||||
|
||||
Flags this script understands
|
||||
|
||||
-h, --help
|
||||
show this help
|
||||
-i=N, --iterations=N
|
||||
do N iterations (default ${iterations})
|
||||
-b=NAME,--binary=NAME
|
||||
call the output binary NAME (default ${binary})
|
||||
-l=NAME,--logprefix=NAME
|
||||
the log files generated will start with NAME (default ${logprefix})
|
||||
-race
|
||||
build the binary with race testing enabled
|
||||
-tags=TAGS
|
||||
build the binary with the tags supplied
|
||||
|
||||
Any other flags will be past to go test.
|
||||
|
||||
Example
|
||||
|
||||
$0 flags -race -test.run 'TestRWFileHandleOpenTests'
|
||||
|
||||
"
|
||||
|
||||
if [[ "$@" == "" ]]; then
|
||||
echo "${help}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for i in "$@"
|
||||
do
|
||||
case $i in
|
||||
-h|--help)
|
||||
echo "${help}"
|
||||
exit 1
|
||||
;;
|
||||
-b=*|--binary=*)
|
||||
binary="${i#*=}"
|
||||
shift # past argument=value
|
||||
;;
|
||||
-l=*|--log-prefix=*)
|
||||
logprefix="${i#*=}"
|
||||
shift # past argument=value
|
||||
;;
|
||||
-i=*|--iterations=*)
|
||||
iterations="${i#*=}"
|
||||
shift # past argument=value
|
||||
;;
|
||||
-race|--race|-tags=*|--tags=*)
|
||||
buildflags="${buildflags} $i"
|
||||
shift # past argument with no value
|
||||
;;
|
||||
*)
|
||||
# unknown option
|
||||
flags="${flags} ${i#*=}"
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo -n "Compiling ${buildflags} ${binary} ... "
|
||||
go test ${buildflags} -c -o "${binary}" || {
|
||||
echo "build failed"
|
||||
exit 1
|
||||
}
|
||||
echo "OK"
|
||||
|
||||
for i in $(seq -w ${iterations}); do
|
||||
echo -n "Test ${buildflags} ${flags} ${i} "
|
||||
log="${logprefix}${i}.log"
|
||||
./${binary} ${flags} > ${log} 2>&1
|
||||
ok=$?
|
||||
if [[ ${ok} == 0 ]]; then
|
||||
echo "OK"
|
||||
rm ${log}
|
||||
else
|
||||
echo "FAIL - log in ${log}"
|
||||
fi
|
||||
done
|
||||
+1
-4
@@ -14,7 +14,4 @@ else
|
||||
echo "Use '$0 $version delete' to actually delete files"
|
||||
fi
|
||||
|
||||
rclone ${dry_run} -vv -P --checkers 16 --transfers 16 delete \
|
||||
--include "/${version}**" \
|
||||
--include "/branch/${version}**" \
|
||||
memstore:beta-rclone-org
|
||||
rclone ${dry_run} -P --fast-list --checkers 16 --transfers 16 delete --include "**${version}**" memstore:beta-rclone-org
|
||||
+1
-1
@@ -2,5 +2,5 @@
|
||||
echo Setting environment variables for mingw+WinFsp compile
|
||||
set GOPATH=Z:\go
|
||||
rem set PATH=C:\Program Files\mingw-w64\i686-7.1.0-win32-dwarf-rt_v5-rev0\mingw32\bin;%PATH%
|
||||
set PATH=C:\Program Files\mingw-w64\x86_64-8.1.0-win32-seh-rt_v6-rev0\mingw64\bin;%GOPATH%/bin;%PATH%
|
||||
set PATH=C:\Program Files\mingw-w64\x86_64-8.1.0-win32-seh-rt_v6-rev0\mingw64\bin;%PATH%
|
||||
set CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse
|
||||
+11
-116
@@ -2,128 +2,24 @@ package check
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
download = false
|
||||
oneway = false
|
||||
combined = ""
|
||||
missingOnSrc = ""
|
||||
missingOnDst = ""
|
||||
match = ""
|
||||
differ = ""
|
||||
errFile = ""
|
||||
download = false
|
||||
oneway = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
AddFlags(cmdFlags)
|
||||
}
|
||||
|
||||
// AddFlags adds the check flags to the cmdFlags command
|
||||
func AddFlags(cmdFlags *pflag.FlagSet) {
|
||||
flags.BoolVarP(cmdFlags, &download, "download", "", download, "Check by downloading rather than with hash.")
|
||||
flags.BoolVarP(cmdFlags, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on remote")
|
||||
flags.StringVarP(cmdFlags, &combined, "combined", "", combined, "Make a combined report of changes to this file")
|
||||
flags.StringVarP(cmdFlags, &missingOnSrc, "missing-on-src", "", missingOnSrc, "Report all files missing from the source to this file")
|
||||
flags.StringVarP(cmdFlags, &missingOnDst, "missing-on-dst", "", missingOnDst, "Report all files missing from the destination to this file")
|
||||
flags.StringVarP(cmdFlags, &match, "match", "", match, "Report all matching files to this file")
|
||||
flags.StringVarP(cmdFlags, &differ, "differ", "", differ, "Report all non-matching files to this file")
|
||||
flags.StringVarP(cmdFlags, &errFile, "error", "", errFile, "Report all files with errors (hashing or reading) to this file")
|
||||
}
|
||||
|
||||
// FlagsHelp describes the flags for the help
|
||||
var FlagsHelp = strings.Replace(`
|
||||
If you supply the |--one-way| flag, it will only check that files in
|
||||
the source match the files in the destination, not the other way
|
||||
around. This means that extra files in the destination that are not in
|
||||
the source will not be detected.
|
||||
|
||||
The |--differ|, |--missing-on-dst|, |--missing-on-src|, |--src-only|
|
||||
and |--error| flags write paths, one per line, to the file name (or
|
||||
stdout if it is |-|) supplied. What they write is described in the
|
||||
help below. For example |--differ| will write all paths which are
|
||||
present on both the source and destination but different.
|
||||
|
||||
The |--combined| flag will write a file (or stdout) which contains all
|
||||
file paths with a symbol and then a space and then the path to tell
|
||||
you what happened to it. These are reminiscent of diff files.
|
||||
|
||||
- |= path| means path was found in source and destination and was identical
|
||||
- |- path| means path was missing on the source, so only in the destination
|
||||
- |+ path| means path was missing on the destination, so only in the source
|
||||
- |* path| means path was present in source and destination but different.
|
||||
- |! path| means there was an error reading or hashing the source or dest.
|
||||
`, "|", "`", -1)
|
||||
|
||||
// GetCheckOpt gets the options corresponding to the check flags
|
||||
func GetCheckOpt(fsrc, fdst fs.Fs) (opt *operations.CheckOpt, close func(), err error) {
|
||||
closers := []io.Closer{}
|
||||
|
||||
opt = &operations.CheckOpt{
|
||||
Fsrc: fsrc,
|
||||
Fdst: fdst,
|
||||
OneWay: oneway,
|
||||
}
|
||||
|
||||
open := func(name string, pout *io.Writer) error {
|
||||
if name == "" {
|
||||
return nil
|
||||
}
|
||||
if name == "-" {
|
||||
*pout = os.Stdout
|
||||
return nil
|
||||
}
|
||||
out, err := os.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*pout = out
|
||||
closers = append(closers, out)
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = open(combined, &opt.Combined); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = open(missingOnSrc, &opt.MissingOnSrc); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = open(missingOnDst, &opt.MissingOnDst); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = open(match, &opt.Match); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = open(differ, &opt.Differ); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err = open(errFile, &opt.Error); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
close = func() {
|
||||
for _, closer := range closers {
|
||||
err := closer.Close()
|
||||
if err != nil {
|
||||
fs.Errorf(nil, "Failed to close report output: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return opt, close, nil
|
||||
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -141,20 +37,19 @@ If you supply the --download flag, it will download the data from
|
||||
both remotes and check them against each other on the fly. This can
|
||||
be useful for remotes that don't support hashes or if you really want
|
||||
to check all the data.
|
||||
` + FlagsHelp,
|
||||
|
||||
If you supply the --one-way flag, it will only check that files in source
|
||||
match the files in destination, not the other way around. Meaning extra files in
|
||||
destination that are not in the source will not trigger an error.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, fdst := cmd.NewFsSrcDst(args)
|
||||
cmd.Run(false, true, command, func() error {
|
||||
opt, close, err := GetCheckOpt(fsrc, fdst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
cmd.Run(false, false, command, func() error {
|
||||
if download {
|
||||
return operations.CheckDownload(context.Background(), opt)
|
||||
return operations.CheckDownload(context.Background(), fdst, fsrc, oneway)
|
||||
}
|
||||
return operations.Check(context.Background(), opt)
|
||||
return operations.Check(context.Background(), fdst, fsrc, oneway)
|
||||
})
|
||||
},
|
||||
}
|
||||
@@ -11,9 +11,11 @@ package cmount
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/testy"
|
||||
"github.com/rclone/rclone/vfs/vfstest"
|
||||
)
|
||||
|
||||
func TestMount(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
vfstest.RunTests(t, false, mount)
|
||||
}
|
||||
+1
-3
@@ -71,9 +71,7 @@ recently very efficiently like this:
|
||||
|
||||
rclone copy --max-age 24h --no-traverse /path/to/src remote:
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
|
||||
|
||||
**Note**: Use the ` + "`--dry-run` or the `--interactive`/`-i`" + ` flag to test without copying anything.
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
|
||||
@@ -6,17 +6,22 @@ import (
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/backend/crypt"
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/cmd/check"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/hash"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
// Globals
|
||||
var (
|
||||
oneway = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlag := commandDefinition.Flags()
|
||||
check.AddFlags(cmdFlag)
|
||||
flags.BoolVarP(cmdFlag, &oneway, "one-way", "", oneway, "Check one way only, source files must exist on destination")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
@@ -45,7 +50,11 @@ the files in remote:path.
|
||||
rclone cryptcheck remote:path encryptedremote:path
|
||||
|
||||
After it has run it will log the status of the encryptedremote:.
|
||||
` + check.FlagsHelp,
|
||||
|
||||
If you supply the --one-way flag, it will only check that files in source
|
||||
match the files in destination, not the other way around. Meaning extra files in
|
||||
destination that are not in the source will not trigger an error.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(2, 2, command, args)
|
||||
fsrc, fdst := cmd.NewFsSrcDst(args)
|
||||
@@ -70,40 +79,39 @@ func cryptCheck(ctx context.Context, fdst, fsrc fs.Fs) error {
|
||||
}
|
||||
fs.Infof(nil, "Using %v for hash comparisons", hashType)
|
||||
|
||||
opt, close, err := check.GetCheckOpt(fsrc, fcrypt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer close()
|
||||
|
||||
// checkIdentical checks to see if dst and src are identical
|
||||
//
|
||||
// it returns true if differences were found
|
||||
// it also returns whether it couldn't be hashed
|
||||
opt.Check = func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
|
||||
checkIdentical := func(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool) {
|
||||
cryptDst := dst.(*crypt.Object)
|
||||
underlyingDst := cryptDst.UnWrap()
|
||||
underlyingHash, err := underlyingDst.Hash(ctx, hashType)
|
||||
if err != nil {
|
||||
return true, false, errors.Wrapf(err, "error reading hash from underlying %v", underlyingDst)
|
||||
err = fs.CountError(err)
|
||||
fs.Errorf(dst, "Error reading hash from underlying %v: %v", underlyingDst, err)
|
||||
return true, false
|
||||
}
|
||||
if underlyingHash == "" {
|
||||
return false, true, nil
|
||||
return false, true
|
||||
}
|
||||
cryptHash, err := fcrypt.ComputeHash(ctx, cryptDst, src, hashType)
|
||||
if err != nil {
|
||||
return true, false, errors.Wrap(err, "error computing hash")
|
||||
err = fs.CountError(err)
|
||||
fs.Errorf(dst, "Error computing hash: %v", err)
|
||||
return true, false
|
||||
}
|
||||
if cryptHash == "" {
|
||||
return false, true, nil
|
||||
return false, true
|
||||
}
|
||||
if cryptHash != underlyingHash {
|
||||
err = errors.Errorf("hashes differ (%s:%s) %q vs (%s:%s) %q", fdst.Name(), fdst.Root(), cryptHash, fsrc.Name(), fsrc.Root(), underlyingHash)
|
||||
err = fs.CountError(err)
|
||||
fs.Errorf(src, err.Error())
|
||||
return true, false, nil
|
||||
return true, false
|
||||
}
|
||||
return false, false, nil
|
||||
return false, false
|
||||
}
|
||||
|
||||
return operations.CheckFn(ctx, opt)
|
||||
return operations.CheckFn(ctx, fcrypt, fsrc, checkIdentical, oneway)
|
||||
}
|
||||
@@ -34,10 +34,8 @@ merged.
|
||||
|
||||
The ` + "`" + `dedupe` + "`" + ` command will delete all but one of any identical (same
|
||||
md5sum) files it finds without confirmation. This means that for most
|
||||
duplicated files the ` + "`" + `dedupe` + "`" + ` command will not be interactive.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
duplicated files the ` + "`" + `dedupe` + "`" + ` command will not be interactive. You
|
||||
can use ` + "`" + `--dry-run` + "`" + ` to see what would happen without doing anything.
|
||||
|
||||
Here is an example run.
|
||||
|
||||
|
||||
@@ -45,9 +45,6 @@ Then delete
|
||||
|
||||
That reads "delete everything with a minimum size of 100 MB", hence
|
||||
delete all files bigger than 100MBytes.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
|
||||
@@ -88,9 +88,9 @@ func main() {
|
||||
}
|
||||
|
||||
records := [][]string{
|
||||
{"", ""},
|
||||
{"", ""},
|
||||
{"Bytes", "Char"},
|
||||
[]string{"", ""},
|
||||
[]string{"", ""},
|
||||
[]string{"Bytes", "Char"},
|
||||
}
|
||||
for _, r := range remoteNames {
|
||||
records[0] = append(records[0], hRemoteMap[r]...)
|
||||
|
||||
+7
-29
@@ -3,57 +3,35 @@ package link
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/config/flags"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
var (
|
||||
expire = fs.Duration(time.Hour * 24 * 365 * 100)
|
||||
unlink = false
|
||||
)
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
cmdFlags := commandDefinition.Flags()
|
||||
flags.FVarP(cmdFlags, &expire, "expire", "", "The amount of time that the link will be valid")
|
||||
flags.BoolVarP(cmdFlags, &unlink, "unlink", "", unlink, "Remove existing public link to file/folder")
|
||||
}
|
||||
|
||||
var commandDefinition = &cobra.Command{
|
||||
Use: "link remote:path",
|
||||
Short: `Generate public link to file/folder.`,
|
||||
Long: `rclone link will create, retrieve or remove a public link to the given
|
||||
file or folder.
|
||||
Long: `
|
||||
rclone link will create or retrieve a public link to the given file or folder.
|
||||
|
||||
rclone link remote:path/to/file
|
||||
rclone link remote:path/to/folder/
|
||||
rclone link --unlink remote:path/to/folder/
|
||||
rclone link --expire 1d remote:path/to/file
|
||||
|
||||
If you supply the --expire flag, it will set the expiration time
|
||||
otherwise it will use the default (100 years). **Note** not all
|
||||
backends support the --expire flag - if the backend doesn't support it
|
||||
then the link returned won't expire.
|
||||
|
||||
Use the --unlink flag to remove existing public links to the file or
|
||||
folder. **Note** not all backends support "--unlink" flag - those that
|
||||
don't will just ignore it.
|
||||
|
||||
If successful, the last line of the output will contain the
|
||||
link. Exact capabilities depend on the remote, but the link will
|
||||
always by default be created with the least constraints – e.g. no
|
||||
expiry, no password protection, accessible without account.
|
||||
If successful, the last line of the output will contain the link. Exact
|
||||
capabilities depend on the remote, but the link will always be created with
|
||||
the least constraints – e.g. no expiry, no password protection, accessible
|
||||
without account.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fsrc, remote := cmd.NewFsFile(args[0])
|
||||
cmd.Run(false, false, command, func() error {
|
||||
link, err := operations.PublicLink(context.Background(), fsrc, remote, expire, unlink)
|
||||
link, err := operations.PublicLink(context.Background(), fsrc, remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -2,10 +2,8 @@ package mkdir
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/rclone/rclone/cmd"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/operations"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
@@ -20,9 +18,6 @@ var commandDefinition = &cobra.Command{
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
fdst := cmd.NewFsDir(args)
|
||||
if !fdst.Features().CanHaveEmptyDirectories && strings.Contains(fdst.Root(), "/") {
|
||||
fs.Logf(fdst, "Warning: running mkdir on a remote which can't have empty directories does nothing")
|
||||
}
|
||||
cmd.Run(true, false, command, func() error {
|
||||
return operations.Mkdir(context.Background(), fdst, "")
|
||||
})
|
||||
|
||||
@@ -5,9 +5,11 @@ package mount2
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/rclone/rclone/fstest/testy"
|
||||
"github.com/rclone/rclone/vfs/vfstest"
|
||||
)
|
||||
|
||||
func TestMount(t *testing.T) {
|
||||
testy.SkipUnreliable(t)
|
||||
vfstest.RunTests(t, false, mount)
|
||||
}
|
||||
+23
-118
@@ -4,35 +4,27 @@ import (
|
||||
"context"
|
||||
"log"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rclone/rclone/fs"
|
||||
"github.com/rclone/rclone/fs/rc"
|
||||
)
|
||||
|
||||
// MountInfo defines the configuration for a mount
|
||||
type MountInfo struct {
|
||||
unmountFn UnmountFn
|
||||
MountPoint string `json:"MountPoint"`
|
||||
MountedOn time.Time `json:"MountedOn"`
|
||||
Fs string `json:"Fs"`
|
||||
}
|
||||
|
||||
var (
|
||||
// mutex to protect all the variables in this block
|
||||
mountMu sync.Mutex
|
||||
// Mount functions available
|
||||
mountFns = map[string]MountFn{}
|
||||
// Map of mounted path => MountInfo
|
||||
liveMounts = map[string]MountInfo{}
|
||||
mountFns map[string]MountFn
|
||||
// Map of mounted path => unmount function
|
||||
unmountFns map[string]UnmountFn
|
||||
)
|
||||
|
||||
// AddRc adds mount and unmount functionality to rc
|
||||
func AddRc(mountUtilName string, mountFunction MountFn) {
|
||||
mountMu.Lock()
|
||||
defer mountMu.Unlock()
|
||||
if mountFns == nil {
|
||||
mountFns = make(map[string]MountFn)
|
||||
}
|
||||
if unmountFns == nil {
|
||||
unmountFns = make(map[string]UnmountFn)
|
||||
}
|
||||
// rcMount allows the mount command to be run from rc
|
||||
mountFns[mountUtilName] = mountFunction
|
||||
}
|
||||
@@ -62,7 +54,7 @@ Eg
|
||||
})
|
||||
}
|
||||
|
||||
// mountRc allows the mount command to be run from rc
|
||||
// rcMount allows the mount command to be run from rc
|
||||
func mountRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
mountPoint, err := in.GetString("mountPoint")
|
||||
if err != nil {
|
||||
@@ -71,9 +63,6 @@ func mountRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
|
||||
mountType, err := in.GetString("mountType")
|
||||
|
||||
mountMu.Lock()
|
||||
defer mountMu.Unlock()
|
||||
|
||||
if err != nil || mountType == "" {
|
||||
if mountFns["mount"] != nil {
|
||||
mountType = "mount"
|
||||
@@ -91,20 +80,11 @@ func mountRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
}
|
||||
|
||||
if mountFns[mountType] != nil {
|
||||
_, _, unmountFn, err := mountFns[mountType](fdst, mountPoint)
|
||||
|
||||
_, _, unmountFns[mountPoint], err = mountFns[mountType](fdst, mountPoint)
|
||||
if err != nil {
|
||||
log.Printf("mount FAILED: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
// Add mount to list if mount point was successfully created
|
||||
liveMounts[mountPoint] = MountInfo{
|
||||
unmountFn: unmountFn,
|
||||
MountedOn: time.Now(),
|
||||
Fs: fdst.Name(),
|
||||
MountPoint: mountPoint,
|
||||
}
|
||||
|
||||
fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType)
|
||||
return nil, nil
|
||||
}
|
||||
@@ -116,7 +96,7 @@ func init() {
|
||||
Path: "mount/unmount",
|
||||
AuthRequired: true,
|
||||
Fn: unMountRc,
|
||||
Title: "Unmount selected active mount",
|
||||
Title: "Unmount all active mounts",
|
||||
Help: `
|
||||
rclone allows Linux, FreeBSD, macOS and Windows to
|
||||
mount any of Rclone's cloud storage systems as a file system with
|
||||
@@ -133,18 +113,23 @@ Eg
|
||||
})
|
||||
}
|
||||
|
||||
// unMountRc allows the umount command to be run from rc
|
||||
// rcMount allows the umount command to be run from rc
|
||||
func unMountRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
mountPoint, err := in.GetString("mountPoint")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
mountMu.Lock()
|
||||
defer mountMu.Unlock()
|
||||
err = performUnMount(mountPoint)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
if unmountFns != nil && unmountFns[mountPoint] != nil {
|
||||
err := unmountFns[mountPoint]()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
unmountFns[mountPoint] = nil
|
||||
} else {
|
||||
return nil, errors.New("mount not found")
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@@ -173,8 +158,6 @@ Eg
|
||||
// mountTypesRc returns a list of available mount types.
|
||||
func mountTypesRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
var mountTypes = []string{}
|
||||
mountMu.Lock()
|
||||
defer mountMu.Unlock()
|
||||
for mountType := range mountFns {
|
||||
mountTypes = append(mountTypes, mountType)
|
||||
}
|
||||
@@ -183,81 +166,3 @@ func mountTypesRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
"mountTypes": mountTypes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
rc.Add(rc.Call{
|
||||
Path: "mount/listmounts",
|
||||
AuthRequired: true,
|
||||
Fn: listMountsRc,
|
||||
Title: "Show current mount points",
|
||||
Help: `This shows currently mounted points, which can be used for performing an unmount
|
||||
|
||||
This takes no parameters and returns
|
||||
|
||||
- mountPoints: list of current mount points
|
||||
|
||||
Eg
|
||||
|
||||
rclone rc mount/listmounts
|
||||
`,
|
||||
})
|
||||
}
|
||||
|
||||
// listMountsRc returns a list of current mounts
|
||||
func listMountsRc(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
var mountTypes = []MountInfo{}
|
||||
mountMu.Lock()
|
||||
defer mountMu.Unlock()
|
||||
for _, a := range liveMounts {
|
||||
mountTypes = append(mountTypes, a)
|
||||
}
|
||||
return rc.Params{
|
||||
"mountPoints": mountTypes,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
rc.Add(rc.Call{
|
||||
Path: "mount/unmountall",
|
||||
AuthRequired: true,
|
||||
Fn: unmountAll,
|
||||
Title: "Show current mount points",
|
||||
Help: `This shows currently mounted points, which can be used for performing an unmount
|
||||
|
||||
This takes no parameters and returns error if unmount does not succeed.
|
||||
|
||||
Eg
|
||||
|
||||
rclone rc mount/unmountall
|
||||
`,
|
||||
})
|
||||
}
|
||||
|
||||
// unmountAll unmounts all the created mounts
|
||||
func unmountAll(_ context.Context, in rc.Params) (out rc.Params, err error) {
|
||||
mountMu.Lock()
|
||||
defer mountMu.Unlock()
|
||||
for key, mountInfo := range liveMounts {
|
||||
err = performUnMount(mountInfo.MountPoint)
|
||||
if err != nil {
|
||||
fs.Debugf(nil, "Couldn't unmount : %s", key)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// performUnMount unmounts the specified mountPoint
|
||||
func performUnMount(mountPoint string) (err error) {
|
||||
mountInfo, ok := liveMounts[mountPoint]
|
||||
if ok {
|
||||
err := mountInfo.unmountFn()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
delete(liveMounts, mountPoint)
|
||||
} else {
|
||||
return errors.New("mount not found")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+1
-1
@@ -49,7 +49,7 @@ option when moving a small number of files into a large destination
|
||||
can speed transfers up greatly.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
--dry-run flag.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
|
||||
`,
|
||||
|
||||
@@ -44,7 +44,7 @@ modification time or MD5SUM. src will be deleted on successful
|
||||
transfer.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
--dry-run flag.
|
||||
|
||||
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics.
|
||||
`,
|
||||
|
||||
@@ -19,9 +19,6 @@ var commandDefinition = &cobra.Command{
|
||||
Remove the path and all of its contents. Note that this does not obey
|
||||
include/exclude filters - everything will be removed. Use ` + "`" + `delete` + "`" + ` if
|
||||
you want to selectively delete files.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
`,
|
||||
Run: func(command *cobra.Command, args []string) {
|
||||
cmd.CheckArgs(1, 1, command, args)
|
||||
|
||||
@@ -77,12 +77,10 @@ func (cds *contentDirectoryService) cdsObjectToUpnpavObject(cdsObject object, fi
|
||||
}
|
||||
|
||||
if fileInfo.IsDir() {
|
||||
defaultChildCount := 1
|
||||
obj.Class = "object.container.storageFolder"
|
||||
obj.Title = fileInfo.Name()
|
||||
return upnpav.Container{
|
||||
Object: obj,
|
||||
ChildCount: &defaultChildCount,
|
||||
Object: obj,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -10,14 +10,14 @@ import (
|
||||
var Help = `
|
||||
### Server options
|
||||
|
||||
Use ` + "`--addr`" + ` to specify which IP address and port the server should
|
||||
listen on, eg ` + "`--addr 1.2.3.4:8000` or `--addr :8080`" + ` to listen to all
|
||||
Use --addr to specify which IP address and port the server should
|
||||
listen on, eg --addr 1.2.3.4:8000 or --addr :8080 to listen to all
|
||||
IPs.
|
||||
|
||||
Use ` + "`--name`" + ` to choose the friendly server name, which is by
|
||||
Use --name to choose the friendly server name, which is by
|
||||
default "rclone (hostname)".
|
||||
|
||||
Use ` + "`--log-trace` in conjunction with `-vv`" + ` to enable additional debug
|
||||
Use --log-trace in conjunction with -vv to enable additional debug
|
||||
logging of all UPNP traffic.
|
||||
`
|
||||
|
||||
|
||||
+93
-33
@@ -1,15 +1,17 @@
|
||||
// Package ftp implements an FTP server for rclone
|
||||
|
||||
//+build !plan9,go1.13
|
||||
//+build !plan9
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/user"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"sync"
|
||||
|
||||
@@ -26,7 +28,7 @@ import (
|
||||
"github.com/rclone/rclone/vfs/vfsflags"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
ftp "goftp.io/server/core"
|
||||
ftp "goftp.io/server"
|
||||
)
|
||||
|
||||
// Options contains options for the http Server
|
||||
@@ -112,11 +114,13 @@ You can set a single username and password with the --user and --pass flags.
|
||||
|
||||
// server contains everything to run the server
|
||||
type server struct {
|
||||
f fs.Fs
|
||||
srv *ftp.Server
|
||||
opt Options
|
||||
vfs *vfs.VFS
|
||||
proxy *proxy.Proxy
|
||||
f fs.Fs
|
||||
srv *ftp.Server
|
||||
opt Options
|
||||
vfs *vfs.VFS
|
||||
proxy *proxy.Proxy
|
||||
pendingMu sync.Mutex
|
||||
pending map[string]*Driver // pending Driver~s that haven't got their VFS
|
||||
}
|
||||
|
||||
// Make a new FTP to serve the remote
|
||||
@@ -131,8 +135,9 @@ func newServer(f fs.Fs, opt *Options) (*server, error) {
|
||||
}
|
||||
|
||||
s := &server{
|
||||
f: f,
|
||||
opt: *opt,
|
||||
f: f,
|
||||
opt: *opt,
|
||||
pending: make(map[string]*Driver),
|
||||
}
|
||||
if proxyflags.Opt.AuthProxy != "" {
|
||||
s.proxy = proxy.New(&proxyflags.Opt)
|
||||
@@ -146,7 +151,7 @@ func newServer(f fs.Fs, opt *Options) (*server, error) {
|
||||
Factory: s, // implemented by NewDriver method
|
||||
Hostname: host,
|
||||
Port: portNum,
|
||||
PublicIP: opt.PublicIP,
|
||||
PublicIp: opt.PublicIP,
|
||||
PassivePorts: opt.PassivePorts,
|
||||
Auth: s, // implemented by CheckPasswd method
|
||||
Logger: &Logger{},
|
||||
@@ -195,13 +200,78 @@ func (l *Logger) PrintResponse(sessionID string, code int, message string) {
|
||||
fs.Infof(sessionID, "< %d %s", code, message)
|
||||
}
|
||||
|
||||
// CheckPasswd handle auth based on configuration
|
||||
// findID finds the connection ID of the calling program. It does
|
||||
// this in an incredibly hacky way by looking in the stack trace.
|
||||
//
|
||||
// This is not used - the one in Driver should be called instead
|
||||
// callerName should be the name of the function that we are looking
|
||||
// for with a trailing '('
|
||||
//
|
||||
// What is really needed is a change of calling protocol so
|
||||
// CheckPassword is called with the connection.
|
||||
func findID(callerName []byte) (string, error) {
|
||||
// Dump the stack in this format
|
||||
// github.com/rclone/rclone/vendor/goftp.io/server.(*Conn).Serve(0xc0000b2680)
|
||||
// /home/ncw/go/src/github.com/rclone/rclone/vendor/goftp.io/server/conn.go:116 +0x11d
|
||||
buf := make([]byte, 4096)
|
||||
n := runtime.Stack(buf, false)
|
||||
buf = buf[:n]
|
||||
|
||||
// look for callerName first
|
||||
i := bytes.Index(buf, callerName)
|
||||
if i < 0 {
|
||||
return "", errors.Errorf("findID: caller name not found in:\n%s", buf)
|
||||
}
|
||||
buf = buf[i+len(callerName):]
|
||||
|
||||
// find next ')'
|
||||
i = bytes.IndexByte(buf, ')')
|
||||
if i < 0 {
|
||||
return "", errors.Errorf("findID: end of args not found in:\n%s", buf)
|
||||
}
|
||||
buf = buf[:i]
|
||||
|
||||
// trim off first argument
|
||||
// find next ','
|
||||
i = bytes.IndexByte(buf, ',')
|
||||
if i >= 0 {
|
||||
buf = buf[:i]
|
||||
}
|
||||
|
||||
return string(buf), nil
|
||||
}
|
||||
|
||||
var connServeFunction = []byte("(*Conn).Serve(")
|
||||
|
||||
// CheckPasswd handle auth based on configuration
|
||||
func (s *server) CheckPasswd(user, pass string) (ok bool, err error) {
|
||||
err = errors.New("internal error: server.CheckPasswd should never be called")
|
||||
fs.Errorf(nil, "Error: %v", err)
|
||||
return false, err
|
||||
var VFS *vfs.VFS
|
||||
if s.proxy != nil {
|
||||
VFS, _, err = s.proxy.Call(user, pass, false)
|
||||
if err != nil {
|
||||
fs.Infof(nil, "proxy login failed: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
id, err := findID(connServeFunction)
|
||||
if err != nil {
|
||||
fs.Infof(nil, "proxy login failed: failed to read ID from stack: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
s.pendingMu.Lock()
|
||||
d := s.pending[id]
|
||||
delete(s.pending, id)
|
||||
s.pendingMu.Unlock()
|
||||
if d == nil {
|
||||
return false, errors.Errorf("proxy login failed: failed to find pending Driver under ID %q", id)
|
||||
}
|
||||
d.vfs = VFS
|
||||
} else {
|
||||
ok = s.opt.BasicUser == user && (s.opt.BasicPass == "" || s.opt.BasicPass == pass)
|
||||
if !ok {
|
||||
fs.Infof(nil, "login failed: bad credentials")
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// NewDriver starts a new session for each client connection
|
||||
@@ -221,25 +291,15 @@ type Driver struct {
|
||||
lock sync.Mutex
|
||||
}
|
||||
|
||||
// CheckPasswd handle auth based on configuration
|
||||
func (d *Driver) CheckPasswd(user, pass string) (ok bool, err error) {
|
||||
s := d.s
|
||||
if s.proxy != nil {
|
||||
var VFS *vfs.VFS
|
||||
VFS, _, err = s.proxy.Call(user, pass, false)
|
||||
if err != nil {
|
||||
fs.Infof(nil, "proxy login failed: %v", err)
|
||||
return false, nil
|
||||
}
|
||||
d.vfs = VFS
|
||||
} else {
|
||||
ok = s.opt.BasicUser == user && (s.opt.BasicPass == "" || s.opt.BasicPass == pass)
|
||||
if !ok {
|
||||
fs.Infof(nil, "login failed: bad credentials")
|
||||
return false, nil
|
||||
}
|
||||
//Init a connection
|
||||
func (d *Driver) Init(c *ftp.Conn) {
|
||||
defer log.Trace("", "Init session")("")
|
||||
if d.s.proxy != nil {
|
||||
id := fmt.Sprintf("%p", c)
|
||||
d.s.pendingMu.Lock()
|
||||
d.s.pending[id] = d
|
||||
d.s.pendingMu.Unlock()
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
//Stat get information on file or folder
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
//
|
||||
// We skip tests on platforms with troublesome character mappings
|
||||
|
||||
//+build !windows,!darwin,!plan9,go1.13
|
||||
//+build !windows,!darwin,!plan9
|
||||
|
||||
package ftp
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
_ "github.com/rclone/rclone/backend/local"
|
||||
@@ -16,7 +17,8 @@ import (
|
||||
"github.com/rclone/rclone/fs/config/configmap"
|
||||
"github.com/rclone/rclone/fs/config/obscure"
|
||||
"github.com/stretchr/testify/assert"
|
||||
ftp "goftp.io/server/core"
|
||||
"github.com/stretchr/testify/require"
|
||||
ftp "goftp.io/server"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -68,3 +70,10 @@ func TestFTP(t *testing.T) {
|
||||
|
||||
servetest.Run(t, "ftp", start)
|
||||
}
|
||||
|
||||
func TestFindID(t *testing.T) {
|
||||
id, err := findID([]byte("TestFindID("))
|
||||
require.NoError(t, err)
|
||||
// id should be the argument to this function
|
||||
assert.Equal(t, fmt.Sprintf("%p", t), id)
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
// Build for unsupported platforms to stop go complaining
|
||||
// about "no buildable Go source files "
|
||||
|
||||
// +build plan9 !go1.13
|
||||
// +build plan9
|
||||
|
||||
package ftp
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ type StartFn func(f fs.Fs) (configmap.Simple, func())
|
||||
// run runs the server then runs the unit tests for the remote against
|
||||
// it.
|
||||
func run(t *testing.T, name string, start StartFn, useProxy bool) {
|
||||
fstest.Initialise()
|
||||
|
||||
fremote, _, clean, err := fstest.RandomRemote()
|
||||
assert.NoError(t, err)
|
||||
defer clean()
|
||||
@@ -96,7 +98,6 @@ func run(t *testing.T, name string, start StartFn, useProxy bool) {
|
||||
// Run runs the server then runs the unit tests for the remote against
|
||||
// it.
|
||||
func Run(t *testing.T, name string, start StartFn) {
|
||||
fstest.Initialise()
|
||||
t.Run("Normal", func(t *testing.T) {
|
||||
run(t, name, start, false)
|
||||
})
|
||||
|
||||
+1
-3
@@ -30,9 +30,7 @@ modification time or MD5SUM. Destination is updated to match
|
||||
source, including deleting files if necessary.
|
||||
|
||||
**Important**: Since this can cause data loss, test first with the
|
||||
` + "`--dry-run` or the `--interactive`/`-i`" + ` flag.
|
||||
|
||||
rclone sync -i SOURCE remote:DESTINATION
|
||||
` + "`" + `--dry-run` + "`" + ` flag to see exactly what would be copied and deleted.
|
||||
|
||||
Note that files in the destination won't be deleted if there were any
|
||||
errors at any point.
|
||||
|
||||
+2
-8
@@ -19,11 +19,8 @@ var (
|
||||
localTime bool
|
||||
)
|
||||
|
||||
const (
|
||||
defaultLayout string = "060102"
|
||||
layoutDateWithTime = "2006-01-02T15:04:05"
|
||||
layoutDateWithTimeNano = "2006-01-02T15:04:05.999999999"
|
||||
)
|
||||
const defaultLayout string = "060102"
|
||||
const layoutDateWithTime = "2006-01-02T15:04:05"
|
||||
|
||||
func init() {
|
||||
cmd.Root.AddCommand(commandDefinition)
|
||||
@@ -48,7 +45,6 @@ time instead of the current time. Times may be specified as one of:
|
||||
|
||||
- 'YYMMDD' - eg. 17.10.30
|
||||
- 'YYYY-MM-DDTHH:MM:SS' - eg. 2006-01-02T15:04:05
|
||||
- 'YYYY-MM-DDTHH:MM:SS.SSS' - eg. 2006-01-02T15:04:05.123456789
|
||||
|
||||
Note that --timestamp is in UTC if you want local time then add the
|
||||
--localtime flag.
|
||||
@@ -69,8 +65,6 @@ func Touch(ctx context.Context, fsrc fs.Fs, srcFileName string) (err error) {
|
||||
layout := defaultLayout
|
||||
if len(timeAsArgument) == len(layoutDateWithTime) {
|
||||
layout = layoutDateWithTime
|
||||
} else if len(timeAsArgument) > len(layoutDateWithTime) {
|
||||
layout = layoutDateWithTimeNano
|
||||
}
|
||||
var timeAtrFromFlags time.Time
|
||||
if localTime {
|
||||
|
||||
+1
-1
@@ -61,7 +61,7 @@ func init() {
|
||||
flags.BoolVarP(cmdFlags, &opts.DirSort, "dirsfirst", "", false, "List directories before files (-U disables).")
|
||||
flags.StringVarP(cmdFlags, &sort, "sort", "", "", "Select sort: name,version,size,mtime,ctime.")
|
||||
// Graphics
|
||||
flags.BoolVarP(cmdFlags, &opts.NoIndent, "noindent", "", false, "Don't print indentation lines.")
|
||||
flags.BoolVarP(cmdFlags, &opts.NoIndent, "noindent", "i", false, "Don't print indentation lines.")
|
||||
flags.BoolVarP(cmdFlags, &opts.Colorize, "color", "C", false, "Turn colorization on always.")
|
||||
}
|
||||
|
||||
|
||||
@@ -377,21 +377,3 @@ put them back in again.` >}}
|
||||
* Martin Michlmayr <tbm@cyrius.com>
|
||||
* Brandon McNama <bmcnama@pagerduty.com>
|
||||
* Daniel Slyman <github@skylayer.eu>
|
||||
* Alex Guerrero <guerrero@users.noreply.github.com>
|
||||
* Matteo Pietro Dazzi <matteopietro.dazzi@gft.com>
|
||||
* edwardxml <56691903+edwardxml@users.noreply.github.com>
|
||||
* Roman Kredentser <shareed2k@gmail.com>
|
||||
* Kamil Trzciński <ayufan@ayufan.eu>
|
||||
* Zac Rubin <z-0@users.noreply.github.com>
|
||||
* Vincent Feltz <psycho@feltzv.fr>
|
||||
* Heiko Bornholdt <bornholdt@informatik.uni-hamburg.de>
|
||||
* Matteo Pietro Dazzi <matteopietro.dazzi@gmail.com>
|
||||
* jtagcat <gitlab@c7.ee>
|
||||
* Petri Salminen <petri@salminen.dev>
|
||||
* Tim Burke <tim.burke@gmail.com>
|
||||
* Kai Lüke <kai@kinvolk.io>
|
||||
* Garrett Squire <github@garrettsquire.com>
|
||||
* Evan Harris <eharris@puremagic.com>
|
||||
* Kevin <keyam@microsoft.com>
|
||||
* Morten Linderud <morten@linderud.pw>
|
||||
* Dmitry Ustalov <dmitry.ustalov@gmail.com>
|
||||
@@ -65,7 +65,7 @@ List the contents of a container
|
||||
Sync `/home/local/directory` to the remote container, deleting any excess
|
||||
files in the container.
|
||||
|
||||
rclone sync -i /home/local/directory remote:container
|
||||
rclone sync /home/local/directory remote:container
|
||||
|
||||
### --fast-list ###
|
||||
|
||||
|
||||
+1
-7
@@ -70,7 +70,7 @@ List the contents of a bucket
|
||||
Sync `/home/local/directory` to the remote bucket, deleting any
|
||||
excess files in the bucket.
|
||||
|
||||
rclone sync -i /home/local/directory remote:bucket
|
||||
rclone sync /home/local/directory remote:bucket
|
||||
|
||||
### Application Keys ###
|
||||
|
||||
@@ -117,12 +117,6 @@ the following characters are also replaced:
|
||||
Invalid UTF-8 bytes will also be [replaced](/overview/#invalid-utf8),
|
||||
as they can't be used in JSON strings.
|
||||
|
||||
Note that in 2020-05 Backblaze started allowing \ characters in file
|
||||
names. Rclone hasn't changed its encoding as this could cause syncs to
|
||||
re-transfer files. If you want rclone not to replace \ then see the
|
||||
`--b2-encoding` flag below and remove the `BackSlash` from the
|
||||
string. This can be set in the config.
|
||||
|
||||
### SHA1 checksums ###
|
||||
|
||||
The SHA1 checksums of the files are checked on upload and download and
|
||||
|
||||
@@ -226,12 +226,6 @@ normally 8MB so increasing `--transfers` will increase memory use.
|
||||
Depending on the enterprise settings for your user, the item will
|
||||
either be actually deleted from Box or moved to the trash.
|
||||
|
||||
Emptying the trash is supported via the rclone however cleanup command
|
||||
however this deletes every trashed file and folder individually so it
|
||||
may take a very long time.
|
||||
Emptying the trash via the WebUI does not have this limitation
|
||||
so it is advised to empty the trash via the WebUI.
|
||||
|
||||
### Root folder ID ###
|
||||
|
||||
You can set the `root_folder_id` for rclone. This is the directory
|
||||
|
||||
@@ -343,6 +343,8 @@ The username of the Plex user
|
||||
|
||||
The password of the Plex user
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
- Config: plex_password
|
||||
- Env Var: RCLONE_CACHE_PLEX_PASSWORD
|
||||
- Type: string
|
||||
|
||||
@@ -250,7 +250,7 @@ chunk naming scheme is to:
|
||||
- Create another directory (most probably on the same cloud storage)
|
||||
and configure a new remote with desired metadata format,
|
||||
hash type, chunk naming etc.
|
||||
- Now run `rclone sync -i oldchunks: newchunks:` and all your data
|
||||
- Now run `rclone sync oldchunks: newchunks:` and all your data
|
||||
will be transparently converted in transfer.
|
||||
This may take some time, yet chunker will try server-side
|
||||
copy if possible.
|
||||
|
||||
@@ -65,7 +65,7 @@ See the [global flags page](/flags/) for global options not listed here.
|
||||
* [rclone move](/commands/rclone_move/) - Move files from source to dest.
|
||||
* [rclone moveto](/commands/rclone_moveto/) - Move file or directory from source to dest.
|
||||
* [rclone ncdu](/commands/rclone_ncdu/) - Explore a remote with a text based user interface.
|
||||
* [rclone obscure](/commands/rclone_obscure/) - Obscure password for use in the rclone.conf
|
||||
* [rclone obscure](/commands/rclone_obscure/) - Obscure password for use in the rclone config file
|
||||
* [rclone purge](/commands/rclone_purge/) - Remove the path and all of its contents.
|
||||
* [rclone rc](/commands/rclone_rc/) - Run a command against a running rclone.
|
||||
* [rclone rcat](/commands/rclone_rcat/) - Copies standard input to file on remote.
|
||||
|
||||
@@ -1,17 +1,30 @@
|
||||
---
|
||||
title: "rclone obscure"
|
||||
description: "Obscure password for use in the rclone.conf"
|
||||
description: "Obscure password for use in the rclone config file"
|
||||
slug: rclone_obscure
|
||||
url: /commands/rclone_obscure/
|
||||
# autogenerated - DO NOT EDIT, instead edit the source code in cmd/obscure/ and as part of making a release run "make commanddocs"
|
||||
---
|
||||
# rclone obscure
|
||||
|
||||
Obscure password for use in the rclone.conf
|
||||
Obscure password for use in the rclone config file
|
||||
|
||||
## Synopsis
|
||||
|
||||
Obscure password for use in the rclone.conf
|
||||
In the rclone config file, human readable passwords are
|
||||
obscured. Obscuring them is done by encrypting them and writing them
|
||||
out in base64. This is **not** a secure way of encrypting these
|
||||
passwords as rclone can decrypt them - it is to prevent "eyedropping"
|
||||
- namely someone seeing a password in the rclone config file by
|
||||
accident.
|
||||
|
||||
Many equally important things (like access tokens) are not obscured in
|
||||
the config file. However it is very hard to shoulder surf a 64
|
||||
character hex token.
|
||||
|
||||
If you want to encrypt the config file then please use config file
|
||||
encryption - see [rclone config](/commands/rclone_config/) for more
|
||||
info.
|
||||
|
||||
```
|
||||
rclone obscure password [flags]
|
||||
|
||||
@@ -328,6 +328,8 @@ NB If filename_encryption is "off" then this option will do nothing.
|
||||
|
||||
Password or pass phrase for encryption.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
- Config: password
|
||||
- Env Var: RCLONE_CRYPT_PASSWORD
|
||||
- Type: string
|
||||
@@ -338,6 +340,8 @@ Password or pass phrase for encryption.
|
||||
Password or pass phrase for salt. Optional but recommended.
|
||||
Should be different to the previous password.
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
- Config: password2
|
||||
- Env Var: RCLONE_CRYPT_PASSWORD2
|
||||
- Type: string
|
||||
@@ -433,7 +437,7 @@ as `eremote:`.
|
||||
|
||||
To sync the two remotes you would do
|
||||
|
||||
rclone sync -i remote:crypt remote2:crypt
|
||||
rclone sync remote:crypt remote2:crypt
|
||||
|
||||
And to check the integrity you would do
|
||||
|
||||
|
||||
+14
-117
@@ -73,9 +73,6 @@ storage system in the config file then the sub path, eg
|
||||
|
||||
You can define as many storage paths as you like in the config file.
|
||||
|
||||
Please use the [`-i` / `--interactive`](#interactive) flag while
|
||||
learning rclone to avoid accidental data loss.
|
||||
|
||||
Subcommands
|
||||
-----------
|
||||
|
||||
@@ -83,7 +80,7 @@ rclone uses a system of subcommands. For example
|
||||
|
||||
rclone ls remote:path # lists a remote
|
||||
rclone copy /local/path remote:path # copies /local/path to the remote
|
||||
rclone sync -i /local/path remote:path # syncs /local/path to the remote
|
||||
rclone sync /local/path remote:path # syncs /local/path to the remote
|
||||
|
||||
The main rclone commands with most used first
|
||||
|
||||
@@ -203,11 +200,6 @@ To copy files and directories in `https://example.com/path/to/dir` to `/tmp/dir`
|
||||
To copy files and directories from `example.com` in the relative
|
||||
directory `path/to/dir` to `/tmp/dir` using sftp.
|
||||
|
||||
### Valid remote names
|
||||
|
||||
- Remote names may only contain 0-9, A-Z ,a-z ,_ , - and space.
|
||||
- Remote names may not start with -.
|
||||
|
||||
Quoting and the shell
|
||||
---------------------
|
||||
|
||||
@@ -254,11 +246,11 @@ file or directory like this then use the full path starting with a
|
||||
|
||||
So to sync a directory called `sync:me` to a remote called `remote:` use
|
||||
|
||||
rclone sync -i ./sync:me remote:path
|
||||
rclone sync ./sync:me remote:path
|
||||
|
||||
or
|
||||
|
||||
rclone sync -i /full/path/to/sync:me remote:path
|
||||
rclone sync /full/path/to/sync:me remote:path
|
||||
|
||||
Server Side Copy
|
||||
----------------
|
||||
@@ -291,8 +283,8 @@ same.
|
||||
|
||||
This can be used when scripting to make aged backups efficiently, eg
|
||||
|
||||
rclone sync -i remote:current-backup remote:previous-backup
|
||||
rclone sync -i /path/to/files remote:current-backup
|
||||
rclone sync remote:current-backup remote:previous-backup
|
||||
rclone sync /path/to/files remote:current-backup
|
||||
|
||||
Options
|
||||
-------
|
||||
@@ -334,7 +326,7 @@ directory must not overlap the destination directory.
|
||||
|
||||
For example
|
||||
|
||||
rclone sync -i /path/to/local remote:current --backup-dir remote:old
|
||||
rclone sync /path/to/local remote:current --backup-dir remote:old
|
||||
|
||||
will sync `/path/to/local` to `remote:current`, but for any files
|
||||
which would have been updated or deleted will be stored in
|
||||
@@ -609,7 +601,7 @@ Add an HTTP header for all download transactions. The flag can be repeated to
|
||||
add multiple headers.
|
||||
|
||||
```
|
||||
rclone sync -i s3:test/src ~/dst --header-download "X-Amz-Meta-Test: Foo" --header-download "X-Amz-Meta-Test2: Bar"
|
||||
rclone sync s3:test/src ~/dst --header-download "X-Amz-Meta-Test: Foo" --header-download "X-Amz-Meta-Test2: Bar"
|
||||
```
|
||||
|
||||
See the GitHub issue [here](https://github.com/rclone/rclone/issues/59) for
|
||||
@@ -621,7 +613,7 @@ Add an HTTP header for all upload transactions. The flag can be repeated to add
|
||||
multiple headers.
|
||||
|
||||
```
|
||||
rclone sync -i ~/src s3:test/dst --header-upload "Content-Disposition: attachment; filename='cool.html'" --header-upload "X-Amz-Meta-Test: FooBar"
|
||||
rclone sync ~/src s3:test/dst --header-upload "Content-Disposition: attachment; filename='cool.html'" --header-upload "X-Amz-Meta-Test: FooBar"
|
||||
```
|
||||
|
||||
See the GitHub issue [here](https://github.com/rclone/rclone/issues/59) for
|
||||
@@ -694,45 +686,7 @@ This can be useful as an additional layer of protection for immutable
|
||||
or append-only data sets (notably backup archives), where modification
|
||||
implies corruption and should not be propagated.
|
||||
|
||||
### -i / --interactive {#interactive}
|
||||
|
||||
This flag can be used to tell rclone that you wish a manual
|
||||
confirmation before destructive operations.
|
||||
|
||||
It is **recommended** that you use this flag while learning rclone
|
||||
especially with `rclone sync`.
|
||||
|
||||
For example
|
||||
|
||||
```
|
||||
$ rclone delete -i /tmp/dir
|
||||
rclone: delete "important-file.txt"?
|
||||
y) Yes, this is OK (default)
|
||||
n) No, skip this
|
||||
s) Skip all delete operations with no more questions
|
||||
!) Do all delete operations with no more questions
|
||||
q) Exit rclone now.
|
||||
y/n/s/!/q> n
|
||||
```
|
||||
|
||||
The options mean
|
||||
|
||||
- `y`: **Yes**, this operation should go ahead. You can also press Return
|
||||
for this to happen. You'll be asked every time unless you choose `s`
|
||||
or `!`.
|
||||
- `n`: **No**, do not do this operation. You'll be asked every time unless
|
||||
you choose `s` or `!`.
|
||||
- `s`: **Skip** all the following operations of this type with no more
|
||||
questions. This takes effect until rclone exits. If there are any
|
||||
different kind of operations you'll be prompted for them.
|
||||
- `!`: **Do all** the following operations with no more
|
||||
questions. Useful if you've decided that you don't mind rclone doing
|
||||
that kind of operation. This takes effect until rclone exits . If
|
||||
there are any different kind of operations you'll be prompted for
|
||||
them.
|
||||
- `q`: **Quit** rclone now, just in case!
|
||||
|
||||
### --leave-root ####
|
||||
## --leave-root ###
|
||||
|
||||
During rmdirs it will not remove root directory, even if it's empty.
|
||||
|
||||
@@ -1096,41 +1050,6 @@ is fixed all non-ASCII characters will be replaced with `.` when
|
||||
|
||||
This flag will limit rclone's output to error messages only.
|
||||
|
||||
### --refresh-times ###
|
||||
|
||||
The `--refresh-times` flag can be used to update modification times of
|
||||
existing files when they are out of sync on backends which don't
|
||||
support hashes.
|
||||
|
||||
This is useful if you uploaded files with the incorrect timestamps and
|
||||
you now wish to correct them.
|
||||
|
||||
This flag is **only** useful for destinations which don't support
|
||||
hashes (eg `crypt`).
|
||||
|
||||
This can be used any of the sync commands `sync`, `copy` or `move`.
|
||||
|
||||
To use this flag you will need to be doing a modification time sync
|
||||
(so not using `--size-only` or `--checksum`). The flag will have no
|
||||
effect when using `--size-only` or `--checksum`.
|
||||
|
||||
If this flag is used when rclone comes to upload a file it will check
|
||||
to see if there is an existing file on the destination. If this file
|
||||
matches the source with size (and checksum if available) but has a
|
||||
differing timestamp then instead of re-uploading it, rclone will
|
||||
update the timestamp on the destination file. If the checksum does not
|
||||
match rclone will upload the new file. If the checksum is absent (eg
|
||||
on a `crypt` backend) then rclone will update the timestamp.
|
||||
|
||||
Note that some remotes can't set the modification time without
|
||||
re-uploading the file so this flag is less useful on them.
|
||||
|
||||
Normally if you are doing a modification time sync rclone will update
|
||||
modification times without `--refresh-times` provided that the remote
|
||||
supports checksums **and** the checksums match on the file. However if the
|
||||
checksums are absent then rclone will upload the file rather than
|
||||
setting the timestamp as this is the safe behaviour.
|
||||
|
||||
### --retries int ###
|
||||
|
||||
Retry the entire sync if it fails this many times it fails (default 3).
|
||||
@@ -1237,7 +1156,7 @@ or with `--backup-dir`. See `--backup-dir` for more info.
|
||||
|
||||
For example
|
||||
|
||||
rclone sync -i /path/to/local/file remote:current --suffix .bak
|
||||
rclone sync /path/to/local/file remote:current --suffix .bak
|
||||
|
||||
will sync `/path/to/local` to `remote:current`, but for any files
|
||||
which would have been updated or deleted have .bak added.
|
||||
@@ -1849,31 +1768,9 @@ mys3:
|
||||
Note that if you want to create a remote using environment variables
|
||||
you must create the `..._TYPE` variable as above.
|
||||
|
||||
### Precedence
|
||||
|
||||
The various different methods of backend configuration are read in
|
||||
this order and the first one with a value is used.
|
||||
|
||||
- Flag values as supplied on the command line, eg `--drive-use-trash`.
|
||||
- Remote specific environment vars, eg `RCLONE_CONFIG_MYREMOTE_USE_TRASH` (see above).
|
||||
- Backend specific environment vars, eg `RCLONE_DRIVE_USE_TRASH`.
|
||||
- Config file, eg `use_trash = false`.
|
||||
- Default values, eg `true` - these can't be changed.
|
||||
|
||||
So if both `--drive-use-trash` is supplied on the config line and an
|
||||
environment variable `RCLONE_DRIVE_USE_TRASH` is set, the command line
|
||||
flag will take preference.
|
||||
|
||||
For non backend configuration the order is as follows:
|
||||
|
||||
- Flag values as supplied on the command line, eg `--stats 5s`.
|
||||
- Environment vars, eg `RCLONE_STATS=5s`.
|
||||
- Default values, eg `1m` - these can't be changed.
|
||||
|
||||
### Other environment variables ###
|
||||
|
||||
- `RCLONE_CONFIG_PASS` set to contain your config file password (see [Configuration Encryption](#configuration-encryption) section)
|
||||
- `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` (or the lowercase versions thereof).
|
||||
- `HTTPS_PROXY` takes precedence over `HTTP_PROXY` for https requests.
|
||||
- The environment values may be either a complete URL or a "host[:port]" for, in which case the "http" scheme is assumed.
|
||||
- `RCLONE_CONFIG_DIR` - rclone **sets** this variable for use in config files and sub processes to point to the directory holding the config file.
|
||||
* `RCLONE_CONFIG_PASS` set to contain your config file password (see [Configuration Encryption](#configuration-encryption) section)
|
||||
* `HTTP_PROXY`, `HTTPS_PROXY` and `NO_PROXY` (or the lowercase versions thereof).
|
||||
* `HTTPS_PROXY` takes precedence over `HTTP_PROXY` for https requests.
|
||||
* The environment values may be either a complete URL or a "host[:port]" for, in which case the "http" scheme is assumed.
|
||||
@@ -11,8 +11,7 @@ Rclone Download {{< version >}}
|
||||
|:-------:|:-------:|:-----:|:-----:|:----:|:----:|:-------:|:------:|:-------:|:-----:|:-------:|
|
||||
| Intel/AMD - 64 Bit | {{< download windows amd64 >}} | {{< download osx amd64 >}} | {{< download linux amd64 >}} | {{< download linux amd64 deb >}} | {{< download linux amd64 rpm >}} | {{< download freebsd amd64 >}} | {{< download netbsd amd64 >}} | {{< download openbsd amd64 >}} | {{< download plan9 amd64 >}} | {{< download solaris amd64 >}} |
|
||||
| Intel/AMD - 32 Bit | {{< download windows 386 >}} | {{< download osx 386 >}} | {{< download linux 386 >}} | {{< download linux 386 deb >}} | {{< download linux 386 rpm >}} | {{< download freebsd 386 >}} | {{< download netbsd 386 >}} | {{< download openbsd 386 >}} | {{< download plan9 386 >}} | - |
|
||||
| ARMv6 - 32 Bit | - | - | {{< download linux arm >}} | {{< download linux arm deb >}} | {{< download linux arm rpm >}} | {{< download freebsd arm >}} | {{< download netbsd arm >}} | - | - | - |
|
||||
| ARMv7 - 32 Bit | - | - | {{< download linux arm-v7 >}} | {{< download linux arm-v7 deb >}} | {{< download linux arm-v7 rpm >}} | {{< download freebsd arm-v7 >}} | {{< download netbsd arm-v7 >}} | - | - | - |
|
||||
| ARM - 32 Bit | - | - | {{< download linux arm >}} | {{< download linux arm deb >}} | {{< download linux arm rpm >}} | {{< download freebsd arm >}} | {{< download netbsd arm >}} | - | - | - |
|
||||
| ARM - 64 Bit | - | - | {{< download linux arm64 >}} | {{< download linux arm64 deb >}} | {{< download linux arm64 rpm >}} | - | - | - | - | - |
|
||||
| MIPS - Big Endian | - | - | {{< download linux mips >}} | {{< download linux mips deb >}} | {{< download linux mips rpm >}} | - | - | - | - | - |
|
||||
| MIPS - Little Endian | - | - | {{< download linux mipsle >}} | {{< download linux mipsle deb >}} | {{< download linux mipsle rpm >}} | - | - | - | - | - |
|
||||
@@ -71,8 +70,7 @@ script) from a URL which doesn't change then you can use these links.
|
||||
|:-------:|:-------:|:-----:|:-----:|:----:|:----:|:-------:|:------:|:-------:|:-----:|:-------:|
|
||||
| Intel/AMD - 64 Bit | {{< cdownload windows amd64 >}} | {{< cdownload osx amd64 >}} | {{< cdownload linux amd64 >}} | {{< cdownload linux amd64 deb >}} | {{< cdownload linux amd64 rpm >}} | {{< cdownload freebsd amd64 >}} | {{< cdownload netbsd amd64 >}} | {{< cdownload openbsd amd64 >}} | {{< cdownload plan9 amd64 >}} | {{< cdownload solaris amd64 >}} |
|
||||
| Intel/AMD - 32 Bit | {{< cdownload windows 386 >}} | {{< cdownload osx 386 >}} | {{< cdownload linux 386 >}} | {{< cdownload linux 386 deb >}} | {{< cdownload linux 386 rpm >}} | {{< cdownload freebsd 386 >}} | {{< cdownload netbsd 386 >}} | {{< cdownload openbsd 386 >}} | {{< cdownload plan9 386 >}} | - |
|
||||
| ARMv6 - 32 Bit | - | - | {{< cdownload linux arm >}} | {{< cdownload linux arm deb >}} | {{< cdownload linux arm rpm >}} | {{< cdownload freebsd arm >}} | {{< cdownload netbsd arm >}} | - | - | - |
|
||||
| ARMv7 - 32 Bit | - | - | {{< cdownload linux arm-v7 >}} | {{< cdownload linux arm-v7 deb >}} | {{< cdownload linux arm-v7 rpm >}} | {{< cdownload freebsd arm-v7 >}} | {{< cdownload netbsd arm-v7 >}} | - | - | - |
|
||||
| ARM - 32 Bit | - | - | {{< cdownload linux arm >}} | {{< cdownload linux arm deb >}} | {{< cdownload linux arm rpm >}} | {{< cdownload freebsd arm >}} | {{< cdownload netbsd arm >}} | - | - | - |
|
||||
| ARM - 64 Bit | - | - | {{< cdownload linux arm64 >}} | {{< cdownload linux arm64 deb >}} | {{< cdownload linux arm64 rpm >}} | - | - | - | - | - |
|
||||
| MIPS - Big Endian | - | - | {{< cdownload linux mips >}} | {{< cdownload linux mips deb >}} | {{< cdownload linux mips rpm >}} | - | - | - | - | - |
|
||||
| MIPS - Little Endian | - | - | {{< cdownload linux mipsle >}} | {{< cdownload linux mipsle deb >}} | {{< cdownload linux mipsle rpm >}} | - | - | - | - | - |
|
||||
|
||||
@@ -802,6 +802,9 @@ Size of listing chunk 100-1000. 0 to disable.
|
||||
|
||||
Impersonate this user when using a service account.
|
||||
|
||||
Note that if this is used then "root_folder_id" will be ignored.
|
||||
|
||||
|
||||
- Config: impersonate
|
||||
- Env Var: RCLONE_DRIVE_IMPERSONATE
|
||||
- Type: string
|
||||
@@ -1197,10 +1200,3 @@ for rclone to be able to get its token-id (but as this only happens during
|
||||
the remote configuration, it's not such a big deal).
|
||||
|
||||
(Thanks to @balazer on github for these instructions.)
|
||||
|
||||
Sometimes, creation of an OAuth consent in Google API Console fails due to an error message
|
||||
“The request failed because changes to one of the field of the resource is not supported”.
|
||||
As a convenient workaround, the necessary Google Drive API key can be created on the
|
||||
[Python Quickstart](https://developers.google.com/drive/api/v3/quickstart/python) page.
|
||||
Just push the Enable the Drive API button to receive the Client ID and Secret.
|
||||
Note that it will automatically create a new project in the API Console.
|
||||
@@ -197,12 +197,6 @@ of this document](https://www.dropbox.com/en/help/145). Rclone will
|
||||
issue an error message `File name disallowed - not uploading` if it
|
||||
attempts to upload one of those file names, but the sync won't fail.
|
||||
|
||||
Some errors may occur if you try to sync copyright-protected files
|
||||
because Dropbox has its own [copyright detector](https://techcrunch.com/2014/03/30/how-dropbox-knows-when-youre-sharing-copyrighted-stuff-without-actually-looking-at-your-stuff/) that
|
||||
prevents this sort of file being downloaded. This will return the error `ERROR :
|
||||
/path/to/your/file: Failed to copy: failed to open source object:
|
||||
path/restricted_content/.`
|
||||
|
||||
If you have more than 10,000 files in a directory then `rclone purge
|
||||
dropbox:dir` will return the error `Failed to purge: There are too
|
||||
many files involved in this operation`. As a work-around do an
|
||||
|
||||
+3
-3
@@ -34,7 +34,7 @@ The syncs would be incremental (on a file by file basis).
|
||||
|
||||
Eg
|
||||
|
||||
rclone sync -i drive:Folder s3:bucket
|
||||
rclone sync drive:Folder s3:bucket
|
||||
|
||||
|
||||
### Using rclone from multiple locations at the same time ###
|
||||
@@ -43,8 +43,8 @@ You can use rclone from multiple places at the same time if you choose
|
||||
different subdirectory for the output, eg
|
||||
|
||||
```
|
||||
Server A> rclone sync -i /tmp/whatever remote:ServerA
|
||||
Server B> rclone sync -i /tmp/whatever remote:ServerB
|
||||
Server A> rclone sync /tmp/whatever remote:ServerA
|
||||
Server B> rclone sync /tmp/whatever remote:ServerB
|
||||
```
|
||||
|
||||
If you sync to the same directory then you should use rclone copy
|
||||
|
||||
@@ -452,7 +452,7 @@ from the sync on the destination.
|
||||
|
||||
If for example you did a sync from `A` to `B` without the `--min-size 50k` flag
|
||||
|
||||
rclone sync -i A: B:
|
||||
rclone sync A: B:
|
||||
|
||||
Then you repeated it like this with the `--delete-excluded`
|
||||
|
||||
@@ -510,7 +510,7 @@ Imagine, you have the following directory structure:
|
||||
|
||||
You can exclude `dir3` from sync by running the following command:
|
||||
|
||||
rclone sync -i --exclude-if-present .ignore dir1 remote:backup
|
||||
rclone sync --exclude-if-present .ignore dir1 remote:backup
|
||||
|
||||
Currently only one filename is supported, i.e. `--exclude-if-present`
|
||||
should not be used multiple times.
|
||||
+14
-14
@@ -144,7 +144,7 @@ These flags are available for every command.
|
||||
--use-json-log Use json log format.
|
||||
--use-mmap Use mmap allocator (see docs).
|
||||
--use-server-modtime Use server modified time instead of object metadata
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.0")
|
||||
--user-agent string Set the user-agent to a specified string. The default is rclone/ version (default "rclone/v1.52.2")
|
||||
-v, --verbose count Print lots more stuff (repeat for more)
|
||||
```
|
||||
|
||||
@@ -205,7 +205,7 @@ and may be set in the config file.
|
||||
--cache-db-wait-time Duration How long to wait for the DB to be available - 0 is unlimited (default 1s)
|
||||
--cache-info-age Duration How long to cache file structure information (directory listings, file size, times etc). (default 6h0m0s)
|
||||
--cache-plex-insecure string Skip all certificate verification when connecting to the Plex server
|
||||
--cache-plex-password string The password of the Plex user
|
||||
--cache-plex-password string The password of the Plex user (obscured)
|
||||
--cache-plex-url string The URL of the Plex server
|
||||
--cache-plex-username string The username of the Plex user
|
||||
--cache-read-retries int How many times to retry a read from a cache storage. (default 10)
|
||||
@@ -225,8 +225,8 @@ and may be set in the config file.
|
||||
-L, --copy-links Follow symlinks and copy the pointed to item.
|
||||
--crypt-directory-name-encryption Option to either encrypt directory names or leave them intact. (default true)
|
||||
--crypt-filename-encryption string How to encrypt the filenames. (default "standard")
|
||||
--crypt-password string Password or pass phrase for encryption.
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended.
|
||||
--crypt-password string Password or pass phrase for encryption. (obscured)
|
||||
--crypt-password2 string Password or pass phrase for salt. Optional but recommended. (obscured)
|
||||
--crypt-remote string Remote to encrypt/decrypt.
|
||||
--crypt-show-mapping For all files listed show how the names encrypt.
|
||||
--drive-acknowledge-abuse Set to allow files which return cannotDownloadAbusiveFile to be downloaded.
|
||||
@@ -277,7 +277,7 @@ and may be set in the config file.
|
||||
--ftp-encoding MultiEncoder This sets the encoding for the backend. (default Slash,Del,Ctl,RightSpace,Dot)
|
||||
--ftp-host string FTP host to connect to
|
||||
--ftp-no-check-certificate Do not verify the TLS certificate of the server
|
||||
--ftp-pass string FTP password
|
||||
--ftp-pass string FTP password (obscured)
|
||||
--ftp-port string FTP port, leave blank to use default (21)
|
||||
--ftp-tls Use FTP over TLS (Implicit)
|
||||
--ftp-user string FTP username, leave blank for current username, $USER
|
||||
@@ -314,7 +314,7 @@ and may be set in the config file.
|
||||
--koofr-encoding MultiEncoder This sets the encoding for the backend. (default Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--koofr-endpoint string The Koofr API endpoint to use (default "https://app.koofr.net")
|
||||
--koofr-mountid string Mount ID of the mount to use. If omitted, the primary mount is used.
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password)
|
||||
--koofr-password string Your Koofr password for rclone (generate one at https://app.koofr.net/app/admin/preferences/password) (obscured)
|
||||
--koofr-setmtime Does the backend support setting modification time. Set this to false if you use a mount ID that points to a Dropbox or Amazon Drive backend. (default true)
|
||||
--koofr-user string Your Koofr user name
|
||||
-l, --links Translate symlinks to/from regular files with a '.rclonelink' extension
|
||||
@@ -327,7 +327,7 @@ and may be set in the config file.
|
||||
--local-nounc string Disable UNC (long path names) conversion on Windows
|
||||
--mailru-check-hash What should copy do if file checksum is mismatched or invalid (default true)
|
||||
--mailru-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,Del,Ctl,InvalidUtf8,Dot)
|
||||
--mailru-pass string Password
|
||||
--mailru-pass string Password (obscured)
|
||||
--mailru-speedup-enable Skip full upload if there is another file with same data hash. (default true)
|
||||
--mailru-speedup-file-patterns string Comma separated list of file name patterns eligible for speedup (put by hash). (default "*.mkv,*.avi,*.mp4,*.mp3,*.zip,*.gz,*.rar,*.pdf")
|
||||
--mailru-speedup-max-disk SizeSuffix This option allows you to disable speedup (put by hash) for large files (default 3G)
|
||||
@@ -336,7 +336,7 @@ and may be set in the config file.
|
||||
--mega-debug Output more debug from Mega.
|
||||
--mega-encoding MultiEncoder This sets the encoding for the backend. (default Slash,InvalidUtf8,Dot)
|
||||
--mega-hard-delete Delete files permanently rather than putting them into the trash.
|
||||
--mega-pass string Password.
|
||||
--mega-pass string Password. (obscured)
|
||||
--mega-user string User name
|
||||
-x, --one-file-system Don't cross filesystem boundaries (unix/macOS only).
|
||||
--onedrive-chunk-size SizeSuffix Chunk size to upload files with - must be multiple of 320k (327,680 bytes). (default 10M)
|
||||
@@ -349,7 +349,7 @@ and may be set in the config file.
|
||||
--onedrive-server-side-across-configs Allow server side operations (eg copy) to work across different onedrive configs.
|
||||
--opendrive-chunk-size SizeSuffix Files will be uploaded in chunks this size. (default 10M)
|
||||
--opendrive-encoding MultiEncoder This sets the encoding for the backend. (default Slash,LtGt,DoubleQuote,Colon,Question,Asterisk,Pipe,BackSlash,LeftSpace,LeftCrLfHtVt,RightSpace,RightCrLfHtVt,InvalidUtf8,Dot)
|
||||
--opendrive-password string Password.
|
||||
--opendrive-password string Password. (obscured)
|
||||
--opendrive-username string Username
|
||||
--pcloud-client-id string Pcloud App Client Id
|
||||
--pcloud-client-secret string Pcloud App Client Secret
|
||||
@@ -400,19 +400,19 @@ and may be set in the config file.
|
||||
--seafile-create-library Should rclone create a library if it doesn't exist
|
||||
--seafile-encoding MultiEncoder This sets the encoding for the backend. (default Slash,DoubleQuote,BackSlash,Ctl,InvalidUtf8)
|
||||
--seafile-library string Name of the library. Leave blank to access all non-encrypted libraries.
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line.
|
||||
--seafile-pass string Password
|
||||
--seafile-library-key string Library password (for encrypted libraries only). Leave blank if you pass it through the command line. (obscured)
|
||||
--seafile-pass string Password (obscured)
|
||||
--seafile-url string URL of seafile host to connect to
|
||||
--seafile-user string User name (usually email address)
|
||||
--sftp-ask-password Allow asking for SFTP password when needed.
|
||||
--sftp-disable-hashcheck Disable the execution of SSH commands to determine if remote file hashing is available.
|
||||
--sftp-host string SSH host to connect to
|
||||
--sftp-key-file string Path to PEM-encoded private key file, leave blank or set key-use-agent to use ssh-agent.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file.
|
||||
--sftp-key-file-pass string The passphrase to decrypt the PEM-encoded private key file. (obscured)
|
||||
--sftp-key-pem string Raw PEM-encoded private key, If specified, will override key_file parameter.
|
||||
--sftp-key-use-agent When set forces the usage of the ssh-agent.
|
||||
--sftp-md5sum-command string The command used to read md5 hashes. Leave blank for autodetect.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent.
|
||||
--sftp-pass string SSH password, leave blank to use ssh-agent. (obscured)
|
||||
--sftp-path-override string Override path used by SSH connection.
|
||||
--sftp-port string SSH port, leave blank to use default (22)
|
||||
--sftp-set-modtime Set the modified time on the remote if set. (default true)
|
||||
@@ -470,7 +470,7 @@ and may be set in the config file.
|
||||
--union-upstreams string List of space separated upstreams.
|
||||
--webdav-bearer-token string Bearer token instead of user/pass (eg a Macaroon)
|
||||
--webdav-bearer-token-command string Command to run to get a bearer token
|
||||
--webdav-pass string Password.
|
||||
--webdav-pass string Password. (obscured)
|
||||
--webdav-url string URL of http host to connect to
|
||||
--webdav-user string User name
|
||||
--webdav-vendor string Name of the Webdav site/service/software you are using
|
||||
|
||||
+8
-29
@@ -10,10 +10,6 @@ FTP is the File Transfer Protocol. FTP support is provided using the
|
||||
[github.com/jlaffaye/ftp](https://godoc.org/github.com/jlaffaye/ftp)
|
||||
package.
|
||||
|
||||
Paths are specified as `remote:path`. If the path does not begin with
|
||||
a `/` it is relative to the home directory of the user. An empty path
|
||||
`remote:` refers to the user's home directory.
|
||||
|
||||
Here is an example of making an FTP configuration. First run
|
||||
|
||||
rclone config
|
||||
@@ -65,9 +61,6 @@ password:
|
||||
Use FTP over TLS (Implicit)
|
||||
Enter a boolean value (true or false). Press Enter for the default ("false").
|
||||
tls>
|
||||
Use FTP over TLS (Explicit)
|
||||
Enter a boolean value (true or false). Press Enter for the default ("false").
|
||||
explicit_tls>
|
||||
Remote config
|
||||
--------------------
|
||||
[remote]
|
||||
@@ -98,7 +91,7 @@ List the contents of a directory
|
||||
Sync `/home/local/directory` to the remote directory, deleting any
|
||||
excess files in the directory.
|
||||
|
||||
rclone sync -i /home/local/directory remote:directory
|
||||
rclone sync /home/local/directory remote:directory
|
||||
|
||||
### Modified time ###
|
||||
|
||||
@@ -179,6 +172,8 @@ FTP port, leave blank to use default (21)
|
||||
|
||||
FTP password
|
||||
|
||||
**NB** Input to this must be obscured - see [rclone obscure](/commands/rclone_obscure/).
|
||||
|
||||
- Config: pass
|
||||
- Env Var: RCLONE_FTP_PASS
|
||||
- Type: string
|
||||
@@ -188,29 +183,11 @@ FTP password
|
||||
|
||||
Use FTP over TLS (Implicit)
|
||||
|
||||
When using implicit FTP over TLS the client will connect using TLS
|
||||
right from the start, which in turn breaks the compatibility with
|
||||
non-TLS-aware servers. This is usually served over port 990 rather
|
||||
than port 21. Cannot be used in combination with explicit FTP.
|
||||
|
||||
- Config: tls
|
||||
- Env Var: RCLONE_FTP_TLS
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
#### --ftp-explicit-tls
|
||||
|
||||
Use FTP over TLS (Explicit)
|
||||
|
||||
When using explicit FTP over TLS the client explicitly request
|
||||
security from the server in order to upgrade a plain text connection
|
||||
to an encrypted one. Cannot be used in combination with implicit FTP.
|
||||
|
||||
- Config: explicit_tls
|
||||
- Env Var: RCLONE_FTP_TLS
|
||||
- Type: bool
|
||||
- Default: false
|
||||
|
||||
### Advanced Options
|
||||
|
||||
Here are the advanced options specific to ftp (FTP Connection).
|
||||
@@ -257,9 +234,8 @@ See: the [encoding section in the overview](/overview/#encoding) for more info.
|
||||
|
||||
### Limitations ###
|
||||
|
||||
Note that FTP does have its own implementation of : `--dump headers`,
|
||||
`--dump bodies`, `--dump auth` for debugging which isn't the same as
|
||||
the HTTP based backends - it has less fine grained control.
|
||||
Note that since FTP isn't HTTP based the following flags don't work
|
||||
with it: `--dump-headers`, `--dump-bodies`, `--dump-auth`
|
||||
|
||||
Note that `--timeout` isn't supported (but `--contimeout` is).
|
||||
|
||||
@@ -269,3 +245,6 @@ FTP could support server side move but doesn't yet.
|
||||
|
||||
Note that the ftp backend does not support the `ftp_proxy` environment
|
||||
variable yet.
|
||||
|
||||
Note that while implicit FTP over TLS is supported,
|
||||
explicit FTP over TLS is not.
|
||||
@@ -165,7 +165,7 @@ List the contents of a bucket
|
||||
Sync `/home/local/directory` to the remote bucket, deleting any excess
|
||||
files in the bucket.
|
||||
|
||||
rclone sync -i /home/local/directory remote:bucket
|
||||
rclone sync /home/local/directory remote:bucket
|
||||
|
||||
### Service Account support ###
|
||||
|
||||
@@ -194,13 +194,6 @@ the rclone config file, you can set `service_account_credentials` with
|
||||
the actual contents of the file instead, or set the equivalent
|
||||
environment variable.
|
||||
|
||||
### Anonymous Access ###
|
||||
|
||||
For downloads of objects that permit public access you can configure rclone
|
||||
to use anonymous access by setting `anonymous` to `true`.
|
||||
With unauthorized access you can't write or create files but only read or list
|
||||
those buckets and objects that have public read access.
|
||||
|
||||
### Application Default Credentials ###
|
||||
|
||||
If no other source of credentials is provided, rclone will fall back
|
||||
|
||||
@@ -112,7 +112,7 @@ List the contents of an album
|
||||
Sync `/home/local/images` to the Google Photos, removing any excess
|
||||
files in the album.
|
||||
|
||||
rclone sync -i /home/local/image remote:album/newAlbum
|
||||
rclone sync /home/local/image remote:album/newAlbum
|
||||
|
||||
## Layout
|
||||
|
||||
|
||||
@@ -78,7 +78,7 @@ List the contents of a directory
|
||||
|
||||
Sync the remote `directory` to `/home/local/directory`, deleting any excess files.
|
||||
|
||||
rclone sync -i remote:directory /home/local/directory
|
||||
rclone sync remote:directory /home/local/directory
|
||||
|
||||
### Read only ###
|
||||
|
||||
|
||||
Loaded 100 of 2949 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user