Compare commits

...
Author SHA1 Message Date
Nick Craig-Wood dfc5b0460b webdav: add --webdav-base-path flag for unusual servers
See: https://forum.rclone.org/t/how-to-specify-a-webdav-backend-root-path-using-connection-string-syntax/31092
2022-06-08 14:31:25 +01:00
Nick Craig-Wood 26db80c270 ftp: revert to upstream github.com/jlaffaye/ftp from our fork
...now all of our patches have been merged #5810
2022-06-08 11:58:32 +01:00
Nick Craig-Wood 9eb3470c9c Add Matthew Vernon to contributors 2022-06-08 11:58:32 +01:00
Nick Craig-Wood a449dd7d1c Add Jason Zheng to contributors 2022-06-08 11:58:32 +01:00
Nick Craig-Wood fc4fe33703 Add Nick to contributors 2022-06-08 11:58:32 +01:00
Matthew Vernon e11bfacfcf docs: note use of regexp filtering prevents directory optimisation
Discussed in the forum:
https://forum.rclone.org/t/rclone-regex-in-filter-causes-spurious-directory-catch-all-filter/30985

Signed-off-by: Matthew Vernon <mvernon@wikimedia.org>
2022-06-07 17:59:34 +01:00
Jason Zheng a9c49c50a0 ftp: add support for disable_utf8 option - fixes #6209 2022-06-01 19:09:37 +01:00
albertony 8979337313 Add Eric Wolf to contributors 2022-06-01 20:02:25 +02:00
Eric Wolf 7ffab5d998 docs: clarify use of verbosity environment variable (#6208)
RCLONE_VERBOSE does not use a true/false setting, instead using a 0,1,2 setting.
2022-06-01 20:00:43 +02:00
Nick 3ccf222acb sync: overlap check is now filter-sensitive
Previously, the overlap check was based on simple prefix checks of the source and destination paths. Now it actually checks whether the destination is excluded via any filter rule or a "--exclude-if-present"-file.
2022-06-01 18:24:54 +01:00
Nick Craig-Wood 2781f8e2f1 gcs: Fix download of "Content-Encoding: gzip" compressed objects
Before this change, if an object compressed with "Content-Encoding:
gzip" was downloaded, a length and hash mismatch would occur since the
as the go runtime automatically decompressed the object on download.

This change erases the length and hash on compressed objects so they
can be downloaded successfully, at the cost of not being able to check
the length or the hash of the downloaded object.

This also adds the --gcs-download-compressed flag to allow the
compressed files to be downloaded as-is providing compressed objects
with intact size and hash information.

Fixes #2658
2022-05-31 12:10:21 +01:00
Nick Craig-Wood 3d55f69338 dropbox: add logs to show when poll interval limits are exceeded 2022-05-31 12:09:50 +01:00
Nick Craig-Wood cc9bc2cb80 Add Andrey to contributors 2022-05-31 12:09:50 +01:00
Nick Craig-Wood 80ac59ee5b Add Rob Pickerill to contributors 2022-05-31 12:09:50 +01:00
m8rge 5d6a6dd6c0 dropbox: migrate from deprecated api
Change UploadSessionFinishBatch usage to UploadSessionFinishBatchV2. Change in sdk was made in https://github.com/dropbox/dropbox-sdk-go-unofficial/pull/106
2022-05-30 17:24:18 +01:00
Andrey c676e2139d bisync: docs: add Yandex Disk as tested bisync backend 2022-05-30 17:10:25 +01:00
albertony 7361c98b2d Add Mr-Kanister to contributors 2022-05-27 10:34:58 +02:00
Mr-Kanister 5cc47de912 docs/bisync: fixed typo (#6196)
Fixed a little typo in /docs/content/bisync.md
2022-05-27 10:32:25 +02:00
Rob Pickerill 6d342a3c5b azureblob: case insensitive access tier 2022-05-24 09:19:08 +01:00
Nick Craig-Wood 336051870e build: add linux/arm/v6 to docker images - fixes #6192 2022-05-23 18:04:55 +01:00
Nick Craig-Wood 38c6d022bd Add Hugo Laloge to contributors 2022-05-23 18:04:55 +01:00
Hugo Laloge c138367df6 onedrive: Implement --poll-interval for onedrive
Implement ChangeNotifier for onedrive.
Use drive delta queries to listen for modifications.
2022-05-23 11:30:43 +01:00
Nick Craig-Wood da404dc0f2 sync,copy: Fix --fast-list --create-empty-src-dirs and --exclude
Before this change, if --fast-list was in use while doing a sync or
copy with --create-empty-src-dirs and --exclude excluded all the files
from the directory (but not the directory), then the directory would
not be created.

This is also visible with `rclone tree` which uses the same tree
building approach as `rclone sync --fast-list` where the directories
would go missing from the tree view.

This was caused by not adding the parents of excluded files to the
directory tree.

See: https://forum.rclone.org/t/create-empty-src-dirs-issue-with-b2/30856
2022-05-23 10:10:55 +01:00
Nick Craig-Wood 28e43fe7af Add SimonLiu to contributors 2022-05-23 10:10:55 +01:00
SimonLiu 3ec25f437b Update remote_setup.md
Add another option (utilizing SSH Tunnel) for Linux/macOs users to complete the auth on headless box.
2022-05-19 14:13:38 +01:00
Alex JOST a34276e9b3 s3: Add Warsaw location for Scaleway
Add new location in Warsaw (Poland) to endpoints for Scaleway.

More Information:
https://blog.scaleway.com/scaleway-is-now-in-warsaw/
https://www.scaleway.com/en/docs/storage/object/how-to/create-a-bucket/
2022-05-19 14:06:16 +01:00
Nick Craig-Wood c2baacc0a4 union: fix uploading files to union of all bucket based remotes
Before this fix, if uploading to a union consisting of all bucket
based remotes (eg s3), uploads failed with:

    Failed to copy: object not found

This was because the union backend was relying on parent directories
being created to work out which files to upload. If all the upstreams
were bucket based backends which can't hold empty directories, no
directories were created and the upload failed.

This fixes the problem by returning the upstreams used when creating
the directory for the upload, rather than searching for them again
after they've been created.

This will also make the union backend a little more efficient.

Fixes #6170
2022-05-19 13:23:41 +01:00
Nick Craig-Wood fcec4bedbe drive: fix 404 errors on copy/server side copy objects from public folder
Before this change, copying objects from a public folder shared with a
resource key failed with a 404 error.

This is because rclone wasn't supplying the resource Key where it
should have been.

After this change rclone adds the resource Key when trying to download
or server side copy an object.

There may be more places rclone needs to supply the resource key as
this is barely documented in the API documentation.

See: https://forum.rclone.org/t/copying-files-from-a-publicly-accessible-google-drive-folder-added-as-a-shortcut-getting-error-404-server-side-copies-are-disabled/30811
2022-05-18 18:12:19 +01:00
Nick Craig-Wood 813a5e0931 s3: Remove bucket ACL configuration for Cloudflare R2
Bucket ACLs are not supported by Cloudflare R2. All buckets are
private and must be shared using a Cloudflare Worker.
2022-05-17 15:57:09 +01:00
Nick Craig-Wood bd4abb15a3 Add Derek Battams to contributors 2022-05-17 15:57:09 +01:00
Nick Craig-Wood 7f84283539 Add Erik van Velzen to contributors 2022-05-17 15:57:09 +01:00
albertony 47b1a0d6fa docs: improve guide for installing from source - fixes #5809 2022-05-17 12:11:20 +01:00
albertony ce168ecac2 Configurable version suffix independent of version number 2022-05-17 12:10:01 +01:00
Eng Zer Jun 4f0ddb60e7 refactor: replace strings.Replace with strings.ReplaceAll
strings.ReplaceAll(s, old, new) is a wrapper function for
strings.Replace(s, old, new, -1). But strings.ReplaceAll is more
readable and removes the hardcoded -1.

Signed-off-by: Eng Zer Jun <engzerjun@gmail.com>
2022-05-17 11:08:37 +01:00
albertony b929a56f46 Add Christian Galo to contributors 2022-05-16 13:01:55 +02:00
Christian Galo 74af6409d4 docs/rc: added missing global flags (#6083) 2022-05-16 13:00:19 +02:00
albertony 0e77072dcc vfs: error strings should not be capitalized 2022-05-16 12:43:43 +02:00
albertony 2437eb3cce vfs: fix incorrect detection of root in parent directory utility function
When using filepath.Dir, a difference to path.Dir is that it returns os PathSeparator
instead of slash when the path consists entirely of separators.

Also fixed casing of the function name, use OS in all caps instead of Os
as recommended here: https://github.com/golang/go/wiki/CodeReviewComments#initialisms
2022-05-16 12:43:43 +02:00
Michael C Tiernan - MIT-Research Computing Project a12c94caff docs: replaced extended ascii smart quotes (#6171) 2022-05-16 12:34:54 +02:00
Kaspian 542c1616b8 docs: small style fixes 2022-05-13 23:06:45 +01:00
albertony 8697f0bd26 about: improved error message 2022-05-13 12:08:10 +01:00
albertony a9f18f8093 Set proper exit code for errors that are not low-level retried (e.g. size/timestamp changing)
Fixes #5785
2022-05-13 12:02:55 +01:00
Derek Battams 8e5e230b81 b2: use chunksize lib to determine chunksize dynamically
Fixes #4643
2022-05-13 09:25:48 +01:00
Derek Battams c0985e93b7 azureblob: use chunksize lib to determine chunksize dynamically 2022-05-13 09:25:48 +01:00
Derek Battams fb4f7555c7 s3: use chunksize lib to determine chunksize dynamically 2022-05-13 09:25:48 +01:00
Derek Battams f2e7a2e794 chunksize: initial implementation of chunksize helper lib 2022-05-13 09:25:48 +01:00
Erik van Velzen 9e4854955c storj: fix put
The "relative" argument was missing when Put'ing a file. This
sets an incorrect object entry in the cache, leading to the file being
unreadable when using mount functionality.

Fixes #6151
2022-05-12 20:43:54 +01:00
Vincent Murphy 319ac225e4 s3: backend restore command to skip non-GLACIER objects 2022-05-12 20:42:37 +01:00
albertony a9d3283d97 jottacloud: fix listing output of remote with special characters
This fixes the failing integration test: TestIntegration/FsMkdir/FsPutFiles/FsIsFile
2022-05-12 20:41:07 +01:00
Nick Craig-Wood edf0412464 test_all: correct path for Internet Archive test
This is to make it use the rclone project's account and to remove the
/ in the remote name which was crashing the integration tester!
2022-05-12 20:38:05 +01:00
Nick Craig-Wood e6194a4b83 Add Hugal31 to contributors 2022-05-12 20:38:05 +01:00
Nick Craig-Wood 7f05990623 Add Werner to contributors 2022-05-12 20:38:05 +01:00
Nick Craig-Wood e16f2a566f Add Kaspian to contributors 2022-05-12 20:38:05 +01:00
Hugal31 a36fef8a66 rclone.mount: ignore _netdev mount argument - FIxes #5808
Do not trigger an error upon parsing argument starting with underscores.

_netdev was already ignored after parsing.
2022-05-12 20:27:13 +01:00
Werner 6500e1d205 docs: bisync: minor grammar fix 2022-05-12 20:24:52 +01:00
Kaspian 9f7484e4e9 docs: faq: small grammar fix 2022-05-12 20:23:48 +01:00
Michael C Tiernan - MIT-Research Computing Project 0ba702ccf4 install: Pre verify sudo authorization "-v" before calling curl.
The way this was, the curl interferes with the bash invocation. Here it will authenticate first.
2022-05-12 20:19:33 +01:00
Nick Craig-Wood 6f91198b57 s3: Support Cloudflare R2 - fixes #5642 2022-05-12 08:49:20 +01:00
Nick Craig-Wood cf0a72aecd fs: fix FixRangeOption to do nothing on unknown sized objects
FixRangeOption shouldn't be called on an object of unknown size, but
if it is, make sure it does nothing.

See: #5642
2022-05-12 08:49:20 +01:00
Nick Craig-Wood f6fd6ee777 fs: fix FixRangeOption make SeekOptions into absolute RangeOptions
Cloudflare R2 doesn't support range options like `Range: bytes=21-`.

This patch makes FixRangeOption turn a SeekOption into an absolute
RangeOption like this `Range: bytes=21-25` to interoperate with R2.

See: #5642
2022-05-12 08:49:20 +01:00
Nick Craig-Wood 1e66d052fd fs: fix FixRangeOption to make fetch to end Range options absolute
Before this change FixRangeOption was leaving `Range: bytes=21-`
alone, thus not fulfilling its contract of making Range requests
absolute.

As it happens this form isn't supported by Cloudflare R2.

After this change the request is normalised to `Range: bytes=21-25`.

See: #5642
2022-05-12 08:49:20 +01:00
Nick Craig-Wood e5974ac4b0 s3: use PutObject from the aws SDK to upload single part objects
Before this change rclone used presigned requests to upload single
part objects. This was because of a limitation in the SDK which didn't
allow non seekable io.Readers to be passed in.

This is incompatible with some S3 backends, and rclone wasn't adding
the `X-Amz-Content-Sha256: UNSIGNED-PAYLOAD` header which was
incompatible with other S3 backends.

The SDK now allows for this so rclone can use PutObject directly.

This sets the `X-Amz-Content-Sha256: UNSIGNED-PAYLOAD` flag on the PUT
request. However rclone will add a `Content-Md5` header if at all
possible so the body data is still protected.

Note that the old behaviour can still be configured if required with
the `use_presigned_request` config parameter.

Fixes #5422
2022-05-12 08:49:20 +01:00
Nick Craig-Wood 50a0c3482d lib/readers: add FakeSeeker to adapt io.Reader to io.ReadSeeker #5422 2022-05-12 08:49:20 +01:00
Nick Craig-Wood 389a29b017 Add Michael C Tiernan - MIT-Research Computing Project to contributors 2022-05-12 08:49:20 +01:00
Nick Craig-Wood 9dcf9375e8 Add Mateusz Puczyński to contributors 2022-05-12 08:49:20 +01:00
Lesmiscore 1d6d41fb91 backend/internetarchive: fix uploads can take very long time
* fill in empty values for non-wait mode
* add tracking metadata to observe file change
* completely remove getHashes
* remove unreliable update tests

Closes #6150
2022-05-07 21:54:14 +01:00
Michael C Tiernan - MIT-Research Computing Project a3d4307892 install: set the modes on the files and/or directories on macOS
Changes made for macOS specific for that style of system.
Paths are established/defined singularly and modes are set automatically
when created. (Platform specific.)
2022-05-05 17:50:22 +01:00
ehsantdy a446106041 s3: update Arvancloud default values and correct docs 2022-05-02 16:04:01 +01:00
Mateusz Puczyński 607172b6ec go: run go mod tidy and set version to minimum supported go - go1.16
This fixes `go mod tidy` needing manual intervention to build with
previous go versions.
2022-05-01 13:28:20 +01:00
Nick Craig-Wood 94757277bc build: fix go mod tidy removing golang.org/x/mobile dependency
Before this running `go mod tidy` caused the build to break because it
removed the dependency on golang.org/x/mobile and a command line tool
from this package is needed for the build.

This adds an explicit dependency which will mean the tool is always present.
2022-05-01 13:28:20 +01:00
Nick Craig-Wood deab86867c build: support mount on windows/arm64 - all windows binaries now not cgo
This builds all windows binaries without CGO but with cmount.

cgofuse has a compile mode which works without CGO on Windows for
amd64/x86/arm64 architectures so switch to using that.
2022-04-29 18:04:21 +01:00
Nick Craig-Wood c0c5b3bc6b build: add rclone version step 2022-04-29 18:04:21 +01:00
Nick Craig-Wood a947f298e6 cmount: remove cgo from windows build requirements since it builds without 2022-04-29 18:04:21 +01:00
Nick Craig-Wood 1b0128ecb2 cmount: update winfsp/cgofuse dependency for windows/arm64 build 2022-04-29 18:04:21 +01:00
Nick Craig-Wood c5395db1f1 Changelog updates from Version v1.58.1 2022-04-29 13:11:23 +01:00
Nick Craig-Wood 6e5382fc99 Add SwazRGB to contributors 2022-04-29 13:11:23 +01:00
Nick Craig-Wood 134592adaa Add ehsantdy to contributors 2022-04-29 13:11:23 +01:00
Nick Craig-Wood 36e614f550 Add Lesmiscore to contributors 2022-04-29 13:11:23 +01:00
Nick Craig-Wood 7bfed98b48 Add Zsolt Ero to contributors 2022-04-29 13:11:23 +01:00
Nick Craig-Wood f471096fd0 Add Leroy van Logchem to contributors 2022-04-29 13:11:23 +01:00
SwazRGB 4cebade95d b2: Add b2-version-at flag to show file versions at time
Uses b2_list_file_versions to retrieve all file versions, and returns
the one that was active at the specified time

This is especially useful in combination with other backup tools, such
as restic, which may use rclone as a backend.
2022-04-28 16:29:13 +01:00
SwazRGB a8cd18faf3 fs: Implement fs.Time
Similar to fs.Duration but parses into a timestamp instead

Supports parsing from:

* Any of the date formats in parseTimeDates
* A time.Duration offset from now
* parseDurationSuffixes offset from now
2022-04-28 16:29:13 +01:00
ehsantdy e34c543660 s3: Add ArvanCloud AOS to provider list 2022-04-28 10:42:30 +01:00
Lesmiscore 598364ad0f backend/internetarchive: add support for Internet Archive
This adds support for Internet Archive (archive.org) Items.
2022-04-28 10:25:38 +01:00
Zsolt Ero 211dbe9aee docs: Add --multi-thread-streams note to --transfers. 2022-04-27 18:40:39 +01:00
albertony 4829527dac jottacloud: refactor timestamp handling
Jottacloud have several different apis and endpoints using a mix of different
timestamp formats. In existing code the list operation (after the recent liststream
implementation) uses format from golang's time.RFC3339 constant. Uploads (using the
allocate api) uses format from a hard coded constant with value identical to golang's
time.RFC3339. And then we have the classic JFS time format, which is similar to RFC3339
but not identical, using a different constant. Also the naming is a bit confusing,
since the term api is used both as a generic term and also as a reference to the
newer format used in the api subdomain where the allocate endpoint is located.
This commit refactors these things a bit.
2022-04-27 13:41:39 +02:00
albertony cc8dde402f jottacloud: refactor SetModTime function
Now using the utility function for deduplication that was newly implemented to
fix an issue with server-side copy. This function uses the original, and generic,
"jfs" api (and its "cphash" feature), instead of the newer "allocate" api dedicated
for uploads. Both apis support similar deduplication functionaly that we rely on for
the SetModTime operation. One advantage of using the jfs variant is that the allocate
api is specialized for uploads, an initial request performs modtime-only changes and
deduplication if possible but if not possible it creates an incomplete file revision
and returns a special url to be used with a following request to upload missing content.
In the SetModTime function we only sent the first request, using metadata from existing
remote file but different timestamps, which lead to a modtime-only change. If, for some
reason, this should fail it would leave the incomplete revision behind. Probably not
a problem, but the jfs implementation used with this commit is simpler and
a more "standalone" request which either succeeds or fails without expecting additional
requests.
2022-04-27 12:06:36 +02:00
albertony 2b67ad17aa jottacloud: fix issue with server-side copy when destination is in trash
A strange feature (probably bug) in the api used by the server-side copy implementation
in Jottacloud backend is that if the destination file is in trash, the copy request
succeeds but the destination will still be in trash! When this situation occurs in
rclone, the copy command will fail with "Failed to copy: object not found" because
rclone verifies that the file info in the response from the copy request is valid,
and since it is marked as deleted it is treated as invalid.

This commit works around this problem by looking for this situation in the response
from the copy operation, and send an additional request to a built-in deduplication
endpoint that will restore the file from trash.

Fixes #6112
2022-04-27 12:06:36 +02:00
albertony 6da3522499 jottacloud: minor cleanup of upload response
The UploadResponse type included several properties that are no longer returned
by Jottacloud, and the backend implementation did not use them anyway.
2022-04-27 08:40:34 +02:00
albertony 97606bbdef ncdu: refactor accumulated attributes into a struct 2022-04-26 21:12:52 +02:00
albertony a15885dd74 docs/ncdu: document flag prefixes 2022-04-26 21:12:52 +02:00
albertony 87c201c92a ncdu: fix issue where dir size is summed when file sizes are -1
Some backends may not provide size for all objects, and instead
return -1. Existing version included these in directory sums,
with strange results. With this commit rclone ncdu will consider
negative sizes as zero, but add a new prefix flag '~' with a
description that indicates the shown size is inaccurate.

Fixes #6084
2022-04-26 21:12:52 +02:00
albertony d77736c21a docs/size: extend documentation of size command 2022-04-26 19:37:15 +02:00
albertony 86bd5f6922 size: warn about inaccurate results when objects with unknown size 2022-04-26 19:37:15 +02:00
albertony fe271a4e35 docs/drive: mention that google docs count as empty files in directory totals 2022-04-26 19:34:37 +02:00
Leroy van Logchem 75455d4000 azureblob: Calculate Chunksize/blocksize to stay below maxUploadParts 2022-04-26 17:37:40 +01:00
Nick Craig-Wood 82e24f521f webdav: don't override Referer if user sets it - fixes #6040 2022-04-26 08:58:31 +01:00
Nick Craig-Wood 5605e34f7b mount: fix --devname and fusermount: unknown option 'fsname' when mounting via rc
In this commit

f4c40bf79d mount: add --devname to set the device name sent to FUSE for mount display

The --devname parameter was added. However it was soon noticed that
attempting to mount via the rc gave this error:

    mount helper error: fusermount: unknown option 'fsname'
    mount FAILED: fusermount: exit status 1

This was because the DeviceName (and VolumeName) parameter was never
being initialised when the mount was called via the rc.

The fix for this was to refactor the rc interface so it called the
same Mount method as the command line mount which initialised the
DeviceName and VolumeName parameters properly.

This also fixes the cmd/mount tests which were breaking in the same
way but since they aren't normally run on the CI we didn't notice.

Fixes #6044
2022-04-25 12:17:25 +01:00
Nick Craig-Wood 06598531e0 vfs: remove wording which suggests VFS is only for mounting
See: https://forum.rclone.org/t/solved-rclone-serve-read-only/30377/2
2022-04-25 12:17:25 +01:00
albertony b1d43f8d41 jottacloud: fix scope in token request
The existing code in rclone set the value "offline_access+openid",
when encoded in body it will become "offline_access%2Bopenid". I think
this is wrong. Probably an artifact of "double urlencoding" mixup -
either in rclone or in the jottacloud cli tool version it was sniffed
from? It does work, though. The token received will have scopes "email
offline_access" in it, and the same is true if I change to only
sending "offline_access" as scope.

If a proper space delimited list of "offline_access openid"
is used in the request, the response also includes openid scope:
"openid email offline_access". I think this is more correct and this
patch implements this.

See: #6107
2022-04-22 12:52:00 +01:00
Sơn Trần-Nguyễn b53c38c9fd fs/rc/js: correct RC method names 2022-04-22 12:44:04 +01:00
Nick Craig-Wood 03715f6c6b docs: add encoded characters to encoding table 2022-04-21 12:22:04 +01:00
Nick Craig-Wood 07481396e0 lib/encoder: add Semicolon encoding 2022-04-21 12:02:27 +01:00
128 changed files with 34379 additions and 28968 deletions

No files matched your search

+10 -16
View File
@@ -25,7 +25,7 @@ jobs:
strategy:
fail-fast: false
matrix:
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows_amd64', 'windows_386', 'other_os', 'go1.16', 'go1.17']
job_name: ['linux', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.16', 'go1.17']
include:
- job_name: linux
@@ -55,23 +55,12 @@ jobs:
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
deploy: true
- job_name: windows_amd64
- job_name: windows
os: windows-latest
go: '1.18.x'
gotags: cmount
build_flags: '-include "^windows/amd64" -cgo'
build_args: '-buildmode exe'
quicktest: true
racequicktest: true
deploy: true
- job_name: windows_386
os: windows-latest
go: '1.18.x'
gotags: cmount
goarch: '386'
cgo: '1'
build_flags: '-include "^windows/386" -cgo'
cgo: '0'
build_flags: '-include "^windows/"'
build_args: '-buildmode exe'
quicktest: true
deploy: true
@@ -79,7 +68,7 @@ jobs:
- job_name: other_os
os: ubuntu-latest
go: '1.18.x'
build_flags: '-exclude "^(windows/(386|amd64)|darwin/|linux/)"'
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
compile_all: true
deploy: true
@@ -178,6 +167,11 @@ jobs:
run: |
make
- name: Rclone version
shell: bash
run: |
rclone version
- name: Run tests
shell: bash
run: |
@@ -20,7 +20,7 @@ jobs:
with:
tag: beta
imageName: rclone/rclone
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
publish: true
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -28,7 +28,7 @@ jobs:
with:
tag: latest,${{ steps.actual_patch_version.outputs.ACTUAL_PATCH_VERSION }},${{ steps.actual_minor_version.outputs.ACTUAL_MINOR_VERSION }},${{ steps.actual_major_version.outputs.ACTUAL_MAJOR_VERSION }}
imageName: rclone/rclone
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7
platform: linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6
publish: true
dockerHubUser: ${{ secrets.DOCKER_HUB_USER }}
dockerHubPassword: ${{ secrets.DOCKER_HUB_PASSWORD }}
@@ -50,7 +50,7 @@ jobs:
PLUGIN_USER=rclone
docker login --username ${{ secrets.DOCKER_HUB_USER }} \
--password-stdin <<< "${{ secrets.DOCKER_HUB_PASSWORD }}"
for PLUGIN_ARCH in amd64 arm64 arm/v7 ;do
for PLUGIN_ARCH in amd64 arm64 arm/v7 arm/v6 ;do
export PLUGIN_USER PLUGIN_ARCH
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}
make docker-plugin PLUGIN_TAG=${PLUGIN_ARCH/\//-}-${VER#v}
+5 -5
View File
@@ -97,7 +97,7 @@ release_dep_linux:
# Get the release dependencies we only install on Windows
release_dep_windows:
GO111MODULE=off GOOS="" GOARCH="" go get github.com/josephspurrier/goversioninfo/cmd/goversioninfo
GOOS="" GOARCH="" go install github.com/josephspurrier/goversioninfo/cmd/goversioninfo@latest
# Update dependencies
showupdates:
@@ -245,18 +245,18 @@ retag:
startdev:
@echo "Version is $(VERSION)"
@echo "Next version is $(NEXT_VERSION)"
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_VERSION)-DEV\"\n" | gofmt > fs/version.go
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -n "$(NEXT_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_VERSION)" > VERSION
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
git commit -m "Start $(NEXT_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
startstable:
@echo "Version is $(VERSION)"
@echo "Next stable version is $(NEXT_PATCH_VERSION)"
echo -e "package fs\n\n// Version of rclone\nvar Version = \"$(NEXT_PATCH_VERSION)-DEV\"\n" | gofmt > fs/version.go
echo -e "package fs\n\n// VersionTag of rclone\nvar VersionTag = \"$(NEXT_PATCH_VERSION)\"\n" | gofmt > fs/versiontag.go
echo -n "$(NEXT_PATCH_VERSION)" > docs/layouts/partials/version.html
echo "$(NEXT_PATCH_VERSION)" > VERSION
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/version.go VERSION docs/layouts/partials/version.html
git commit -m "Start $(NEXT_PATCH_VERSION)-DEV development" fs/versiontag.go VERSION docs/layouts/partials/version.html
winzip:
zip -9 rclone-$(TAG).zip rclone.exe
+3
View File
@@ -29,6 +29,8 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* Box [:page_facing_up:](https://rclone.org/box/)
* Ceph [:page_facing_up:](https://rclone.org/s3/#ceph)
* China Mobile Ecloud Elastic Object Storage (EOS) [:page_facing_up:](https://rclone.org/s3/#china-mobile-ecloud-eos)
* Cloudflare R2 [:page_facing_up:](https://rclone.org/s3/#cloudflare-r2)
* Arvan Cloud Object Storage (AOS) [:page_facing_up:](https://rclone.org/s3/#arvan-cloud-object-storage-aos)
* Citrix ShareFile [:page_facing_up:](https://rclone.org/sharefile/)
* DigitalOcean Spaces [:page_facing_up:](https://rclone.org/s3/#digitalocean-spaces)
* Digi Storage [:page_facing_up:](https://rclone.org/koofr/#digi-storage)
@@ -42,6 +44,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* HDFS (Hadoop Distributed Filesystem) [:page_facing_up:](https://rclone.org/hdfs/)
* HTTP [:page_facing_up:](https://rclone.org/http/)
* Hubic [:page_facing_up:](https://rclone.org/hubic/)
* Internet Archive [:page_facing_up:](https://rclone.org/internetarchive/)
* Jottacloud [:page_facing_up:](https://rclone.org/jottacloud/)
* IBM COS S3 [:page_facing_up:](https://rclone.org/s3/#ibm-cos-s3)
* Koofr [:page_facing_up:](https://rclone.org/koofr/)
+1
View File
@@ -22,6 +22,7 @@ import (
_ "github.com/rclone/rclone/backend/hdfs"
_ "github.com/rclone/rclone/backend/http"
_ "github.com/rclone/rclone/backend/hubic"
_ "github.com/rclone/rclone/backend/internetarchive"
_ "github.com/rclone/rclone/backend/jottacloud"
_ "github.com/rclone/rclone/backend/koofr"
_ "github.com/rclone/rclone/backend/local"
+17 -12
View File
@@ -26,6 +26,7 @@ import (
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/go-autorest/autorest/adal"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
@@ -43,8 +44,9 @@ import (
const (
minSleep = 10 * time.Millisecond
maxSleep = 10 * time.Second
decayConstant = 1 // bigger for slower decay, exponential
maxListChunkSize = 5000 // number of items to read at once
decayConstant = 1 // bigger for slower decay, exponential
maxListChunkSize = 5000 // number of items to read at once
maxUploadParts = 50000 // maximum allowed number of parts/blocks in a multi-part upload
modTimeKey = "mtime"
timeFormatIn = time.RFC3339
timeFormatOut = "2006-01-02T15:04:05.000000000Z07:00"
@@ -371,15 +373,9 @@ func (o *Object) split() (container, containerPath string) {
// validateAccessTier checks if azureblob supports user supplied tier
func validateAccessTier(tier string) bool {
switch tier {
case string(azblob.AccessTierHot),
string(azblob.AccessTierCool),
string(azblob.AccessTierArchive):
// valid cases
return true
default:
return false
}
return strings.EqualFold(tier, string(azblob.AccessTierHot)) ||
strings.EqualFold(tier, string(azblob.AccessTierCool)) ||
strings.EqualFold(tier, string(azblob.AccessTierArchive))
}
// validatePublicAccess checks if azureblob supports use supplied public access level
@@ -1689,8 +1685,17 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
uploadParts := int64(maxUploadParts)
if uploadParts < 1 {
uploadParts = 1
} else if uploadParts > maxUploadParts {
uploadParts = maxUploadParts
}
// calculate size of parts/blocks
partSize := chunksize.Calculator(o, int(uploadParts), o.fs.opt.ChunkSize)
putBlobOptions := azblob.UploadStreamToBlockBlobOptions{
BufferSize: int(o.fs.opt.ChunkSize),
BufferSize: int(partSize),
MaxBuffers: o.fs.opt.UploadConcurrency,
Metadata: o.meta,
BlobHTTPHeaders: httpHeaders,
+22
View File
@@ -61,3 +61,25 @@ func TestServicePrincipalFileFailure(t *testing.T) {
assert.Error(t, err)
assert.EqualError(t, err, "error creating service principal token: parameter 'secret' cannot be empty")
}
func TestValidateAccessTier(t *testing.T) {
tests := map[string]struct {
accessTier string
want bool
}{
"hot": {"hot", true},
"HOT": {"HOT", true},
"Hot": {"Hot", true},
"cool": {"cool", true},
"archive": {"archive", true},
"empty": {"", false},
"unknown": {"unknown", false},
}
for name, test := range tests {
t.Run(name, func(t *testing.T) {
got := validateAccessTier(test.accessTier)
assert.Equal(t, test.want, got)
})
}
}
+32 -2
View File
@@ -64,7 +64,8 @@ const (
// Globals
var (
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
errNotWithVersions = errors.New("can't modify or delete files in --b2-versions mode")
errNotWithVersionAt = errors.New("can't modify or delete files in --b2-version-at mode")
)
// Register with Fs
@@ -106,6 +107,11 @@ in the [b2 integrations checklist](https://www.backblaze.com/b2/docs/integration
Help: "Include old versions in directory listings.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
Default: false,
Advanced: true,
}, {
Name: "version_at",
Help: "Show file versions as they were at the specified time.\n\nNote that when using this no file write operations are permitted,\nso you can't upload files or delete them.",
Default: fs.Time{},
Advanced: true,
}, {
Name: "hard_delete",
Help: "Permanently delete files on remote removal, otherwise hide files.",
@@ -211,6 +217,7 @@ type Options struct {
Endpoint string `config:"endpoint"`
TestMode string `config:"test_mode"`
Versions bool `config:"versions"`
VersionAt fs.Time `config:"version_at"`
HardDelete bool `config:"hard_delete"`
UploadCutoff fs.SizeSuffix `config:"upload_cutoff"`
CopyCutoff fs.SizeSuffix `config:"copy_cutoff"`
@@ -696,9 +703,12 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
Method: "POST",
Path: "/b2_list_file_names",
}
if hidden {
if hidden || f.opt.VersionAt.IsSet() {
opts.Path = "/b2_list_file_versions"
}
lastFileName := ""
for {
var response api.ListFileNamesResponse
err := f.pacer.Call(func() (bool, error) {
@@ -728,7 +738,21 @@ func (f *Fs) list(ctx context.Context, bucket, directory, prefix string, addBuck
if addBucket {
remote = path.Join(bucket, remote)
}
if f.opt.VersionAt.IsSet() {
if time.Time(file.UploadTimestamp).After(time.Time(f.opt.VersionAt)) {
// Ignore versions that were created after the specified time
continue
}
if file.Name == lastFileName {
// Ignore versions before the already returned version
continue
}
}
// Send object
lastFileName = file.Name
err = fn(remote, file, isDirectory)
if err != nil {
if err == errEndList {
@@ -1828,6 +1852,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
if o.fs.opt.Versions {
return errNotWithVersions
}
if o.fs.opt.VersionAt.IsSet() {
return errNotWithVersionAt
}
size := src.Size()
bucket, bucketPath := o.split()
@@ -1983,6 +2010,9 @@ func (o *Object) Remove(ctx context.Context) error {
if o.fs.opt.Versions {
return errNotWithVersions
}
if o.fs.opt.VersionAt.IsSet() {
return errNotWithVersionAt
}
if o.fs.opt.HardDelete {
return o.fs.deleteByID(ctx, o.id, bucketPath)
}
+4 -5
View File
@@ -18,6 +18,7 @@ import (
"github.com/rclone/rclone/backend/b2/api"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/rest"
@@ -88,21 +89,19 @@ type largeUpload struct {
// newLargeUpload starts an upload of object o from in with metadata in src
//
// If newInfo is set then metadata from that will be used instead of reading it from src
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, chunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
remote := o.remote
func (f *Fs) newLargeUpload(ctx context.Context, o *Object, in io.Reader, src fs.ObjectInfo, defaultChunkSize fs.SizeSuffix, doCopy bool, newInfo *api.File) (up *largeUpload, err error) {
size := src.Size()
parts := int64(0)
sha1SliceSize := int64(maxParts)
chunkSize := defaultChunkSize
if size == -1 {
fs.Debugf(o, "Streaming upload with --b2-chunk-size %s allows uploads of up to %s and will fail only when that limit is reached.", f.opt.ChunkSize, maxParts*f.opt.ChunkSize)
} else {
chunkSize = chunksize.Calculator(src, maxParts, defaultChunkSize)
parts = size / int64(chunkSize)
if size%int64(chunkSize) != 0 {
parts++
}
if parts > maxParts {
return nil, fmt.Errorf("%q too big (%d bytes) makes too many parts %d > %d - increase --b2-chunk-size", remote, size, parts, maxParts)
}
sha1SliceSize = parts
}
+1 -1
View File
@@ -1747,7 +1747,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("About not supported")
return nil, errors.New("not supported by underlying remote")
}
return do(ctx)
}
+4 -4
View File
@@ -515,7 +515,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
strRegex := regexp.QuoteMeta(pattern)
strRegex = reHashes.ReplaceAllLiteralString(strRegex, reDataOrCtrl)
strRegex = strings.Replace(strRegex, "\\*", mainNameRegStr, -1)
strRegex = strings.ReplaceAll(strRegex, "\\*", mainNameRegStr)
strRegex = fmt.Sprintf("^%s(?:%s|%s)?$", strRegex, tempSuffixRegStr, tempSuffixRegOld)
f.nameRegexp = regexp.MustCompile(strRegex)
@@ -524,7 +524,7 @@ func (f *Fs) setChunkNameFormat(pattern string) error {
if numDigits > 1 {
fmtDigits = fmt.Sprintf("%%0%dd", numDigits)
}
strFmt := strings.Replace(pattern, "%", "%%", -1)
strFmt := strings.ReplaceAll(pattern, "%", "%%")
strFmt = strings.Replace(strFmt, "*", "%s", 1)
f.dataNameFmt = reHashes.ReplaceAllLiteralString(strFmt, fmtDigits)
f.ctrlNameFmt = reHashes.ReplaceAllLiteralString(strFmt, "_%s")
@@ -1895,7 +1895,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
func (f *Fs) CleanUp(ctx context.Context) error {
do := f.base.Features().CleanUp
if do == nil {
return errors.New("can't CleanUp")
return errors.New("not supported by underlying remote")
}
return do(ctx)
}
@@ -1904,7 +1904,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.base.Features().About
if do == nil {
return nil, errors.New("About not supported")
return nil, errors.New("not supported by underlying remote")
}
return do(ctx)
}
+2 -2
View File
@@ -906,7 +906,7 @@ func (f *Fs) DirMove(ctx context.Context, src fs.Fs, srcRemote, dstRemote string
func (f *Fs) CleanUp(ctx context.Context) error {
do := f.Fs.Features().CleanUp
if do == nil {
return errors.New("can't CleanUp: not supported by underlying remote")
return errors.New("not supported by underlying remote")
}
return do(ctx)
}
@@ -915,7 +915,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("can't About: not supported by underlying remote")
return nil, errors.New("not supported by underlying remote")
}
return do(ctx)
}
+2 -2
View File
@@ -597,7 +597,7 @@ func (f *Fs) PutUnchecked(ctx context.Context, in io.Reader, src fs.ObjectInfo,
func (f *Fs) CleanUp(ctx context.Context) error {
do := f.Fs.Features().CleanUp
if do == nil {
return errors.New("can't CleanUp")
return errors.New("not supported by underlying remote")
}
return do(ctx)
}
@@ -606,7 +606,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("About not supported")
return nil, errors.New("not supported by underlying remote")
}
return do(ctx)
}
+22 -7
View File
@@ -70,7 +70,7 @@ const (
// 1<<18 is the minimum size supported by the Google uploader, and there is no maximum.
minChunkSize = fs.SizeSuffix(googleapi.MinUploadChunkSize)
defaultChunkSize = 8 * fs.Mebi
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks"
partialFields = "id,name,size,md5Checksum,trashed,explicitlyTrashed,modifiedTime,createdTime,mimeType,parents,webViewLink,shortcutDetails,exportLinks,resourceKey"
listRGrouping = 50 // number of IDs to search at once when using ListR
listRInputBuffer = 1000 // size of input buffer when using ListR
defaultXDGIcon = "text-html"
@@ -660,6 +660,7 @@ type baseObject struct {
mimeType string // The object MIME type
bytes int64 // size of the object
parents []string // IDs of the parent directories
resourceKey *string // resourceKey is needed for link shared objects
}
type documentObject struct {
baseObject
@@ -829,8 +830,8 @@ func (f *Fs) list(ctx context.Context, dirIDs []string, title string, directorie
if title != "" {
searchTitle := f.opt.Enc.FromStandardName(title)
// Escaping the backslash isn't documented but seems to work
searchTitle = strings.Replace(searchTitle, `\`, `\\`, -1)
searchTitle = strings.Replace(searchTitle, `'`, `\'`, -1)
searchTitle = strings.ReplaceAll(searchTitle, `\`, `\\`)
searchTitle = strings.ReplaceAll(searchTitle, `'`, `\'`)
var titleQuery bytes.Buffer
_, _ = fmt.Fprintf(&titleQuery, "(name='%s'", searchTitle)
@@ -1319,12 +1320,16 @@ func (f *Fs) newRegularObject(remote string, info *drive.File) fs.Object {
}
}
}
return &Object{
o := &Object{
baseObject: f.newBaseObject(remote, info),
url: fmt.Sprintf("%sfiles/%s?alt=media", f.svc.BasePath, actualID(info.Id)),
md5sum: strings.ToLower(info.Md5Checksum),
v2Download: f.opt.V2DownloadMinSize != -1 && info.Size >= int64(f.opt.V2DownloadMinSize),
}
if info.ResourceKey != "" {
o.resourceKey = &info.ResourceKey
}
return o
}
// newDocumentObject creates an fs.Object for a google docs drive.File
@@ -2429,11 +2434,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
var info *drive.File
err = f.pacer.Call(func() (bool, error) {
info, err = f.svc.Files.Copy(id, createInfo).
copy := f.svc.Files.Copy(id, createInfo).
Fields(partialFields).
SupportsAllDrives(true).
KeepRevisionForever(f.opt.KeepRevisionForever).
Context(ctx).Do()
KeepRevisionForever(f.opt.KeepRevisionForever)
srcObj.addResourceKey(copy.Header())
info, err = copy.Context(ctx).Do()
return f.shouldRetry(ctx, err)
})
if err != nil {
@@ -3530,6 +3536,14 @@ func (o *baseObject) Storable() bool {
return true
}
// addResourceKey adds a X-Goog-Drive-Resource-Keys header for this
// object if required.
func (o *baseObject) addResourceKey(header http.Header) {
if o.resourceKey != nil {
header.Add("X-Goog-Drive-Resource-Keys", fmt.Sprintf("%s/%s", o.id, *o.resourceKey))
}
}
// httpResponse gets an http.Response object for the object
// using the url and method passed in
func (o *baseObject) httpResponse(ctx context.Context, url, method string, options []fs.OpenOption) (req *http.Request, res *http.Response, err error) {
@@ -3545,6 +3559,7 @@ func (o *baseObject) httpResponse(ctx context.Context, url, method string, optio
// Don't supply range requests for 0 length objects as they always fail
delete(req.Header, "Range")
}
o.addResourceKey(req.Header)
err = o.fs.pacer.Call(func() (bool, error) {
res, err = o.fs.client.Do(req)
if err == nil {
+4 -19
View File
@@ -118,12 +118,12 @@ func (b *batcher) Batching() bool {
}
// finishBatch commits the batch, returning a batch status to poll or maybe complete
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (batchStatus *files.UploadSessionFinishBatchLaunch, err error) {
func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionFinishArg) (complete *files.UploadSessionFinishBatchResult, err error) {
var arg = &files.UploadSessionFinishBatchArg{
Entries: items,
}
err = b.f.pacer.Call(func() (bool, error) {
batchStatus, err = b.f.srv.UploadSessionFinishBatch(arg)
complete, err = b.f.srv.UploadSessionFinishBatchV2(arg)
// If error is insufficient space then don't retry
if e, ok := err.(files.UploadSessionFinishAPIError); ok {
if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.WriteErrorInsufficientSpace {
@@ -137,7 +137,7 @@ func (b *batcher) finishBatch(ctx context.Context, items []*files.UploadSessionF
if err != nil {
return nil, fmt.Errorf("batch commit failed: %w", err)
}
return batchStatus, nil
return complete, nil
}
// finishBatchJobStatus waits for the batch to complete returning completed entries
@@ -199,26 +199,11 @@ func (b *batcher) commitBatch(ctx context.Context, items []*files.UploadSessionF
fs.Debugf(b.f, "Committing %s", desc)
// finalise the batch getting either a result or a job id to poll
batchStatus, err := b.finishBatch(ctx, items)
complete, err := b.finishBatch(ctx, items)
if err != nil {
return err
}
// check whether batch is complete
var complete *files.UploadSessionFinishBatchResult
switch batchStatus.Tag {
case "async_job_id":
// wait for batch to complete
complete, err = b.finishBatchJobStatus(ctx, batchStatus)
if err != nil {
return err
}
case "complete":
complete = batchStatus.Complete
default:
return fmt.Errorf("batch returned unknown status %q", batchStatus.Tag)
}
// Check we got the right number of entries
entries := complete.Entries
if len(entries) != len(results) {
+3 -1
View File
@@ -1269,7 +1269,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, err)
})
if err != nil {
return nil, fmt.Errorf("about failed: %w", err)
return nil, err
}
var total uint64
if q.Allocation != nil {
@@ -1370,10 +1370,12 @@ func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.
if timeout < 30 {
timeout = 30
fs.Debugf(f, "Increasing poll interval to minimum 30s")
}
if timeout > 480 {
timeout = 480
fs.Debugf(f, "Decreasing poll interval to maximum 480s")
}
err = f.pacer.Call(func() (bool, error) {
+10 -1
View File
@@ -15,7 +15,7 @@ import (
"sync"
"time"
"github.com/rclone/ftp"
"github.com/jlaffaye/ftp"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config"
@@ -100,6 +100,11 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`,
Help: "Disable using MLSD even if server advertises support.",
Default: false,
Advanced: true,
}, {
Name: "disable_utf8",
Help: "Disable using UTF-8 even if server advertises support.",
Default: false,
Advanced: true,
}, {
Name: "writing_mdtm",
Help: "Use MDTM to set modification time (VsFtpd quirk)",
@@ -184,6 +189,7 @@ type Options struct {
SkipVerifyTLSCert bool `config:"no_check_certificate"`
DisableEPSV bool `config:"disable_epsv"`
DisableMLSD bool `config:"disable_mlsd"`
DisableUTF8 bool `config:"disable_utf8"`
WritingMDTM bool `config:"writing_mdtm"`
IdleTimeout fs.Duration `config:"idle_timeout"`
CloseTimeout fs.Duration `config:"close_timeout"`
@@ -338,6 +344,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) {
if f.opt.DisableMLSD {
ftpConfig = append(ftpConfig, ftp.DialWithDisabledMLSD(true))
}
if f.opt.DisableUTF8 {
ftpConfig = append(ftpConfig, ftp.DialWithDisabledUTF8(true))
}
if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff {
ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout)))
}
@@ -24,6 +24,7 @@ import (
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/rclone/rclone/fs"
@@ -304,6 +305,21 @@ rclone does if you know the bucket exists already.
`,
Default: false,
Advanced: true,
}, {
Name: "download_compressed",
Help: `If set this will download compressed objects as-is.
It is possible to upload objects to GCS with "Content-Encoding: gzip"
set. Normally rclone will transparently decompress these files on
download. This means that rclone can't check the hash or the size of
the file as both of these refer to the compressed object.
If this flag is set then rclone will download files with
"Content-Encoding: gzip" as they are received. This means that rclone
can check the size and hash but the file contents will be compressed.
`,
Advanced: true,
Default: false,
}, {
Name: config.ConfigEncoding,
Help: config.ConfigEncodingHelp,
@@ -327,21 +343,23 @@ type Options struct {
Location string `config:"location"`
StorageClass string `config:"storage_class"`
NoCheckBucket bool `config:"no_check_bucket"`
DownloadCompressed bool `config:"download_compressed"`
Enc encoder.MultiEncoder `config:"encoding"`
}
// Fs represents a remote storage server
type Fs struct {
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed options
features *fs.Features // optional features
svc *storage.Service // the connection to the storage server
client *http.Client // authorized client
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
cache *bucket.Cache // cache of bucket status
pacer *fs.Pacer // To pace the API calls
name string // name of this remote
root string // the path we are working on if any
opt Options // parsed options
features *fs.Features // optional features
svc *storage.Service // the connection to the storage server
client *http.Client // authorized client
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
cache *bucket.Cache // cache of bucket status
pacer *fs.Pacer // To pace the API calls
warnCompressed sync.Once // warn once about compressed files
}
// Object describes a storage object
@@ -355,6 +373,7 @@ type Object struct {
bytes int64 // Bytes in the object
modTime time.Time // Modified time of the object
mimeType string
gzipped bool // set if object has Content-Encoding: gzip
}
// ------------------------------------------------------------
@@ -975,6 +994,7 @@ func (o *Object) setMetaData(info *storage.Object) {
o.url = info.MediaLink
o.bytes = int64(info.Size)
o.mimeType = info.ContentType
o.gzipped = info.ContentEncoding == "gzip"
// Read md5sum
md5sumData, err := base64.StdEncoding.DecodeString(info.Md5Hash)
@@ -1013,6 +1033,15 @@ func (o *Object) setMetaData(info *storage.Object) {
} else {
o.modTime = modTime
}
// If gunzipping then size and md5sum are unknown
if o.gzipped && !o.fs.opt.DownloadCompressed {
o.bytes = -1
o.md5sum = ""
o.fs.warnCompressed.Do(func() {
fs.Logf(o.fs, "Decompressing 'Content-Encoding: gzip' compressed file. Use --gcs-download-compressed to override")
})
}
}
// readObjectInfo reads the definition for an object
@@ -1113,6 +1142,15 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read
return nil, err
}
fs.FixRangeOption(options, o.bytes)
if o.gzipped && o.fs.opt.DownloadCompressed {
// Allow files which are stored on the cloud storage system
// compressed to be downloaded without being decompressed. Note
// that setting this here overrides the automatic decompression
// in the Transport.
//
// See: https://cloud.google.com/storage/docs/transcoding
req.Header.Set("Accept-Encoding", "gzip")
}
fs.OpenOptionAddHTTPHeaders(req.Header, options)
var res *http.Response
err = o.fs.pacer.Call(func() (bool, error) {
+1 -1
View File
@@ -562,7 +562,7 @@ func (f *Fs) list(ctx context.Context, filter api.SearchFilter, fn listFn) (err
for i := range items {
item := &result.MediaItems[i]
remote := item.Filename
remote = strings.Replace(remote, "/", "/", -1)
remote = strings.ReplaceAll(remote, "/", "/")
err = fn(remote, item, false)
if err != nil {
return err
+2 -2
View File
@@ -282,7 +282,7 @@ func (f *Fs) CleanUp(ctx context.Context) error {
if do := f.Fs.Features().CleanUp; do != nil {
return do(ctx)
}
return errors.New("CleanUp not supported")
return errors.New("not supported by underlying remote")
}
// About gets quota information from the Fs
@@ -290,7 +290,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
if do := f.Fs.Features().About; do != nil {
return do(ctx)
}
return nil, errors.New("About not supported")
return nil, errors.New("not supported by underlying remote")
}
// ChangeNotify calls the passed function with a path that has had changes.
+1 -1
View File
@@ -119,7 +119,7 @@ func (f *Fs) getCredentials(ctx context.Context) (err error) {
defer fs.CheckClose(resp.Body, &err)
if resp.StatusCode < 200 || resp.StatusCode > 299 {
body, _ := ioutil.ReadAll(resp.Body)
bodyStr := strings.TrimSpace(strings.Replace(string(body), "\n", " ", -1))
bodyStr := strings.TrimSpace(strings.ReplaceAll(string(body), "\n", " "))
return fmt.Errorf("failed to get credentials: %s: %s", resp.Status, bodyStr)
}
decoder := json.NewDecoder(resp.Body)
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,17 @@
// Test internetarchive filesystem interface
package internetarchive_test
import (
"testing"
"github.com/rclone/rclone/backend/internetarchive"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestIA:lesmi-rclone-test/",
NilObject: (*internetarchive.Object)(nil),
})
}
+65 -43
View File
@@ -8,42 +8,69 @@ import (
)
const (
// default time format for almost all request and responses
timeFormat = "2006-01-02-T15:04:05Z0700"
// the API server seems to use a different format
apiTimeFormat = "2006-01-02T15:04:05Z07:00"
// default time format historically used for all request and responses.
// Similar to time.RFC3339, but with an extra '-' in front of 'T',
// and no ':' separator in timezone offset. Some newer endpoints have
// moved to proper time.RFC3339 conformant format instead.
jottaTimeFormat = "2006-01-02-T15:04:05Z0700"
)
// Time represents time values in the Jottacloud API. It uses a custom RFC3339 like format.
type Time time.Time
// UnmarshalXML turns XML into a Time
func (t *Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// unmarshalXML turns XML into a Time
func unmarshalXMLTime(d *xml.Decoder, start xml.StartElement, timeFormat string) (time.Time, error) {
var v string
if err := d.DecodeElement(&v, &start); err != nil {
return err
return time.Time{}, err
}
if v == "" {
*t = Time(time.Time{})
return nil
return time.Time{}, nil
}
newTime, err := time.Parse(timeFormat, v)
if err == nil {
*t = Time(newTime)
return newTime, nil
}
return time.Time{}, err
}
// JottaTime represents time values in the classic API using a custom RFC3339 like format
type JottaTime time.Time
// String returns JottaTime string in Jottacloud classic format
func (t JottaTime) String() string { return time.Time(t).Format(jottaTimeFormat) }
// UnmarshalXML turns XML into a JottaTime
func (t *JottaTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
tm, err := unmarshalXMLTime(d, start, jottaTimeFormat)
*t = JottaTime(tm)
return err
}
// MarshalXML turns a Time into XML
func (t *Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
// MarshalXML turns a JottaTime into XML
func (t *JottaTime) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
return e.EncodeElement(t.String(), start)
}
// Return Time string in Jottacloud format
func (t Time) String() string { return time.Time(t).Format(timeFormat) }
// Rfc3339Time represents time values in the newer APIs using standard RFC3339 format
type Rfc3339Time time.Time
// APIString returns Time string in Jottacloud API format
func (t Time) APIString() string { return time.Time(t).Format(apiTimeFormat) }
// String returns Rfc3339Time string in Jottacloud RFC3339 format
func (t Rfc3339Time) String() string { return time.Time(t).Format(time.RFC3339) }
// UnmarshalXML turns XML into a Rfc3339Time
func (t *Rfc3339Time) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
tm, err := unmarshalXMLTime(d, start, time.RFC3339)
*t = Rfc3339Time(tm)
return err
}
// MarshalXML turns a Rfc3339Time into XML
func (t *Rfc3339Time) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
return e.EncodeElement(t.String(), start)
}
// MarshalJSON turns a Rfc3339Time into JSON
func (t *Rfc3339Time) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf("\"%s\"", t.String())), nil
}
// LoginToken is struct representing the login token generated in the WebUI
type LoginToken struct {
@@ -122,16 +149,11 @@ type AllocateFileResponse struct {
// UploadResponse after an upload
type UploadResponse struct {
Name string `json:"name"`
Path string `json:"path"`
Kind string `json:"kind"`
ContentID string `json:"content_id"`
Bytes int64 `json:"bytes"`
Md5 string `json:"md5"`
Created int64 `json:"created"`
Modified int64 `json:"modified"`
Deleted interface{} `json:"deleted"`
Mime string `json:"mime"`
Path string `json:"path"`
ContentID string `json:"content_id"`
Bytes int64 `json:"bytes"`
Md5 string `json:"md5"`
Modified int64 `json:"modified"`
}
// DeviceRegistrationResponse is the response to registering a device
@@ -338,9 +360,9 @@ type JottaFolder struct {
Name string `xml:"name,attr"`
Deleted Flag `xml:"deleted,attr"`
Path string `xml:"path"`
CreatedAt Time `xml:"created"`
ModifiedAt Time `xml:"modified"`
Updated Time `xml:"updated"`
CreatedAt JottaTime `xml:"created"`
ModifiedAt JottaTime `xml:"modified"`
Updated JottaTime `xml:"updated"`
Folders []JottaFolder `xml:"folders>folder"`
Files []JottaFile `xml:"files>file"`
}
@@ -365,17 +387,17 @@ GET http://www.jottacloud.com/JFS/<account>/<device>/<mountpoint>/.../<file>
// JottaFile represents a Jottacloud file
type JottaFile struct {
XMLName xml.Name
Name string `xml:"name,attr"`
Deleted Flag `xml:"deleted,attr"`
PublicURI string `xml:"publicURI"`
PublicSharePath string `xml:"publicSharePath"`
State string `xml:"currentRevision>state"`
CreatedAt Time `xml:"currentRevision>created"`
ModifiedAt Time `xml:"currentRevision>modified"`
Updated Time `xml:"currentRevision>updated"`
Size int64 `xml:"currentRevision>size"`
MimeType string `xml:"currentRevision>mime"`
MD5 string `xml:"currentRevision>md5"`
Name string `xml:"name,attr"`
Deleted Flag `xml:"deleted,attr"`
PublicURI string `xml:"publicURI"`
PublicSharePath string `xml:"publicSharePath"`
State string `xml:"currentRevision>state"`
CreatedAt JottaTime `xml:"currentRevision>created"`
ModifiedAt JottaTime `xml:"currentRevision>modified"`
Updated JottaTime `xml:"currentRevision>updated"`
Size int64 `xml:"currentRevision>size"`
MimeType string `xml:"currentRevision>mime"`
MD5 string `xml:"currentRevision>md5"`
}
// Error is a custom Error for wrapping Jottacloud error responses
+71 -66
View File
@@ -191,7 +191,7 @@ machines.`)
m.Set("auth_code", "")
return fs.ConfigGoto("legacy_do_auth")
case "legacy_auth_code":
authCode := strings.Replace(config.Result, "-", "", -1) // remove any "-" contained in the code so we have a 6 digit number
authCode := strings.ReplaceAll(config.Result, "-", "") // remove any "-" contained in the code so we have a 6 digit number
m.Set("auth_code", authCode)
return fs.ConfigGoto("legacy_do_auth")
case "legacy_do_auth":
@@ -519,7 +519,7 @@ func doTokenAuth(ctx context.Context, apiSrv *rest.Client, loginTokenBase64 stri
values.Set("client_id", defaultClientID)
values.Set("grant_type", "password")
values.Set("password", loginToken.AuthToken)
values.Set("scope", "offline_access+openid")
values.Set("scope", "openid offline_access")
values.Set("username", loginToken.Username)
values.Encode()
opts = rest.Opts{
@@ -649,7 +649,7 @@ func errorHandler(resp *http.Response) error {
// Jottacloud wants '+' to be URL encoded even though the RFC states it's not reserved
func urlPathEscape(in string) string {
return strings.Replace(rest.URLPathEscape(in), "+", "%2B", -1)
return strings.ReplaceAll(rest.URLPathEscape(in), "+", "%2B")
}
// filePathRaw returns an unescaped file path (f.root, file)
@@ -932,26 +932,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return entries, nil
}
type listStreamTime time.Time
func (c *listStreamTime) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var v string
if err := d.DecodeElement(&v, &start); err != nil {
return err
}
t, err := time.Parse(time.RFC3339, v)
if err != nil {
return err
}
*c = listStreamTime(t)
return nil
}
func (c listStreamTime) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf("\"%s\"", time.Time(c).Format(time.RFC3339))), nil
}
func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, filesystem *Fs, callback func(fs.DirEntry) error) error {
func parseListRStream(ctx context.Context, r io.Reader, filesystem *Fs, callback func(fs.DirEntry) error) error {
type stats struct {
Folders int `xml:"folders"`
@@ -960,12 +941,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
var expected, actual stats
type xmlFile struct {
Path string `xml:"path"`
Name string `xml:"filename"`
Checksum string `xml:"md5"`
Size int64 `xml:"size"`
Modified listStreamTime `xml:"modified"`
Created listStreamTime `xml:"created"`
Path string `xml:"path"`
Name string `xml:"filename"`
Checksum string `xml:"md5"`
Size int64 `xml:"size"`
Modified api.Rfc3339Time `xml:"modified"` // Note: Liststream response includes 3 decimal milliseconds, but we ignore them since there is second precision everywhere else
Created api.Rfc3339Time `xml:"created"`
}
type xmlFolder struct {
@@ -987,8 +968,12 @@ func parseListRStream(ctx context.Context, r io.Reader, trimPrefix string, files
})
}
// liststream paths are /mountpoint/root/path
// so the returned paths should have /mountpoint/root/ trimmed
// as the caller is expecting path.
pathPrefix := filesystem.opt.Enc.FromStandardPath(path.Join("/", filesystem.opt.Mountpoint, filesystem.root))
trimPathPrefix := func(p string) string {
p = strings.TrimPrefix(p, trimPrefix)
p = strings.TrimPrefix(p, pathPrefix)
p = strings.TrimPrefix(p, "/")
return p
}
@@ -1071,11 +1056,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return shouldRetry(ctx, resp, err)
}
// liststream paths are /mountpoint/root/path
// so the returned paths should have /mountpoint/root/ trimmed
// as the caller is expecting path.
trimPrefix := path.Join("/", f.opt.Mountpoint, f.root)
err = parseListRStream(ctx, resp.Body, trimPrefix, f, func(d fs.DirEntry) error {
err = parseListRStream(ctx, resp.Body, f, func(d fs.DirEntry) error {
if d.Remote() == dir {
return nil
}
@@ -1210,6 +1191,45 @@ func (f *Fs) Purge(ctx context.Context, dir string) error {
return f.purgeCheck(ctx, dir, false)
}
// createOrUpdate tries to make remote file match without uploading.
// If the remote file exists, and has matching size and md5, only
// timestamps are updated. If the file does not exist or does does
// not match size and md5, but matching content can be constructed
// from deduplication, the file will be updated/created. If the file
// is currently in trash, but can be made to match, it will be
// restored. Returns ErrorObjectNotFound if upload will be necessary
// to get a matching remote file.
func (f *Fs) createOrUpdate(ctx context.Context, file string, modTime time.Time, size int64, md5 string) (info *api.JottaFile, err error) {
opts := rest.Opts{
Method: "POST",
Path: f.filePath(file),
Parameters: url.Values{},
ExtraHeaders: make(map[string]string),
}
opts.Parameters.Set("cphash", "true")
fileDate := api.JottaTime(modTime).String()
opts.ExtraHeaders["JSize"] = strconv.FormatInt(size, 10)
opts.ExtraHeaders["JMd5"] = md5
opts.ExtraHeaders["JCreated"] = fileDate
opts.ExtraHeaders["JModified"] = fileDate
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallXML(ctx, &opts, nil, &info)
return shouldRetry(ctx, resp, err)
})
if apiErr, ok := err.(*api.Error); ok {
// does not exist, i.e. not matching size and md5, and not possible to make it by deduplication
if apiErr.StatusCode == http.StatusNotFound {
return nil, fs.ErrorObjectNotFound
}
}
return info, nil
}
// copyOrMoves copies or moves directories or files depending on the method parameter
func (f *Fs) copyOrMove(ctx context.Context, method, src, dest string) (info *api.JottaFile, err error) {
opts := rest.Opts{
@@ -1253,6 +1273,12 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
}
info, err := f.copyOrMove(ctx, "cp", srcObj.filePath(), remote)
// if destination was a trashed file then after a successfull copy the copied file is still in trash (bug in api?)
if err == nil && bool(info.Deleted) && !f.opt.TrashedOnly && info.State == "COMPLETED" {
fs.Debugf(src, "Server-side copied to trashed destination, restoring")
info, err = f.createOrUpdate(ctx, remote, srcObj.modTime, srcObj.size, srcObj.md5)
}
if err != nil {
return nil, fmt.Errorf("couldn't copy file: %w", err)
}
@@ -1554,40 +1580,19 @@ func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error {
return err
}
// prepare allocate request with existing metadata but changed timestamps
var resp *http.Response
var options []fs.OpenOption
opts := rest.Opts{
Method: "POST",
Path: "files/v1/allocate",
Options: options,
ExtraHeaders: make(map[string]string),
}
fileDate := api.Time(modTime).APIString()
var request = api.AllocateFileRequest{
Bytes: o.size,
Created: fileDate,
Modified: fileDate,
Md5: o.md5,
Path: path.Join(o.fs.opt.Mountpoint, o.fs.opt.Enc.FromStandardPath(path.Join(o.fs.root, o.remote))),
}
// send it
var response api.AllocateFileResponse
err = o.fs.pacer.Call(func() (bool, error) {
resp, err = o.fs.apiSrv.CallJSON(ctx, &opts, &request, &response)
return shouldRetry(ctx, resp, err)
})
// request check/update with existing metadata and new modtime
// (note that if size/md5 does not match, the file content will
// also be modified if deduplication is possible, i.e. it is
// important to use correct/latest values)
_, err = o.fs.createOrUpdate(ctx, o.remote, modTime, o.size, o.md5)
if err != nil {
if err == fs.ErrorObjectNotFound {
// file was modified (size/md5 changed) between readMetaData and createOrUpdate?
return errors.New("metadata did not match")
}
return err
}
// check response
if response.State != "COMPLETED" {
// could be the file was modified (size/md5 changed) between readMetaData and the allocate request
return errors.New("metadata did not match")
}
// update local metadata
o.modTime = modTime
return nil
@@ -1725,7 +1730,7 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
Options: options,
ExtraHeaders: make(map[string]string),
}
fileDate := api.Time(src.ModTime(ctx)).APIString()
fileDate := api.Rfc3339Time(src.ModTime(ctx)).String()
// the allocate request
var request = api.AllocateFileRequest{
+141 -1
View File
@@ -301,6 +301,10 @@ type siteResource struct {
type siteResponse struct {
Sites []siteResource `json:"value"`
}
type deltaResponse struct {
DeltaLink string `json:"@odata.deltaLink"`
Value []api.Item `json:"value"`
}
// Get the region and graphURL from the config
func getRegionURL(m configmap.Mapper) (region, graphURL string) {
@@ -1502,7 +1506,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("about failed: %w", err)
return nil, err
}
q := drive.Quota
// On (some?) Onedrive sharepoints these are all 0 so return unknown in that case
@@ -2302,6 +2306,142 @@ func (f *Fs) canonicalDriveID(driveID string) (canonicalDriveID string) {
return canonicalDriveID
}
// ChangeNotify calls the passed function with a path that has had changes.
// If the implementation uses polling, it should adhere to the given interval.
//
// Automatically restarts itself in case of unexpected behavior of the remote.
//
// Close the returned channel to stop being notified.
//
// The Onedrive implementation gives the whole hierarchy up to the top when
// an object is changed. For instance, if a/b/c is changed, this function
// will call notifyFunc with a, a/b and a/b/c.
func (f *Fs) ChangeNotify(ctx context.Context, notifyFunc func(string, fs.EntryType), pollIntervalChan <-chan time.Duration) {
go func() {
// get the StartPageToken early so all changes from now on get processed
nextDeltaToken, err := f.changeNotifyStartPageToken(ctx)
if err != nil {
fs.Errorf(f, "Could not get first deltaLink: %s", err)
return
}
fs.Debugf(f, "Next delta token is: %s", nextDeltaToken)
var ticker *time.Ticker
var tickerC <-chan time.Time
for {
select {
case pollInterval, ok := <-pollIntervalChan:
if !ok {
if ticker != nil {
ticker.Stop()
}
return
}
if ticker != nil {
ticker.Stop()
ticker, tickerC = nil, nil
}
if pollInterval != 0 {
ticker = time.NewTicker(pollInterval)
tickerC = ticker.C
}
case <-tickerC:
fs.Debugf(f, "Checking for changes on remote")
nextDeltaToken, err = f.changeNotifyRunner(ctx, notifyFunc, nextDeltaToken)
if err != nil {
fs.Infof(f, "Change notify listener failure: %s", err)
}
}
}
}()
}
func (f *Fs) changeNotifyStartPageToken(ctx context.Context) (nextDeltaToken string, err error) {
delta, err := f.changeNotifyNextChange(ctx, "latest")
parsedURL, err := url.Parse(delta.DeltaLink)
if err != nil {
return
}
nextDeltaToken = parsedURL.Query().Get("token")
return
}
func (f *Fs) changeNotifyNextChange(ctx context.Context, token string) (delta deltaResponse, err error) {
opts := f.buildDriveDeltaOpts(token)
_, err = f.srv.CallJSON(ctx, &opts, nil, &delta)
return
}
func (f *Fs) buildDriveDeltaOpts(token string) rest.Opts {
rootURL := graphAPIEndpoint[f.opt.Region] + "/v1.0/drives"
return rest.Opts{
Method: "GET",
RootURL: rootURL,
Path: "/" + f.driveID + "/root/delta",
Parameters: map[string][]string{"token": {token}},
}
}
func (f *Fs) changeNotifyRunner(ctx context.Context, notifyFunc func(string, fs.EntryType), deltaToken string) (nextDeltaToken string, err error) {
delta, err := f.changeNotifyNextChange(ctx, deltaToken)
parsedURL, err := url.Parse(delta.DeltaLink)
if err != nil {
return
}
nextDeltaToken = parsedURL.Query().Get("token")
for _, item := range delta.Value {
isDriveRootFolder := item.GetParentReference().ID == ""
if isDriveRootFolder {
continue
}
fullPath, err := getItemFullPath(&item)
if err != nil {
fs.Errorf(f, "Could not get item full path: %s", err)
continue
}
if fullPath == f.root {
continue
}
relName, insideRoot := getRelativePathInsideBase(f.root, fullPath)
if !insideRoot {
continue
}
if item.GetFile() != nil {
notifyFunc(relName, fs.EntryObject)
} else if item.GetFolder() != nil {
notifyFunc(relName, fs.EntryDirectory)
}
}
return
}
func getItemFullPath(item *api.Item) (fullPath string, err error) {
err = nil
fullPath = item.GetName()
if parent := item.GetParentReference(); parent != nil && parent.Path != "" {
pathParts := strings.SplitN(parent.Path, ":", 2)
if len(pathParts) != 2 {
err = fmt.Errorf("invalid parent path: %s", parent.Path)
return
}
if pathParts[1] != "" {
fullPath = strings.TrimPrefix(pathParts[1], "/") + "/" + fullPath
}
}
return
}
// getRelativePathInsideBase checks if `target` is inside `base`. If so, it
// returns a relative path for `target` based on `base` and a boolean `true`.
// Otherwise returns "", false.
+1 -1
View File
@@ -906,7 +906,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("about failed: %w", err)
return nil, err
}
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used
+2 -2
View File
@@ -783,10 +783,10 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("CreateDir http: %w", err)
return nil, err
}
if err = info.AsErr(); err != nil {
return nil, fmt.Errorf("CreateDir: %w", err)
return nil, err
}
usage = &fs.Usage{
Used: fs.NewUsageValue(int64(info.SpaceUsed)),
+1 -1
View File
@@ -647,7 +647,7 @@ func (f *Fs) About(ctx context.Context) (usage *fs.Usage, err error) {
return shouldRetry(ctx, err)
})
if err != nil {
return nil, fmt.Errorf("about failed: %w", err)
return nil, err
}
return &fs.Usage{
Total: fs.NewUsageValue(ai.Disk.Size), // quota of bytes that can be used
+258 -91
View File
@@ -32,9 +32,11 @@ import (
"github.com/aws/aws-sdk-go/aws/endpoints"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
v4 "github.com/aws/aws-sdk-go/aws/signer/v4"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/ncw/swift/v2"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/chunksize"
"github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
@@ -58,7 +60,7 @@ import (
func init() {
fs.Register(&fs.RegInfo{
Name: "s3",
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
Description: "Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi",
NewFs: NewFs,
CommandHelp: commandHelp,
Options: []fs.Option{{
@@ -78,6 +80,12 @@ func init() {
}, {
Value: "ChinaMobile",
Help: "China Mobile Ecloud Elastic Object Storage (EOS)",
}, {
Value: "Cloudflare",
Help: "Cloudflare R2 Storage",
}, {
Value: "ArvanCloud",
Help: "Arvan Cloud Object Storage (AOS)",
}, {
Value: "DigitalOcean",
Help: "Digital Ocean Spaces",
@@ -293,11 +301,22 @@ func init() {
}, {
Value: "fr-par",
Help: "Paris, France",
}, {
Value: "pl-waw",
Help: "Warsaw, Poland",
}},
}, {
Name: "region",
Help: "Region to connect to.",
Provider: "Cloudflare",
Examples: []fs.OptionExample{{
Value: "auto",
Help: "R2 buckets are automatically distributed across Cloudflare's data centers for low latency.",
}},
}, {
Name: "region",
Help: "Region to connect to.\n\nLeave blank if you are using an S3 clone and you don't have a region.",
Provider: "!AWS,Alibaba,ChinaMobile,RackCorp,Scaleway,Storj,TencentCOS",
Provider: "!AWS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS",
Examples: []fs.OptionExample{{
Value: "",
Help: "Use this if unsure.\nWill use v4 signatures and an empty region.",
@@ -405,6 +424,18 @@ func init() {
Value: "eos-anhui-1.cmecloud.cn",
Help: "Anhui China (Huainan)",
}},
}, {
// ArvanCloud endpoints: https://www.arvancloud.com/en/products/cloud-storage
Name: "endpoint",
Help: "Endpoint for Arvan Cloud Object Storage (AOS) API.",
Provider: "ArvanCloud",
Examples: []fs.OptionExample{{
Value: "s3.ir-thr-at1.arvanstorage.com",
Help: "The default endpoint - a good choice if you are unsure.\nTehran Iran (Asiatech)",
}, {
Value: "s3.ir-tbz-sh1.arvanstorage.com",
Help: "Tabriz Iran (Shahriar)",
}},
}, {
Name: "endpoint",
Help: "Endpoint for IBM COS S3 API.\n\nSpecify if using an IBM COS On Premise.",
@@ -687,6 +718,9 @@ func init() {
}, {
Value: "s3.fr-par.scw.cloud",
Help: "Paris Endpoint",
}, {
Value: "s3.pl-waw.scw.cloud",
Help: "Warsaw Endpoint",
}},
}, {
Name: "endpoint",
@@ -845,7 +879,7 @@ func init() {
}, {
Name: "endpoint",
Help: "Endpoint for S3 API.\n\nRequired when using an S3 clone.",
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,Scaleway,StackPath,Storj,RackCorp",
Provider: "!AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp",
Examples: []fs.OptionExample{{
Value: "objects-us-east-1.dream.io",
Help: "Dream Objects endpoint",
@@ -898,6 +932,10 @@ func init() {
Value: "s3.ap-northeast-2.wasabisys.com",
Help: "Wasabi AP Northeast 2 (Osaka) endpoint",
Provider: "Wasabi",
}, {
Value: "s3.ir-thr-at1.arvanstorage.com",
Help: "ArvanCloud Tehran Iran (Asiatech) endpoint",
Provider: "ArvanCloud",
}},
}, {
Name: "location_constraint",
@@ -1074,6 +1112,17 @@ func init() {
Value: "anhui1",
Help: "Anhui China (Huainan)",
}},
}, {
Name: "location_constraint",
Help: "Location constraint - must match endpoint.\n\nUsed when creating buckets only.",
Provider: "ArvanCloud",
Examples: []fs.OptionExample{{
Value: "ir-thr-at1",
Help: "Tehran Iran (Asiatech)",
}, {
Value: "ir-tbz-sh1",
Help: "Tabriz Iran (Shahriar)",
}},
}, {
Name: "location_constraint",
Help: "Location constraint - must match endpoint when using IBM Cloud Public.\n\nFor on-prem COS, do not make a selection from this list, hit enter.",
@@ -1240,7 +1289,7 @@ func init() {
}, {
Name: "location_constraint",
Help: "Location constraint - must be set to match the Region.\n\nLeave blank if not sure. Used when creating buckets only.",
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
Provider: "!AWS,IBMCOS,Alibaba,ChinaMobile,Cloudflare,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS",
}, {
Name: "acl",
Help: `Canned ACL used when creating buckets and storing or copying objects.
@@ -1251,7 +1300,7 @@ For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview
Note that this ACL is applied when server-side copying objects as S3
doesn't copy the ACL from the source but rather writes a fresh one.`,
Provider: "!Storj",
Provider: "!Storj,Cloudflare",
Examples: []fs.OptionExample{{
Value: "default",
Help: "Owner gets Full_CONTROL.\nNo one else has access rights (default).",
@@ -1452,6 +1501,15 @@ If you leave it blank, this is calculated automatically from the sse_customer_ke
Value: "STANDARD_IA",
Help: "Infrequent access storage mode",
}},
}, {
// Mapping from here: https://www.arvancloud.com/en/products/cloud-storage
Name: "storage_class",
Help: "The storage class to use when storing new objects in ArvanCloud.",
Provider: "ArvanCloud",
Examples: []fs.OptionExample{{
Value: "STANDARD",
Help: "Standard storage class",
}},
}, {
// Mapping from here: https://intl.cloud.tencent.com/document/product/436/30925
Name: "storage_class",
@@ -1784,6 +1842,20 @@ This should be true, false or left unset to use the default for the provider.
`,
Default: fs.Tristate{},
Advanced: true,
}, {
Name: "use_presigned_request",
Help: `Whether to use a presigned request or PutObject for single part uploads
If this is false rclone will use PutObject from the AWS SDK to upload
an object.
Versions of rclone < 1.59 use presigned requests to upload a single
part object and setting this flag to true will re-enable that
functionality. This shouldn't be necessary except in exceptional
circumstances or for testing.
`,
Default: false,
Advanced: true,
},
}})
}
@@ -1849,6 +1921,7 @@ type Options struct {
DisableHTTP2 bool `config:"disable_http2"`
DownloadURL string `config:"download_url"`
UseMultipartEtag fs.Tristate `config:"use_multipart_etag"`
UsePresignedRequest bool `config:"use_presigned_request"`
}
// Fs represents a remote s3 server
@@ -1860,6 +1933,7 @@ type Fs struct {
ctx context.Context // global context for reading config
features *fs.Features // optional features
c *s3.S3 // the connection to the s3 server
cu *s3.S3 // unsigned connection to the s3 server for PutObject
ses *session.Session // the s3 session
rootBucket string // bucket part of root (if any)
rootDirectory string // directory part of root (if any)
@@ -1992,7 +2066,11 @@ func getClient(ctx context.Context, opt *Options) *http.Client {
}
// s3Connection makes a connection to s3
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *session.Session, error) {
//
// If unsignedBody is set then the connection is configured for
// unsigned bodies which is necessary for PutObject if we don't want
// it to seek
func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S3, *s3.S3, *session.Session, error) {
ci := fs.GetConfig(ctx)
// Make the auth
v := credentials.Value{
@@ -2009,7 +2087,7 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
// start a new AWS session
awsSession, err := session.NewSession()
if err != nil {
return nil, nil, fmt.Errorf("NewSession: %w", err)
return nil, nil, nil, fmt.Errorf("NewSession: %w", err)
}
// first provider to supply a credential set "wins"
@@ -2049,9 +2127,9 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
// if no access key/secret and iam is explicitly disabled then fall back to anon interaction
cred = credentials.AnonymousCredentials
case v.AccessKeyID == "":
return nil, nil, errors.New("access_key_id not found")
return nil, nil, nil, errors.New("access_key_id not found")
case v.SecretAccessKey == "":
return nil, nil, errors.New("secret_access_key not found")
return nil, nil, nil, errors.New("secret_access_key not found")
}
if opt.Region == "" {
@@ -2090,25 +2168,36 @@ func s3Connection(ctx context.Context, opt *Options, client *http.Client) (*s3.S
// (from the shared config file) if the passed-in Options.Config.Credentials is nil.
awsSessionOpts.Config.Credentials = nil
}
// Setting this stops PutObject reading the body twice and seeking
// We add our own Content-MD5 for data protection
awsSessionOpts.Config.S3DisableContentMD5Validation = aws.Bool(true)
ses, err := session.NewSessionWithOptions(awsSessionOpts)
if err != nil {
return nil, nil, err
return nil, nil, nil, err
}
c := s3.New(ses)
if opt.V2Auth || opt.Region == "other-v2-signature" {
fs.Debugf(nil, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
if req.Config.Credentials == credentials.AnonymousCredentials {
return
newC := func(unsignedBody bool) *s3.S3 {
c := s3.New(ses)
if opt.V2Auth || opt.Region == "other-v2-signature" {
fs.Debugf(nil, "Using v2 auth")
signer := func(req *request.Request) {
// Ignore AnonymousCredentials object
if req.Config.Credentials == credentials.AnonymousCredentials {
return
}
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
}
sign(v.AccessKeyID, v.SecretAccessKey, req.HTTPRequest)
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBack(signer)
} else if unsignedBody {
// If the body is unsigned then tell the signer that we aren't signing the payload
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBackNamed(v4.BuildNamedHandler("v4.SignRequestHandler.WithUnsignedPayload", v4.WithUnsignedPayload))
}
c.Handlers.Sign.Clear()
c.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
c.Handlers.Sign.PushBack(signer)
return c
}
return c, ses, nil
return newC(false), newC(true), ses, nil
}
func checkUploadChunkSize(cs fs.SizeSuffix) error {
@@ -2167,6 +2256,13 @@ func setQuirks(opt *Options) {
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
case "Cloudflare":
virtualHostStyle = false
useMultipartEtag = false // currently multipart Etags are random
case "ArvanCloud":
listObjectsV2 = false
virtualHostStyle = false
urlEncodeListings = false
case "DigitalOcean":
urlEncodeListings = false
case "Dreamhost":
@@ -2286,7 +2382,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
opt.SSECustomerKeyMD5 = base64.StdEncoding.EncodeToString(md5sumBinary[:])
}
srv := getClient(ctx, opt)
c, ses, err := s3Connection(ctx, opt, srv)
c, cu, ses, err := s3Connection(ctx, opt, srv)
if err != nil {
return nil, err
}
@@ -2304,6 +2400,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
ci: ci,
ctx: ctx,
c: c,
cu: cu,
ses: ses,
pacer: pc,
cache: bucket.NewCache(),
@@ -2427,11 +2524,12 @@ func (f *Fs) updateRegionForBucket(ctx context.Context, bucket string) error {
// Make a new session with the new region
oldRegion := f.opt.Region
f.opt.Region = region
c, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
c, cu, ses, err := s3Connection(f.ctx, &f.opt, f.srv)
if err != nil {
return fmt.Errorf("creating new session failed: %w", err)
}
f.c = c
f.cu = cu
f.ses = ses
fs.Logf(f, "Switched region to %q from %q", region, oldRegion)
@@ -2874,7 +2972,7 @@ func (f *Fs) Precision() time.Duration {
// pathEscape escapes s as for a URL path. It uses rest.URLPathEscape
// but also escapes '+' for S3 and Digital Ocean spaces compatibility
func pathEscape(s string) string {
return strings.Replace(rest.URLPathEscape(s), "+", "%2B", -1)
return strings.ReplaceAll(rest.URLPathEscape(s), "+", "%2B")
}
// copy does a server-side copy
@@ -3247,6 +3345,10 @@ func (f *Fs) Command(ctx context.Context, name string, arg []string, opt map[str
st.Status = "Not an S3 object"
return
}
if o.storageClass != "GLACIER" && o.storageClass != "DEEP_ARCHIVE" {
st.Status = "Not GLACIER or DEEP_ARCHIVE storage class"
return
}
bucket, bucketPath := o.split()
reqCopy := req
reqCopy.Bucket = &bucket
@@ -3743,7 +3845,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
}
// calculate size of parts
partSize := int(f.opt.ChunkSize)
partSize := f.opt.ChunkSize
// size can be -1 here meaning we don't know the size of the incoming file. We use ChunkSize
// buffers here (default 5 MiB). With a maximum number of parts (10,000) this will be a file of
@@ -3754,11 +3856,7 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
f.opt.ChunkSize, fs.SizeSuffix(int64(partSize)*uploadParts))
})
} else {
// Adjust partSize until the number of parts is small enough.
if size/int64(partSize) >= uploadParts {
// Calculate partition size rounded up to the nearest MiB
partSize = int((((size / uploadParts) >> 20) + 1) << 20)
}
partSize = chunksize.Calculator(o, int(uploadParts), f.opt.ChunkSize)
}
memPool := f.getMemoryPool(int64(partSize))
@@ -3928,6 +4026,117 @@ func (o *Object) uploadMultipart(ctx context.Context, req *s3.PutObjectInput, si
return etag, nil
}
// unWrapAwsError unwraps AWS errors, looking for a non AWS error
//
// It returns true if one was found and the error, or false and the
// error passed in.
func unWrapAwsError(err error) (found bool, outErr error) {
if awsErr, ok := err.(awserr.Error); ok {
var origErrs []error
if batchErr, ok := awsErr.(awserr.BatchError); ok {
origErrs = batchErr.OrigErrs()
} else {
origErrs = []error{awsErr.OrigErr()}
}
for _, origErr := range origErrs {
found, newErr := unWrapAwsError(origErr)
if found {
return found, newErr
}
}
return false, err
}
return true, err
}
// Upload a single part using PutObject
func (o *Object) uploadSinglepartPutObject(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
req.Body = readers.NewFakeSeeker(in, size)
var resp *s3.PutObjectOutput
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
resp, err = o.fs.cu.PutObject(req)
return o.fs.shouldRetry(ctx, err)
})
if err != nil {
// Return the underlying error if we have a Serialization error if possible
//
// Serialization errors are synthesized locally in the SDK (not returned from the
// server). We'll get one if the SDK attempts a retry, however the FakeSeeker will
// remember the previous error from Read and return that.
if do, ok := err.(awserr.Error); ok && do.Code() == request.ErrCodeSerialization {
if found, newErr := unWrapAwsError(err); found {
err = newErr
}
}
return etag, lastModified, err
}
lastModified = time.Now()
etag = aws.StringValue(resp.ETag)
return etag, lastModified, nil
}
// Upload a single part using a presigned request
func (o *Object) uploadSinglepartPresignedRequest(ctx context.Context, req *s3.PutObjectInput, size int64, in io.Reader) (etag string, lastModified time.Time, err error) {
// Create the request
putObj, _ := o.fs.c.PutObjectRequest(req)
// Sign it so we can upload using a presigned request.
//
// Note the SDK didn't used to support streaming to
// PutObject so we used this work-around.
url, headers, err := putObj.PresignRequest(15 * time.Minute)
if err != nil {
return etag, lastModified, fmt.Errorf("s3 upload: sign request: %w", err)
}
if o.fs.opt.V2Auth && headers == nil {
headers = putObj.HTTPRequest.Header
}
// Set request to nil if empty so as not to make chunked encoding
if size == 0 {
in = nil
}
// create the vanilla http request
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
if err != nil {
return etag, lastModified, fmt.Errorf("s3 upload: new request: %w", err)
}
// set the headers we signed and the length
httpReq.Header = headers
httpReq.ContentLength = size
var resp *http.Response
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
var err error
resp, err = o.fs.srv.Do(httpReq)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
body, err := rest.ReadBody(resp)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
return false, nil
}
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
})
if err != nil {
return etag, lastModified, err
}
if resp != nil {
if date, err := http.ParseTime(resp.Header.Get("Date")); err != nil {
lastModified = date
}
etag = resp.Header.Get("Etag")
}
return etag, lastModified, nil
}
// Update the Object from in with modTime and size
func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) error {
bucket, bucketPath := o.split()
@@ -3979,6 +4188,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
ContentType: &mimeType,
Metadata: metadata,
}
if size >= 0 {
req.ContentLength = &size
}
if md5sumBase64 != "" {
req.ContentMD5 = &md5sumBase64
}
@@ -4033,66 +4245,21 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
}
}
var resp *http.Response // response from PUT
var wantETag string // Multipart upload Etag to check
var wantETag string // Multipart upload Etag to check
var gotEtag string // Etag we got from the upload
var lastModified time.Time // Time we got from the upload
if multipart {
wantETag, err = o.uploadMultipart(ctx, &req, size, in)
if err != nil {
return err
}
} else {
// Create the request
putObj, _ := o.fs.c.PutObjectRequest(&req)
// Sign it so we can upload using a presigned request.
//
// Note the SDK doesn't currently support streaming to
// PutObject so we'll use this work-around.
url, headers, err := putObj.PresignRequest(15 * time.Minute)
if err != nil {
return fmt.Errorf("s3 upload: sign request: %w", err)
}
if o.fs.opt.V2Auth && headers == nil {
headers = putObj.HTTPRequest.Header
}
// Set request to nil if empty so as not to make chunked encoding
if size == 0 {
in = nil
}
// create the vanilla http request
httpReq, err := http.NewRequestWithContext(ctx, "PUT", url, in)
if err != nil {
return fmt.Errorf("s3 upload: new request: %w", err)
}
// set the headers we signed and the length
httpReq.Header = headers
httpReq.ContentLength = size
err = o.fs.pacer.CallNoRetry(func() (bool, error) {
var err error
resp, err = o.fs.srv.Do(httpReq)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
body, err := rest.ReadBody(resp)
if err != nil {
return o.fs.shouldRetry(ctx, err)
}
if resp.StatusCode >= 200 && resp.StatusCode < 299 {
return false, nil
}
err = fmt.Errorf("s3 upload: %s: %s", resp.Status, body)
return fserrors.ShouldRetryHTTP(resp, retryErrorCodes), err
})
if err != nil {
return err
if o.fs.opt.UsePresignedRequest {
gotEtag, lastModified, err = o.uploadSinglepartPresignedRequest(ctx, &req, size, in)
} else {
gotEtag, lastModified, err = o.uploadSinglepartPutObject(ctx, &req, size, in)
}
}
if err != nil {
return err
}
// User requested we don't HEAD the object after uploading it
// so make up the object as best we can assuming it got
@@ -4105,11 +4272,11 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
o.mimeType = aws.StringValue(req.ContentType)
o.storageClass = aws.StringValue(req.StorageClass)
// If we have done a single part PUT request then we can read these
if resp != nil {
if date, err := http.ParseTime(resp.Header.Get("Date")); err == nil {
o.lastModified = date
}
o.setMD5FromEtag(resp.Header.Get("Etag"))
if gotEtag != "" {
o.setMD5FromEtag(gotEtag)
}
if !o.lastModified.IsZero() {
o.lastModified = lastModified
}
return nil
}
+3 -3
View File
@@ -109,7 +109,7 @@ when the ssh-agent contains many keys.`,
Default: false,
}, {
Name: "use_insecure_cipher",
Help: `Enable the use of insecure ciphers and key exchange methods.
Help: `Enable the use of insecure ciphers and key exchange methods.
This enables the use of the following insecure ciphers and key exchange methods:
@@ -1220,7 +1220,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
}
stdout, err := f.run(ctx, "df -k "+escapedPath)
if err != nil {
return nil, fmt.Errorf("your remote may not support About: %w", err)
return nil, fmt.Errorf("your remote may not have the required df utility: %w", err)
}
usageTotal, usageUsed, usageAvail := parseUsage(stdout)
@@ -1311,7 +1311,7 @@ var shellEscapeRegex = regexp.MustCompile("[^A-Za-z0-9_.,:/\\@\u0080-\uFFFFFFFF\
// when sending it to a shell.
func shellEscape(str string) string {
safe := shellEscapeRegex.ReplaceAllString(str, `\$0`)
return strings.Replace(safe, "\n", "'\n'", -1)
return strings.ReplaceAll(safe, "\n", "'\n'")
}
// Converts a byte array from the SSH session returned by
+1 -1
View File
@@ -579,7 +579,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
return nil, err
}
return newObjectFromUplink(f, "", upload.Info()), nil
return newObjectFromUplink(f, src.Remote(), upload.Info()), nil
}
// PutStream uploads to the remote path with the modTime given of indeterminate
+18 -13
View File
@@ -141,22 +141,20 @@ func (f *Fs) Hashes() hash.Set {
return f.hashSet
}
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
// mkdir makes the directory passed in and returns the upstreams used
func (f *Fs) mkdir(ctx context.Context, dir string) ([]*upstream.Fs, error) {
upstreams, err := f.create(ctx, dir)
if err == fs.ErrorObjectNotFound {
if dir != parentDir(dir) {
if err := f.Mkdir(ctx, parentDir(dir)); err != nil {
return err
}
upstreams, err = f.create(ctx, dir)
parent := parentDir(dir)
if dir != parent {
upstreams, err = f.mkdir(ctx, parent)
} else if dir == "" {
// If root dirs not created then create them
upstreams, err = f.upstreams, nil
}
}
if err != nil {
return err
return nil, err
}
errs := Errors(make([]error, len(upstreams)))
multithread(len(upstreams), func(i int) {
@@ -165,7 +163,17 @@ func (f *Fs) Mkdir(ctx context.Context, dir string) error {
errs[i] = fmt.Errorf("%s: %w", upstreams[i].Name(), err)
}
})
return errs.Err()
err = errs.Err()
if err != nil {
return nil, err
}
return upstreams, nil
}
// Mkdir makes the root directory of the Fs object
func (f *Fs) Mkdir(ctx context.Context, dir string) error {
_, err := f.mkdir(ctx, dir)
return err
}
// Purge all files in the directory
@@ -449,10 +457,7 @@ func (f *Fs) put(ctx context.Context, in io.Reader, src fs.ObjectInfo, stream bo
srcPath := src.Remote()
upstreams, err := f.create(ctx, srcPath)
if err == fs.ErrorObjectNotFound {
if err := f.Mkdir(ctx, parentDir(srcPath)); err != nil {
return nil, err
}
upstreams, err = f.create(ctx, srcPath)
upstreams, err = f.mkdir(ctx, parentDir(srcPath))
}
if err != nil {
return nil, err
+39 -5
View File
@@ -124,6 +124,22 @@ You can set multiple headers, e.g. '"Cookie","name=value","Authorization","xxx"'
`,
Default: fs.CommaSepList{},
Advanced: true,
}, {
Name: "base_path",
Help: `Base path of expected replies
Normally WebDAV servers return the files they are listing under the
url path as specified above. However some WebDAV servers return files
with URLs that are not under the endpoint URL. This causes rclone to
get confused and return errors like
Item with unknown path received: "/remote.php/webdav/folder1/", "/elsewhere/remote.php/webdav/folder1/"
errors. If that is the case, then set "base_path" to the path
specified in the error message up to the first item, in the above
example "/elsewhere/remote.php/webdav/".
`,
Advanced: true,
}},
})
}
@@ -138,6 +154,7 @@ type Options struct {
BearerTokenCommand string `config:"bearer_token_command"`
Enc encoder.MultiEncoder `config:"encoding"`
Headers fs.CommaSepList `config:"headers"`
BasePath string `config:"base_path"`
}
// Fs represents a remote webdav
@@ -454,7 +471,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e
if err != nil {
return nil, err
}
f.srv.SetHeader("Referer", u.String())
if !f.findHeader(opt.Headers, "Referer") {
f.srv.SetHeader("Referer", u.String())
}
if root != "" && !rootIsDir {
// Check to see if the root actually an existing file
@@ -517,6 +536,17 @@ func (f *Fs) addHeaders(headers fs.CommaSepList) {
}
}
// Returns true if the header was configured
func (f *Fs) findHeader(headers fs.CommaSepList, find string) bool {
for i := 0; i < len(headers); i += 2 {
key := f.opt.Headers[i]
if strings.EqualFold(key, find) {
return true
}
}
return false
}
// fetch the bearer token and set it if successful
func (f *Fs) fetchAndSetBearerToken() error {
if f.opt.BearerTokenCommand == "" {
@@ -680,6 +710,10 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
if err != nil {
return false, fmt.Errorf("couldn't join URL: %w", err)
}
basePath := baseURL.Path
if f.opt.BasePath != "" {
basePath = f.opt.BasePath
}
for i := range result.Responses {
item := &result.Responses[i]
isDir := itemIsDir(item)
@@ -694,11 +728,11 @@ func (f *Fs) listAll(ctx context.Context, dir string, directoriesOnly bool, file
if isDir {
u.Path = addSlash(u.Path)
}
if !strings.HasPrefix(u.Path, baseURL.Path) {
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, baseURL.Path)
if !strings.HasPrefix(u.Path, basePath) {
fs.Debugf(nil, "Item with unknown path received: %q, %q", u.Path, basePath)
continue
}
subPath := u.Path[len(baseURL.Path):]
subPath := u.Path[len(basePath):]
if f.opt.Enc != encoder.EncodeZero {
subPath = f.opt.Enc.ToStandardPath(subPath)
}
@@ -1148,7 +1182,7 @@ func (f *Fs) About(ctx context.Context) (*fs.Usage, error) {
return f.shouldRetry(ctx, resp, err)
})
if err != nil {
return nil, fmt.Errorf("about call failed: %w", err)
return nil, err
}
usage := &fs.Usage{}
if i, err := strconv.ParseInt(q.Used, 10, 64); err == nil && i >= 0 {
+6 -3
View File
@@ -173,8 +173,8 @@ func buildZip(dir string) string {
func buildDebAndRpm(dir, version, goarch string) []string {
// Make internal version number acceptable to .deb and .rpm
pkgVersion := version[1:]
pkgVersion = strings.Replace(pkgVersion, "β", "-beta", -1)
pkgVersion = strings.Replace(pkgVersion, "-", ".", -1)
pkgVersion = strings.ReplaceAll(pkgVersion, "β", "-beta")
pkgVersion = strings.ReplaceAll(pkgVersion, "-", ".")
nfpmArch, ok := goarchToNfpm[goarch]
if !ok {
nfpmArch = goarch
@@ -265,9 +265,12 @@ func buildWindowsResourceSyso(goarch string, versionTag string) string {
"-o",
sysoPath,
}
if goarch == "amd64" {
if strings.Contains(goarch, "64") {
args = append(args, "-64") // Make the syso a 64-bit coff file
}
if strings.Contains(goarch, "arm") {
args = append(args, "-arm") // Make the syso an arm binary
}
args = append(args, jsonPath)
err = runEnv(args, nil)
if err != nil {
+1
View File
@@ -48,6 +48,7 @@ docs = [
"hdfs.md",
"http.md",
"hubic.md",
"internetarchive.md",
"jottacloud.md",
"koofr.md",
"mailru.md",
+1 -1
View File
@@ -102,7 +102,7 @@ see complete list in [documentation](https://rclone.org/overview/#optional-featu
}
u, err := doAbout(context.Background())
if err != nil {
return fmt.Errorf("About call failed: %w", err)
return fmt.Errorf("about call failed: %w", err)
}
if u == nil {
return errors.New("nil usage returned")
+1 -1
View File
@@ -489,7 +489,7 @@ func resolveExitCode(err error) {
os.Exit(exitcode.TransferExceeded)
case fserrors.ShouldRetry(err):
os.Exit(exitcode.RetryError)
case fserrors.IsNoRetryError(err):
case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err):
os.Exit(exitcode.NoRetryError)
case fserrors.IsFatalError(err):
os.Exit(exitcode.FatalError)
+3 -4
View File
@@ -1,7 +1,6 @@
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
// +build cmount
// +build cgo
// +build linux darwin freebsd windows
// +build linux,cgo darwin,cgo freebsd,cgo windows
package cmount
@@ -13,12 +12,12 @@ import (
"sync/atomic"
"time"
"github.com/winfsp/cgofuse/fuse"
"github.com/rclone/rclone/cmd/mountlib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/vfs"
"github.com/winfsp/cgofuse/fuse"
)
const fhUnset = ^uint64(0)
+3 -4
View File
@@ -2,10 +2,9 @@
//
// This uses the cgo based cgofuse library
//go:build cmount && cgo && (linux || darwin || freebsd || windows)
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows)
// +build cmount
// +build cgo
// +build linux darwin freebsd windows
// +build linux,cgo darwin,cgo freebsd,cgo windows
package cmount
@@ -18,12 +17,12 @@ import (
"sync/atomic"
"time"
"github.com/winfsp/cgofuse/fuse"
"github.com/rclone/rclone/cmd/mountlib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/buildinfo"
"github.com/rclone/rclone/vfs"
"github.com/winfsp/cgofuse/fuse"
)
func init() {
+2 -3
View File
@@ -1,7 +1,6 @@
//go:build cmount && cgo && (linux || darwin || freebsd || windows) && (!race || !windows)
//go:build cmount && ((linux && cgo) || (darwin && cgo) || (freebsd && cgo) || windows) && (!race || !windows)
// +build cmount
// +build cgo
// +build linux darwin freebsd windows
// +build linux,cgo darwin,cgo freebsd,cgo windows
// +build !race !windows
// FIXME this doesn't work with the race detector under Windows either
+5 -2
View File
@@ -1,7 +1,10 @@
// Build for cmount for unsupported platforms to stop go complaining
// about "no buildable Go source files "
//go:build (!linux && !darwin && !freebsd && !windows) || !brew || !cgo || !cmount
// +build !linux,!darwin,!freebsd,!windows !brew !cgo !cmount
//go:build !((linux && cgo && cmount) || (darwin && cgo && cmount) || (freebsd && cgo && cmount) || (windows && cmount))
// +build !linux !cgo !cmount
// +build !darwin !cgo !cmount
// +build !freebsd !cgo !cmount
// +build !windows !cmount
package cmount
+2 -2
View File
@@ -1,5 +1,5 @@
//go:build cmount && cgo && windows
// +build cmount,cgo,windows
//go:build cmount && windows
// +build cmount,windows
package cmount
+3 -3
View File
@@ -79,7 +79,7 @@ rclone.org website.`,
var description = map[string]string{}
var addDescription func(root *cobra.Command)
addDescription = func(root *cobra.Command) {
name := strings.Replace(root.CommandPath(), " ", "_", -1) + ".md"
name := strings.ReplaceAll(root.CommandPath(), " ", "_") + ".md"
description[name] = root.Short
for _, c := range root.Commands() {
addDescription(c)
@@ -93,11 +93,11 @@ rclone.org website.`,
base := strings.TrimSuffix(name, path.Ext(name))
data := frontmatter{
Date: now,
Title: strings.Replace(base, "_", " ", -1),
Title: strings.ReplaceAll(base, "_", " "),
Description: description[name],
Slug: base,
URL: "/commands/" + strings.ToLower(base) + "/",
Source: strings.Replace(strings.Replace(base, "rclone", "cmd", -1), "_", "/", -1) + "/",
Source: strings.ReplaceAll(strings.ReplaceAll(base, "rclone", "cmd"), "_", "/") + "/",
}
var buf bytes.Buffer
err := frontmatterTemplate.Execute(&buf, data)
+13 -8
View File
@@ -78,6 +78,17 @@ type MountPoint struct {
ErrChan <-chan error
}
// NewMountPoint makes a new mounting structure
func NewMountPoint(mount MountFn, mountPoint string, f fs.Fs, mountOpt *Options, vfsOpt *vfscommon.Options) *MountPoint {
return &MountPoint{
MountFn: mount,
MountPoint: mountPoint,
Fs: f,
MountOpt: *mountOpt,
VFSOpt: *vfsOpt,
}
}
// Global constants
const (
MaxLeafSize = 1024 // don't pass file names longer than this
@@ -167,14 +178,7 @@ func NewMountCommand(commandName string, hidden bool, mount MountFn) *cobra.Comm
defer cmd.StartStats()()
}
mnt := &MountPoint{
MountFn: mount,
MountPoint: args[1],
Fs: cmd.NewFsDir(args),
MountOpt: Opt,
VFSOpt: vfsflags.Opt,
}
mnt := NewMountPoint(mount, args[1], cmd.NewFsDir(args), &Opt, &vfsflags.Opt)
daemon, err := mnt.Mount()
// Wait for foreground mount, if any...
@@ -253,6 +257,7 @@ func (m *MountPoint) Mount() (daemon *os.Process, err error) {
if err != nil {
return nil, fmt.Errorf("failed to mount FUSE fs: %w", err)
}
m.MountedOn = time.Now()
return nil, nil
}
+3 -12
View File
@@ -10,7 +10,6 @@ import (
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfsflags"
)
@@ -117,23 +116,15 @@ func mountRc(ctx context.Context, in rc.Params) (out rc.Params, err error) {
return nil, err
}
VFS := vfs.New(fdst, &vfsOpt)
_, unmountFn, err := mountFn(VFS, mountPoint, &mountOpt)
mnt := NewMountPoint(mountFn, mountPoint, fdst, &mountOpt, &vfsOpt)
_, err = mnt.Mount()
if err != nil {
log.Printf("mount FAILED: %v", err)
return nil, err
}
// Add mount to list if mount point was successfully created
liveMounts[mountPoint] = &MountPoint{
MountPoint: mountPoint,
MountedOn: time.Now(),
MountFn: mountFn,
UnmountFn: unmountFn,
MountOpt: mountOpt,
VFSOpt: vfsOpt,
Fs: fdst,
}
liveMounts[mountPoint] = mnt
fs.Debugf(nil, "Mount for %s created at %s using %s", fdst.String(), mountPoint, mountType)
return nil, nil
+56 -39
View File
@@ -42,15 +42,31 @@ builds an in memory representation. rclone ncdu can be used during
this scanning phase and you will see it building up the directory
structure as it goes along.
Here are the keys - press '?' to toggle the help on and off
You can interact with the user interface using key presses,
press '?' to toggle the help on and off. The supported keys are:
` + strings.Join(helpText()[1:], "\n ") + `
Listed files/directories may be prefixed by a one-character flag,
some of them combined with a description in brackes at end of line.
These flags have the following meaning:
e means this is an empty directory, i.e. contains no files (but
may contain empty subdirectories)
~ means this is a directory where some of the files (possibly in
subdirectories) have unknown size, and therefore the directory
size may be underestimated (and average size inaccurate, as it
is average of the files with known sizes).
. means an error occurred while reading a subdirectory, and
therefore the directory size may be underestimated (and average
size inaccurate)
! means an error occurred while reading this directory
This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
rclone remotes. It is missing lots of features at the moment
but is useful as it stands.
Note that it might take some time to delete big files/folders. The
Note that it might take some time to delete big files/directories. The
UI won't respond in the meantime since the deletion is done synchronously.
`,
Run: func(command *cobra.Command, args []string) {
@@ -283,9 +299,9 @@ func (u *UI) biggestEntry() (biggest int64) {
return
}
for i := range u.entries {
size, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
if size > biggest {
biggest = size
attrs, _ := u.d.AttrI(u.sortPerm[i])
if attrs.Size > biggest {
biggest = attrs.Size
}
}
return
@@ -297,8 +313,8 @@ func (u *UI) hasEmptyDir() bool {
return false
}
for i := range u.entries {
_, count, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
if isDir && count == 0 {
attrs, _ := u.d.AttrI(u.sortPerm[i])
if attrs.IsDir && attrs.Count == 0 {
return true
}
}
@@ -343,9 +359,9 @@ func (u *UI) Draw() error {
if y >= h-1 {
break
}
size, count, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
attrs, err := u.d.AttrI(u.sortPerm[n])
fg := termbox.ColorWhite
if entriesHaveErrors {
if attrs.EntriesHaveErrors {
fg = termbox.ColorYellow
}
if err != nil {
@@ -356,15 +372,19 @@ func (u *UI) Draw() error {
fg, bg = bg, fg
}
mark := ' '
if isDir {
if attrs.IsDir {
mark = '/'
}
fileFlag := ' '
message := ""
if !readable {
if !attrs.Readable {
message = " [not read yet]"
}
if entriesHaveErrors {
if attrs.CountUnknownSize > 0 {
message = fmt.Sprintf(" [%d of %d files have unknown size, size may be underestimated]", attrs.CountUnknownSize, attrs.Count)
fileFlag = '~'
}
if attrs.EntriesHaveErrors {
message = " [some subdirectories could not be read, size may be underestimated]"
fileFlag = '.'
}
@@ -374,32 +394,29 @@ func (u *UI) Draw() error {
}
extras := ""
if u.showCounts {
ss := operations.CountStringField(count, u.humanReadable, 9) + " "
if count > 0 {
ss := operations.CountStringField(attrs.Count, u.humanReadable, 9) + " "
if attrs.Count > 0 {
extras += ss
} else {
extras += strings.Repeat(" ", len(ss))
}
}
var averageSize float64
if count > 0 {
averageSize = float64(size) / float64(count)
}
if u.showDirAverageSize {
ss := operations.SizeStringField(int64(averageSize), u.humanReadable, 9) + " "
if averageSize > 0 {
avg := attrs.AverageSize()
ss := operations.SizeStringField(int64(avg), u.humanReadable, 9) + " "
if avg > 0 {
extras += ss
} else {
extras += strings.Repeat(" ", len(ss))
}
}
if showEmptyDir {
if isDir && count == 0 && fileFlag == ' ' {
if attrs.IsDir && attrs.Count == 0 && fileFlag == ' ' {
fileFlag = 'e'
}
}
if u.showGraph {
bars := (size + perBar/2 - 1) / perBar
bars := (attrs.Size + perBar/2 - 1) / perBar
// clip if necessary - only happens during startup
if bars > 10 {
bars = 10
@@ -408,7 +425,7 @@ func (u *UI) Draw() error {
}
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
}
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
Linef(0, y, w, fg, bg, ' ', "%c %s %s%c%s%s", fileFlag, operations.SizeStringField(attrs.Size, u.humanReadable, 12), extras, mark, path.Base(entry.Remote()), message)
y++
}
}
@@ -559,14 +576,14 @@ type ncduSort struct {
// Less is part of sort.Interface.
func (ds *ncduSort) Less(i, j int) bool {
var iAvgSize, jAvgSize float64
isize, icount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
jsize, jcount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
iattrs, _ := ds.d.AttrI(ds.sortPerm[i])
jattrs, _ := ds.d.AttrI(ds.sortPerm[j])
iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
if icount > 0 {
iAvgSize = float64(isize / icount)
if iattrs.Count > 0 {
iAvgSize = iattrs.AverageSize()
}
if jcount > 0 {
jAvgSize = float64(jsize / jcount)
if jattrs.Count > 0 {
jAvgSize = jattrs.AverageSize()
}
switch {
@@ -575,33 +592,33 @@ func (ds *ncduSort) Less(i, j int) bool {
case ds.u.sortByName > 0:
break
case ds.u.sortBySize < 0:
if isize != jsize {
return isize < jsize
if iattrs.Size != jattrs.Size {
return iattrs.Size < jattrs.Size
}
case ds.u.sortBySize > 0:
if isize != jsize {
return isize > jsize
if iattrs.Size != jattrs.Size {
return iattrs.Size > jattrs.Size
}
case ds.u.sortByCount < 0:
if icount != jcount {
return icount < jcount
if iattrs.Count != jattrs.Count {
return iattrs.Count < jattrs.Count
}
case ds.u.sortByCount > 0:
if icount != jcount {
return icount > jcount
if iattrs.Count != jattrs.Count {
return iattrs.Count > jattrs.Count
}
case ds.u.sortByAverageSize < 0:
if iAvgSize != jAvgSize {
return iAvgSize < jAvgSize
}
// if avgSize is equal, sort by size
return isize < jsize
return iattrs.Size < jattrs.Size
case ds.u.sortByAverageSize > 0:
if iAvgSize != jAvgSize {
return iAvgSize > jAvgSize
}
// if avgSize is equal, sort by size
return isize > jsize
return iattrs.Size > jattrs.Size
}
// if everything equal, sort by name
return iname < jname
+51 -8
View File
@@ -16,14 +16,42 @@ type Dir struct {
parent *Dir
path string
mu sync.Mutex
count int64
size int64
count int64
countUnknownSize int64
entries fs.DirEntries
dirs map[string]*Dir
readError error
entriesHaveErrors bool
}
// Attrs contains accumulated properties for a directory entry
//
// Files with unknown size are counted separately but also included
// in the total count. They are not included in the size, i.e. treated
// as empty files, which means the size may be underestimated.
type Attrs struct {
Size int64
Count int64
CountUnknownSize int64
IsDir bool
Readable bool
EntriesHaveErrors bool
}
// AverageSize calculates average size of files in directory
//
// If there are files with unknown size, this returns the average over
// files with known sizes, which means it may be under- or
// overestimated.
func (a *Attrs) AverageSize() float64 {
countKnownSize := a.Count - a.CountUnknownSize
if countKnownSize > 0 {
return float64(a.Size) / float64(countKnownSize)
}
return 0
}
// Parent returns the directory above this one
func (d *Dir) Parent() *Dir {
// no locking needed since these are write once in newDir()
@@ -49,7 +77,13 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
for _, entry := range entries {
if o, ok := entry.(fs.Object); ok {
d.count++
d.size += o.Size()
size := o.Size()
if size < 0 {
// Some backends may return -1 because size of object is not known
d.countUnknownSize++
} else {
d.size += size
}
}
}
// Set my directory entry in parent
@@ -62,8 +96,9 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
// Accumulate counts in parents
for ; parent != nil; parent = parent.parent {
parent.mu.Lock()
parent.count += d.count
parent.size += d.size
parent.count += d.count
parent.countUnknownSize += d.countUnknownSize
if d.readError != nil {
parent.entriesHaveErrors = true
}
@@ -91,17 +126,24 @@ func (d *Dir) Remove(i int) {
// Call with d.mu held
func (d *Dir) remove(i int) {
size := d.entries[i].Size()
countUnknownSize := int64(0)
if size < 0 {
size = 0
countUnknownSize = 1
}
count := int64(1)
subDir, ok := d.getDir(i)
if ok {
size = subDir.size
count = subDir.count
countUnknownSize = subDir.countUnknownSize
delete(d.dirs, path.Base(subDir.path))
}
d.size -= size
d.count -= count
d.countUnknownSize -= countUnknownSize
d.entries = append(d.entries[:i], d.entries[i+1:]...)
dir := d
@@ -111,6 +153,7 @@ func (d *Dir) remove(i int) {
parent.dirs[path.Base(dir.path)] = dir
parent.size -= size
parent.count -= count
parent.countUnknownSize -= countUnknownSize
dir = parent
parent.mu.Unlock()
}
@@ -151,19 +194,19 @@ func (d *Dir) Attr() (size int64, count int64) {
}
// AttrI returns the size, count and flags for the i-th directory entry
func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
func (d *Dir) AttrI(i int) (attrs Attrs, err error) {
d.mu.Lock()
defer d.mu.Unlock()
subDir, isDir := d.getDir(i)
if !isDir {
return d.entries[i].Size(), 0, false, true, d.entriesHaveErrors, d.readError
return Attrs{d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors}, d.readError
}
if subDir == nil {
return 0, 0, true, false, false, nil
return Attrs{0, 0, 0, true, false, false}, nil
}
size, count = subDir.Attr()
return size, count, true, true, subDir.entriesHaveErrors, subDir.readError
size, count := subDir.Attr()
return Attrs{size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors}, subDir.readError
}
// Scan the Fs passed in, returning a root directory channel and an
+1 -1
View File
@@ -290,7 +290,7 @@ func list(ctx context.Context) error {
if !ok {
return errors.New("bad JSON")
}
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.Replace(info["Path"].(string), "/", "-", -1))
fmt.Printf("### %s: %s {#%s}\n\n", info["Path"], info["Title"], strings.ReplaceAll(info["Path"].(string), "/", "-"))
fmt.Printf("%s\n\n", info["Help"])
if authRequired := info["AuthRequired"]; authRequired != nil {
if authRequired.(bool) {
-1
View File
@@ -274,7 +274,6 @@ func (vol *Volume) mount(id string) error {
if _, err := vol.mnt.Mount(); err != nil {
return err
}
vol.mnt.MountedOn = time.Now()
vol.mountReqs[id] = nil
vol.drv.monChan <- false // ask monitor to refresh channels
return nil
+1 -1
View File
@@ -43,7 +43,7 @@ var shellUnEscapeRegex = regexp.MustCompile(`\\(.)`)
// Unescape a string that was escaped by rclone
func shellUnEscape(str string) string {
str = strings.Replace(str, "'\n'", "\n", -1)
str = strings.ReplaceAll(str, "'\n'", "\n")
str = shellUnEscapeRegex.ReplaceAllString(str, `$1`)
return str
}
+29 -4
View File
@@ -24,26 +24,51 @@ func init() {
var commandDefinition = &cobra.Command{
Use: "size remote:path",
Short: `Prints the total size and number of objects in remote:path.`,
Long: `
Counts objects in the path and calculates the total size. Prints the
result to standard output.
By default the output is in human-readable format, but shows values in
both human-readable format as well as the raw numbers (global option
` + "`--human-readable`" + ` is not considered). Use option ` + "`--json`" + `
to format output as JSON instead.
Recurses by default, use ` + "`--max-depth 1`" + ` to stop the
recursion.
Some backends do not always provide file sizes, see for example
[Google Photos](/googlephotos/#size) and
[Google Drive](/drive/#limitations-of-google-docs).
Rclone will then show a notice in the log indicating how many such
files were encountered, and count them in as empty files in the output
of the size command.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
var err error
var results struct {
Count int64 `json:"count"`
Bytes int64 `json:"bytes"`
Count int64 `json:"count"`
Bytes int64 `json:"bytes"`
Sizeless int64 `json:"sizeless"`
}
results.Count, results.Bytes, err = operations.Count(context.Background(), fsrc)
results.Count, results.Bytes, results.Sizeless, err = operations.Count(context.Background(), fsrc)
if err != nil {
return err
}
if results.Sizeless > 0 {
fs.Logf(fsrc, "Size may be underestimated due to %d objects with unknown size", results.Sizeless)
}
if jsonOutput {
return json.NewEncoder(os.Stdout).Encode(results)
}
fmt.Printf("Total objects: %s (%d)\n", fs.CountSuffix(results.Count), results.Count)
fmt.Printf("Total size: %s (%d Byte)\n", fs.SizeSuffix(results.Bytes).ByteUnit(), results.Bytes)
if results.Sizeless > 0 {
fmt.Printf("Total objects with unknown size: %s (%d)\n", fs.CountSuffix(results.Sizeless), results.Sizeless)
}
return nil
})
},
+1 -1
View File
@@ -25,7 +25,7 @@ var commandDefinition = &cobra.Command{
cmd.Run(false, false, command, func() error {
ctx := context.Background()
ci := fs.GetConfig(context.Background())
objects, _, err := operations.Count(ctx, fsrc)
objects, _, _, err := operations.Count(ctx, fsrc)
if err != nil {
return err
}
+4 -1
View File
@@ -113,8 +113,10 @@ WebDAV or S3, that work out of the box.)
{{< provider name="Box" home="https://www.box.com/" config="/box/" >}}
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
{{< provider name="Citrix ShareFile" home="http://sharefile.com/" config="/sharefile/" >}}
{{< provider name="C14" home="https://www.online.net/en/storage/c14-cold-storage" config="/s3/#scaleway" >}}
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Digi Storage" home="https://storage.rcs-rds.ro/" config="/koofr/#digi-storage" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
@@ -127,6 +129,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="HDFS" home="https://hadoop.apache.org/" config="/hdfs/" >}}
{{< provider name="HTTP" home="https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol" config="/http/" >}}
{{< provider name="Hubic" home="https://hubic.com/" config="/hubic/" >}}
{{< provider name="Internet Archive" home="https://archive.org/" config="/internetarchive/" >}}
{{< provider name="Jottacloud" home="https://www.jottacloud.com/en/" config="/jottacloud/" >}}
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
{{< provider name="Koofr" home="https://koofr.eu/" config="/koofr/" >}}
@@ -167,7 +170,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="The local filesystem" home="/local/" config="/local/" end="true">}}
{{< /provider_list >}}
Links
## Links
* {{< icon "fa fa-home" >}} [Home page](https://rclone.org/)
* {{< icon "fab fa-github" >}} [GitHub project page for source and bug tracker](https://github.com/rclone/rclone)
+1 -2
View File
@@ -334,6 +334,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+22
View File
@@ -583,3 +583,25 @@ put them back in again.` >}}
* rafma0 <int.main@gmail.com>
* Adrien Rey-Jarthon <jobs@adrienjarthon.com>
* Nick Gooding <73336146+nickgooding@users.noreply.github.com>
* Leroy van Logchem <lr.vanlogchem@gmail.com>
* Zsolt Ero <zsolt.ero@gmail.com>
* Lesmiscore <nao20010128@gmail.com>
* ehsantdy <ehsan.tadayon@arvancloud.com>
* SwazRGB <65694696+swazrgb@users.noreply.github.com>
* Mateusz Puczyński <mati6095@gmail.com>
* Michael C Tiernan - MIT-Research Computing Project <mtiernan@mit.edu>
* Kaspian <34658474+KaspianDev@users.noreply.github.com>
* Werner <EvilOlaf@users.noreply.github.com>
* Hugal31 <hugo.laloge@gmail.com>
* Christian Galo <36752715+cgalo5758@users.noreply.github.com>
* Erik van Velzen <erik@evanv.nl>
* Derek Battams <derek@battams.ca>
* SimonLiu <simonliu009@users.noreply.github.com>
* Hugo Laloge <hla@lescompanions.com>
* Mr-Kanister <68117355+Mr-Kanister@users.noreply.github.com>
* Rob Pickerill <r.pickerill@gmail.com>
* Andrey <to.merge@gmail.com>
* Eric Wolf <19wolf@gmail.com>
* Nick <nick.naumann@mailbox.tu-dresden.de>
* Jason Zheng <jszheng17@gmail.com>
* Matthew Vernon <mvernon@wikimedia.org>
+1 -2
View File
@@ -522,8 +522,7 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
## Azure Storage Emulator Support
+6 -2
View File
@@ -173,6 +173,11 @@ the file instead of hiding it.
Old versions of files, where available, are visible using the
`--b2-versions` flag.
It is also possible to view a bucket as it was at a certain point in time,
using the `--b2-version-at` flag. This will show the file versions as they
were at that time, showing files that have been deleted afterwards, and
hiding files that were created since.
If you wish to remove all the old versions then you can use the
`rclone cleanup remote:bucket` command which will delete all the old
versions of files, leaving the current ones intact. You can also
@@ -561,7 +566,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+3 -2
View File
@@ -106,7 +106,7 @@ Optional Flags:
Arbitrary rclone flags may be specified on the
[bisync command line](/commands/rclone_bisync/), for example
`rclone bsync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
`rclone bisync ./testdir/path1/ gdrive:testdir/path2/ --drive-skip-gdocs -v -v --timeout 10s`
Note that interactions of various rclone flags with bisync process flow
has not been fully tested yet.
@@ -363,6 +363,7 @@ Bisync is considered _BETA_ and has been tested with the following backends:
- OneDrive
- S3
- SFTP
- Yandex Disk
It has not been fully tested with other services yet.
If it works, or sorta works, please let us know and we'll update the list.
@@ -700,7 +701,7 @@ consider using the flag
Google docs exist as virtual files on Google Drive and cannot be transferred
to other filesystems natively. While it is possible to export a Google doc to
a normal file (with `.xlsx` extension, for example), it's not possible
a normal file (with `.xlsx` extension, for example), it is not possible
to import a normal file back into a Google document.
Bisync's handling of Google Doc files is to flag them in the run log output
+1 -2
View File
@@ -466,6 +466,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+42
View File
@@ -5,6 +5,48 @@ description: "Rclone Changelog"
# Changelog
## v1.58.1 - 2022-04-29
[See commits](https://github.com/rclone/rclone/compare/v1.58.0...v1.58.1)
* Bug Fixes
* build: Update github.com/billziss-gh to github.com/winfsp (Nick Craig-Wood)
* filter: Fix timezone of `--min-age`/`-max-age` from UTC to local as documented (Nick Craig-Wood)
* rc/js: Correct RC method names (Sơn Trần-Nguyễn)
* docs
* Fix some links to command pages (albertony)
* Add `--multi-thread-streams` note to `--transfers`. (Zsolt Ero)
* Mount
* Fix `--devname` and fusermount: unknown option 'fsname' when mounting via rc (Nick Craig-Wood)
* VFS
* Remove wording which suggests VFS is only for mounting (Nick Craig-Wood)
* Dropbox
* Fix retries of multipart uploads with incorrect_offset error (Nick Craig-Wood)
* Google Cloud Storage
* Use the s3 pacer to speed up transactions (Nick Craig-Wood)
* pacer: Default the Google pacer to a burst of 100 to fix gcs pacing (Nick Craig-Wood)
* Jottacloud
* Fix scope in token request (albertony)
* Netstorage
* Fix unescaped HTML in documentation (Nick Craig-Wood)
* Make levels of headings consistent (Nick Craig-Wood)
* Add support contacts to netstorage doc (Nil Alexandrov)
* Onedrive
* Note that sharepoint also changes web files (.html, .aspx) (GH)
* Putio
* Handle rate limit errors (Berkan Teber)
* Fix multithread download and other ranged requests (rafma0)
* S3
* Add ChinaMobile EOS to provider list (GuoXingbin)
* Sync providers in config description with providers (Nick Craig-Wood)
* SFTP
* Fix OpenSSH 8.8+ RSA keys incompatibility (KARBOWSKI Piotr)
* Note that Scaleway C14 is deprecating SFTP in favor of S3 (Adrien Rey-Jarthon)
* Storj
* Fix bucket creation on Move (Nick Craig-Wood)
* WebDAV
* Don't override Referer if user sets it (Nick Craig-Wood)
## v1.58.0 - 2022-03-18
[See commits](https://github.com/rclone/rclone/compare/v1.57.0...v1.58.0)
+19 -2
View File
@@ -50,6 +50,7 @@ See the following for detailed instructions for
* [HDFS](/hdfs/)
* [HTTP](/http/)
* [Hubic](/hubic/)
* [Internet Archive](/internetarchive/)
* [Jottacloud](/jottacloud/)
* [Koofr](/koofr/)
* [Mail.ru Cloud](/mailru/)
@@ -787,7 +788,9 @@ See `--compare-dest` and `--backup-dir`.
### --dedupe-mode MODE ###
Mode to run dedupe command in. One of `interactive`, `skip`, `first`, `newest`, `oldest`, `rename`. The default is `interactive`. See the dedupe command for more information as to what these options mean.
Mode to run dedupe command in. One of `interactive`, `skip`, `first`,
`newest`, `oldest`, `rename`. The default is `interactive`.
See the dedupe command for more information as to what these options mean.
### --disable FEATURE,FEATURE,... ###
@@ -1085,7 +1088,12 @@ have a signal to rotate logs.
### --log-format LIST ###
Comma separated list of log format options. Accepted options are `date`, `time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other keywords will be silently ignored. `pid` will tag log messages with process identifier which useful with `rclone mount --daemon`. Other accepted options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants). The default log format is "`date`,`time`".
Comma separated list of log format options. Accepted options are `date`,
`time`, `microseconds`, `pid`, `longfile`, `shortfile`, `UTC`. Any other
keywords will be silently ignored. `pid` will tag log messages with process
identifier which useful with `rclone mount --daemon`. Other accepted
options are explained in the [go documentation](https://pkg.go.dev/log#pkg-constants).
The default log format is "`date`,`time`".
### --log-level LEVEL ###
@@ -1793,6 +1801,8 @@ of timeouts or bigger if you have lots of bandwidth and a fast remote.
The default is to run 4 file transfers in parallel.
Look at --multi-thread-streams if you would like to control single file transfers.
### -u, --update ###
This forces rclone to skip any files which exist on the destination
@@ -1864,6 +1874,9 @@ With `-vv` rclone will become very verbose telling you about every
file it considers and transfers. Please send bug reports with a log
with this setting.
When setting verbosity as an environment variable, use
`RCLONE_VERBOSE=1` or `RCLONE_VERBOSE=2` for `-v` and `-vv` respectively.
### -V, --version ###
Prints the version number
@@ -2229,6 +2242,10 @@ override the environment variable setting.
Or to always use the trash in drive `--drive-use-trash`, set
`RCLONE_DRIVE_USE_TRASH=true`.
Verbosity is slightly different, the environment variable
equivalent of `--verbose` or `-v` is `RCLONE_VERBOSE=1`,
or for `-vv`, `RCLONE_VERBOSE=2`.
The same parser is used for the options and the environment variables
so they take exactly the same form.
+5 -2
View File
@@ -1375,8 +1375,11 @@ and upload the files if you prefer.
### Limitations of Google Docs
Google docs will appear as size -1 in `rclone ls` and as size 0 in
anything which uses the VFS layer, e.g. `rclone mount`, `rclone serve`.
Google docs will appear as size -1 in `rclone ls`, `rclone ncdu` etc,
and as size 0 in anything which uses the VFS layer, e.g. `rclone mount`
and `rclone serve`. When calculating directory totals, e.g. in
`rclone size` and `rclone ncdu`, they will be counted in as empty
files.
This is because rclone can't find out the size of the Google docs
without downloading them.
+1 -1
View File
@@ -31,7 +31,7 @@ the node running rclone would need to have lots of bandwidth.
The syncs would be incremental (on a file by file basis).
Eg
e.g.
rclone sync -i drive:Folder s3:bucket
+1 -2
View File
@@ -192,6 +192,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+10 -2
View File
@@ -131,7 +131,11 @@ The regular expressions used are as defined in the [Go regular
expression reference](https://golang.org/pkg/regexp/syntax/). Regular
expressions should be enclosed in `{{` `}}`. They will match only the
last path segment if the glob doesn't start with `/` or the whole path
name if it does.
name if it does. Note that rclone does not attempt to parse the
supplied regular expression, meaning that using any regular expression
filter will prevent rclone from using [directory filter rules](#directory_filter),
as it will instead check every path against
the supplied regular expression(s).
Here is how the `{{regexp}}` is transformed into an full regular
expression to match the entire path:
@@ -247,7 +251,7 @@ currently a means provided to pass regular expression filter options into
rclone directly though character class filter rules contain character
classes. [Go regular expression reference](https://golang.org/pkg/regexp/syntax/)
### How filter rules are applied to directories
### How filter rules are applied to directories {#directory_filter}
Rclone commands are applied to path/file names not
directories. The entire contents of a directory can be matched
@@ -263,6 +267,10 @@ recurse into subdirectories. This potentially optimises access to a remote
by avoiding listing unnecessary directories. Whether optimisation is
desirable depends on the specific filter rules and source remote content.
If any [regular expression filters](#regexp) are in use, then no
directory recursion optimisation is possible, as rclone must check
every path against the supplied regular expression(s).
Directory recursion optimisation occurs if either:
* A source remote does not support the rclone `ListR` primitive. local,
+1
View File
@@ -337,6 +337,7 @@ and may be set in the config file.
--ftp-concurrency int Maximum number of FTP simultaneous connections, 0 for unlimited
--ftp-disable-epsv Disable using EPSV even if server advertises support
--ftp-disable-mlsd Disable using MLSD even if server advertises support
--ftp-disable-utf8 Disable using UTF-8 even if server advertises support
--ftp-disable-tls13 Disable TLS 1.3 (workaround for FTP servers with buggy TLS)
--ftp-encoding MultiEncoder The encoding for the backend (default Slash,Del,Ctl,RightSpace,Dot)
--ftp-explicit-tls Use Explicit FTPS (FTP over TLS)
+12 -2
View File
@@ -267,6 +267,17 @@ Properties:
- Type: bool
- Default: false
#### --ftp-disable-utf8
Disable using UTF-8 even if server advertises support.
Properties:
- Config: disable_utf8
- Env Var: RCLONE_FTP_DISABLE_UTF8
- Type: bool
- Default: false
#### --ftp-writing-mdtm
Use MDTM to set modification time (VsFtpd quirk)
@@ -395,8 +406,7 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
The implementation of : `--dump headers`,
`--dump bodies`, `--dump auth` for debugging isn't the same as
+1 -2
View File
@@ -609,6 +609,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+1 -2
View File
@@ -220,6 +220,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+92 -38
View File
@@ -3,11 +3,11 @@ title: "Install"
description: "Rclone Installation"
---
# Install #
# Install
Rclone is a Go program and comes as a single binary file.
## Quickstart ##
## Quickstart
* [Download](/downloads/) the relevant binary.
* Extract the `rclone` executable, `rclone.exe` on Windows, from the archive.
@@ -22,20 +22,20 @@ run `rclone -h`.
Already installed rclone can be easily updated to the latest version
using the [rclone selfupdate](/commands/rclone_selfupdate/) command.
## Script installation ##
## Script installation
To install rclone on Linux/macOS/BSD systems, run:
curl https://rclone.org/install.sh | sudo bash
sudo -v ; curl https://rclone.org/install.sh | sudo bash
For beta installation, run:
curl https://rclone.org/install.sh | sudo bash -s beta
sudo -v ; curl https://rclone.org/install.sh | sudo bash -s beta
Note that this script checks the version of rclone installed first and
won't re-download if not needed.
## Linux installation from precompiled binary ##
## Linux installation from precompiled binary
Fetch and unpack
@@ -59,7 +59,7 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
rclone config
## macOS installation with brew ##
## macOS installation with brew
brew install rclone
@@ -68,7 +68,7 @@ NOTE: This version of rclone will not support `mount` any more (see
on macOS, either install a precompiled binary or enable the relevant option
when [installing from source](#install-from-source).
## macOS installation from precompiled binary, using curl ##
## macOS installation from precompiled binary, using curl
To avoid problems with macOS gatekeeper enforcing the binary to be signed and
notarized it is enough to download with `curl`.
@@ -96,20 +96,20 @@ Run `rclone config` to setup. See [rclone config docs](/docs/) for more details.
rclone config
## macOS installation from precompiled binary, using a web browser ##
## macOS installation from precompiled binary, using a web browser
When downloading a binary with a web browser, the browser will set the macOS
gatekeeper quarantine attribute. Starting from Catalina, when attempting to run
`rclone`, a pop-up will appear saying:
“rclone” cannot be opened because the developer cannot be verified.
"rclone" cannot be opened because the developer cannot be verified.
macOS cannot verify that this app is free from malware.
The simplest fix is to run
xattr -d com.apple.quarantine rclone
## Install with docker ##
## Install with docker
The rclone maintains a [docker image for rclone](https://hub.docker.com/r/rclone/rclone).
These images are autobuilt by docker hub from the rclone source based
@@ -188,39 +188,93 @@ ls ~/data/mount
kill %1
```
## Install from source ##
## Install from source
Make sure you have at least [Go](https://golang.org/) go1.16
installed. [Download go](https://golang.org/dl/) if necessary. The
latest release is recommended. Then
Make sure you have git and [Go](https://golang.org/) installed.
Go version 1.16 or newer is required, latest release is recommended.
You can get it from your package manager, or download it from
[golang.org/dl](https://golang.org/dl/). Then you can run the following:
```sh
```
git clone https://github.com/rclone/rclone.git
cd rclone
go build
# If on macOS and mount is wanted, instead run: make GOTAGS=cmount
./rclone version
```
This will leave you a checked out version of rclone you can modify and
send pull requests with. If you use `make` instead of `go build` then
the rclone build will have the correct version information in it.
This will check out the rclone source in subfolder rclone, which you can later
modify and send pull requests with. Then it will build the rclone executable
in the same folder. As an initial check you can now run `./rclone version`
(`.\rclone version` on Windows).
You can also build the latest stable rclone with:
Note that on macOS and Windows the [mount](https://rclone.org/commands/rclone_mount/)
command will not be available unless you specify additional build tag `cmount`.
go get github.com/rclone/rclone
```
go build -tags cmount
```
or the latest version (equivalent to the beta) with
This assumes you have a GCC compatible C compiler (GCC or Clang) in your PATH,
as it uses [cgo](https://pkg.go.dev/cmd/cgo). But on Windows, the
[cgofuse](https://github.com/winfsp/cgofuse) library that the cmount
implementation is based on, also supports building
[without cgo](https://github.com/golang/go/wiki/WindowsDLLs), i.e. by setting
environment variable CGO_ENABLED to value 0 (static linking). This is how the
official Windows release of rclone is being built, starting with version 1.59.
It is still possible to build with cgo on Windows as well, by using the MinGW
port of GCC, e.g. by installing it in a [MSYS2](https://www.msys2.org)
distribution (make sure you install it in the classic mingw64 subsystem, the
ucrt64 version is not compatible).
go get github.com/rclone/rclone@master
Additionally, on Windows, you must install the third party utility
[WinFsp](http://www.secfs.net/winfsp/), with the "Developer" feature selected.
If building with cgo, you must also set environment variable CPATH pointing to
the fuse include directory within the WinFsp installation
(normally `C:\Program Files (x86)\WinFsp\inc\fuse`).
These will build the binary in `$(go env GOPATH)/bin`
(`~/go/bin/rclone` by default) after downloading the source to the go
module cache. Note - do **not** use the `-u` flag here. This causes go
to try to update the dependencies that rclone uses and sometimes these
don't work with the current version of rclone.
You may also add arguments `-ldflags -s` (with or without `-tags cmount`),
to omit symbol table and debug information, making the executable file smaller,
and `-trimpath` to remove references to local file system paths. This is how
the official rclone releases are built.
## Installation with Ansible ##
```
go build -trimpath -ldflags -s -tags cmount
```
Instead of executing the `go build` command directly, you can run it via the
Makefile, which also sets version information and copies the resulting rclone
executable into your GOPATH bin folder (`$(go env GOPATH)/bin`, which
corresponds to `~/go/bin/rclone` by default).
```
make
```
To include mount command on macOS and Windows with Makefile build:
```
make GOTAGS=cmount
```
As an alternative you can download the source, build and install rclone in one
operation, as a regular Go package. The source will be stored it in the Go
module cache, and the resulting executable will be in your GOPATH bin folder
(`$(go env GOPATH)/bin`, which corresponds to `~/go/bin/rclone` by default).
With Go version 1.17 or newer:
```
go install github.com/rclone/rclone@latest
```
With Go versions older than 1.17 (do **not** use the `-u` flag, it causes Go to
try to update the dependencies that rclone uses and sometimes these don't work
with the current version):
```
go get github.com/rclone/rclone
```
## Installation with Ansible
This can be done with [Stefan Weichinger's ansible
role](https://github.com/stefangweichinger/ansible-rclone).
@@ -236,7 +290,7 @@ Instructions
- rclone
```
## Portable installation ##
## Portable installation
As mentioned [above](https://rclone.org/install/#quickstart), rclone is single
executable (`rclone`, or `rclone.exe` on Windows) that you can download as a
@@ -314,7 +368,7 @@ the [PsExec](https://docs.microsoft.com/en-us/sysinternals/downloads/psexec)
utility from Microsoft's Sysinternals suite, which takes option `-s` to
execute commands as the `SYSTEM` user.
#### Start from Startup folder ###
#### Start from Startup folder
To quickly execute an rclone command you can simply create a standard
Windows Explorer shortcut for the complete rclone command you want to run. If you
@@ -329,7 +383,7 @@ functionality to set it to run as different user, or to set conditions or
actions on certain events. Setting up a scheduled task as described below
will often give you better results.
#### Start from Task Scheduler ###
#### Start from Task Scheduler
Task Scheduler is an administrative tool built into Windows, and it can be used to
configure rclone to be started automatically in a highly configurable way, e.g.
@@ -339,14 +393,14 @@ be available to all users it can run as the `SYSTEM` user.
For technical information, see
https://docs.microsoft.com/windows/win32/taskschd/task-scheduler-start-page.
#### Run as service ###
#### Run as service
For running rclone at system startup, you can create a Windows service that executes
your rclone command, as an alternative to scheduled task configured to run at startup.
##### Mount command built-in service integration ####
##### Mount command built-in service integration
For mount commands, Rclone has a built-in Windows service integration via the third-party
For mount commands, rclone has a built-in Windows service integration via the third-party
WinFsp library it uses. Registering as a regular Windows service easy, as you just have to
execute the built-in PowerShell command `New-Service` (requires administrative privileges).
@@ -366,7 +420,7 @@ Windows standard methods for managing network drives. This is currently not
officially supported by Rclone, but with WinFsp version 2019.3 B2 / v1.5B2 or later
it should be possible through path rewriting as described [here](https://github.com/rclone/rclone/issues/3340).
##### Third-party service integration #####
##### Third-party service integration
To Windows service running any rclone command, the excellent third-party utility
[NSSM](http://nssm.cc), the "Non-Sucking Service Manager", can be used.
+10 -6
View File
@@ -12,7 +12,7 @@ set -e
#when adding a tool to the list make sure to also add its corresponding command further in the script
unzip_tools_list=('unzip' '7z' 'busybox')
usage() { echo "Usage: curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
usage() { echo "Usage: sudo -v ; curl https://rclone.org/install.sh | sudo bash [-s beta]" 1>&2; exit 1; }
#check for beta flag
if [ -n "$1" ] && [ "$1" != "beta" ]; then
@@ -81,6 +81,8 @@ case $OS in
;;
Darwin)
OS='osx'
binTgtDir=/usr/local/bin
man1TgtDir=/usr/local/share/man/man1
;;
SunOS)
OS='solaris'
@@ -171,12 +173,14 @@ case "$OS" in
;;
'osx')
#binary
mkdir -p /usr/local/bin
cp rclone /usr/local/bin/rclone.new
mv /usr/local/bin/rclone.new /usr/local/bin/rclone
mkdir -m 0555 -p ${binTgtDir}
cp rclone ${binTgtDir}/rclone.new
mv ${binTgtDir}/rclone.new ${binTgtDir}/rclone
chmod a=x ${binTgtDir}/rclone
#manual
mkdir -p /usr/local/share/man/man1
cp rclone.1 /usr/local/share/man/man1/
mkdir -m 0555 -p ${man1TgtDir}
cp rclone.1 ${man1TgtDir}
chmod a=r ${man1TgtDir}/rclone.1
;;
*)
echo 'OS not supported'
+222
View File
@@ -0,0 +1,222 @@
---
title: "Internet Archive"
description: "Rclone docs for Internet Archive"
---
# {{< icon "fa fa-archive" >}} Internet Archive
The Internet Archive backend utilizes Items on [archive.org](https://archive.org/)
Refer to [IAS3 API documentation](https://archive.org/services/docs/api/ias3.html) for the API this backend uses.
Paths are specified as `remote:bucket` (or `remote:` for the `lsd`
command.) You may put subdirectories in too, e.g. `remote:item/path/to/dir`.
Once you have made a remote (see the provider specific section above)
you can use it like this:
Unlike S3, listing up all items uploaded by you isn't supported.
Make a new item
rclone mkdir remote:item
List the contents of a item
rclone ls remote:item
Sync `/home/local/directory` to the remote item, deleting any excess
files in the item.
rclone sync -i /home/local/directory remote:item
## Notes
Because of Internet Archive's architecture, it enqueues write operations (and extra post-processings) in a per-item queue. You can check item's queue at https://catalogd.archive.org/history/item-name-here . Because of that, all uploads/deletes will not show up immediately and takes some time to be available.
The per-item queue is enqueued to an another queue, Item Deriver Queue. [You can check the status of Item Deriver Queue here.](https://catalogd.archive.org/catalog.php?whereami=1) This queue has a limit, and it may block you from uploading, or even deleting. You should avoid uploading a lot of small files for better behavior.
You can optionally wait for the server's processing to finish, by setting non-zero value to `wait_archive` key.
By making it wait, rclone can do normal file comparison.
Make sure to set a large enough value (e.g. `30m0s` for smaller files) as it can take a long time depending on server's queue.
## Configuration
Here is an example of making an internetarchive configuration.
Most applies to the other providers as well, any differences are described [below](#providers).
First run
rclone config
This will guide you through an interactive setup process.
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> remote
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
XX / InternetArchive Items
\ (internetarchive)
Storage> internetarchive
Option access_key_id.
IAS3 Access Key.
Leave blank for anonymous access.
You can find one here: https://archive.org/account/s3.php
Enter a value. Press Enter to leave empty.
access_key_id> XXXX
Option secret_access_key.
IAS3 Secret Key (password).
Leave blank for anonymous access.
Enter a value. Press Enter to leave empty.
secret_access_key> XXXX
Edit advanced config?
y) Yes
n) No (default)
y/n> y
Option endpoint.
IAS3 Endpoint.
Leave blank for default value.
Enter a string value. Press Enter for the default (https://s3.us.archive.org).
endpoint>
Option front_endpoint.
Host of InternetArchive Frontend.
Leave blank for default value.
Enter a string value. Press Enter for the default (https://archive.org).
front_endpoint>
Option disable_checksum.
Don't store MD5 checksum with object metadata.
Normally rclone will calculate the MD5 checksum of the input before
uploading it so it can ask the server to check the object against checksum.
This is great for data integrity checking but can cause long delays for
large files to start uploading.
Enter a boolean value (true or false). Press Enter for the default (true).
disable_checksum> true
Option encoding.
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Enter a encoder.MultiEncoder value. Press Enter for the default (Slash,Question,Hash,Percent,Del,Ctl,InvalidUtf8,Dot).
encoding>
Edit advanced config?
y) Yes
n) No (default)
y/n> n
--------------------
[remote]
type = internetarchive
access_key_id = XXXX
secret_access_key = XXXX
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
```
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/internetarchive/internetarchive.go then run make backenddocs" >}}
### Standard options
Here are the standard options specific to internetarchive (Internet Archive).
#### --internetarchive-access-key-id
IAS3 Access Key.
Leave blank for anonymous access.
You can find one here: https://archive.org/account/s3.php
Properties:
- Config: access_key_id
- Env Var: RCLONE_INTERNETARCHIVE_ACCESS_KEY_ID
- Type: string
- Required: false
#### --internetarchive-secret-access-key
IAS3 Secret Key (password).
Leave blank for anonymous access.
Properties:
- Config: secret_access_key
- Env Var: RCLONE_INTERNETARCHIVE_SECRET_ACCESS_KEY
- Type: string
- Required: false
### Advanced options
Here are the advanced options specific to internetarchive (Internet Archive).
#### --internetarchive-endpoint
IAS3 Endpoint.
Leave blank for default value.
Properties:
- Config: endpoint
- Env Var: RCLONE_INTERNETARCHIVE_ENDPOINT
- Type: string
- Default: "https://s3.us.archive.org"
#### --internetarchive-front-endpoint
Host of InternetArchive Frontend.
Leave blank for default value.
Properties:
- Config: front_endpoint
- Env Var: RCLONE_INTERNETARCHIVE_FRONT_ENDPOINT
- Type: string
- Default: "https://archive.org"
#### --internetarchive-disable-checksum
Don't ask the server to test against MD5 checksum calculated by rclone.
Normally rclone will calculate the MD5 checksum of the input before
uploading it so it can ask the server to check the object against checksum.
This is great for data integrity checking but can cause long delays for
large files to start uploading.
Properties:
- Config: disable_checksum
- Env Var: RCLONE_INTERNETARCHIVE_DISABLE_CHECKSUM
- Type: bool
- Default: true
#### --internetarchive-wait-archive
Timeout for waiting the server's processing tasks (specifically archive and book_op) to finish.
Only enable if you need to be guaranteed to be reflected after write operations.
0 to disable waiting. No errors to be thrown in case of timeout.
Properties:
- Config: wait_archive
- Env Var: RCLONE_INTERNETARCHIVE_WAIT_ARCHIVE
- Type: Duration
- Default: 0s
#### --internetarchive-encoding
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Properties:
- Config: encoding
- Env Var: RCLONE_INTERNETARCHIVE_ENCODING
- Type: MultiEncoder
- Default: Slash,LtGt,CrLf,Del,Ctl,InvalidUtf8,Dot
{{< rem autogenerated options stop >}}
+1 -2
View File
@@ -177,7 +177,6 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+32 -29
View File
@@ -32,6 +32,7 @@ Here is an overview of the major features of each cloud storage system.
| HDFS | - | Yes | No | No | - |
| HTTP | - | No | No | No | R |
| Hubic | MD5 | Yes | No | No | R/W |
| Internet Archive | MD5, SHA1, CRC32 | Yes | No | No | - |
| Jottacloud | MD5 | Yes | Yes | No | R |
| Koofr | MD5 | No | Yes | No | - |
| Mail.ru Cloud | Mailru ⁶ | Yes | Yes | No | - |
@@ -301,35 +302,36 @@ list of all possible values by passing an invalid value to this
flag, e.g. `--local-encoding "help"`. The command `rclone help flags encoding`
will show you the defaults for the backends.
| Encoding | Characters |
| --------- | ---------- |
| Asterisk | `*` |
| BackQuote | `` ` `` |
| BackSlash | `\` |
| Colon | `:` |
| CrLf | CR 0x0D, LF 0x0A |
| Ctl | All control characters 0x00-0x1F |
| Del | DEL 0x7F |
| Dollar | `$` |
| Dot | `.` or `..` as entire string |
| DoubleQuote | `"` |
| Hash | `#` |
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) |
| LeftCrLfHtVt | CR 0x0D, LF 0x0A,HT 0x09, VT 0x0B on the left of a string |
| LeftPeriod | `.` on the left of a string |
| LeftSpace | SPACE on the left of a string |
| LeftTilde | `~` on the left of a string |
| LtGt | `<`, `>` |
| None | No characters are encoded |
| Percent | `%` |
| Pipe | \| |
| Question | `?` |
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string |
| RightPeriod | `.` on the right of a string |
| RightSpace | SPACE on the right of a string |
| SingleQuote | `'` |
| Slash | `/` |
| SquareBracket | `[`, `]` |
| Encoding | Characters | Encoded as |
| --------- | ---------- | ---------- |
| Asterisk | `*` | `*` |
| BackQuote | `` ` `` | ``` |
| BackSlash | `\` | `\` |
| Colon | `:` | `:` |
| CrLf | CR 0x0D, LF 0x0A | `␍`, `␊` |
| Ctl | All control characters 0x00-0x1F | `␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟` |
| Del | DEL 0x7F | `␡` |
| Dollar | `$` | `$` |
| Dot | `.` or `..` as entire string | `.`, `..` |
| DoubleQuote | `"` | `"` |
| Hash | `#` | `#` |
| InvalidUtf8 | An invalid UTF-8 character (e.g. latin1) | `�` |
| LeftCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the left of a string | `␍`, `␊`, `␉`, `␋` |
| LeftPeriod | `.` on the left of a string | `.` |
| LeftSpace | SPACE on the left of a string | `␠` |
| LeftTilde | `~` on the left of a string | `~` |
| LtGt | `<`, `>` | `<`, `>` |
| None | No characters are encoded | |
| Percent | `%` | `%` |
| Pipe | \| | `|` |
| Question | `?` | `?` |
| RightCrLfHtVt | CR 0x0D, LF 0x0A, HT 0x09, VT 0x0B on the right of a string | `␍`, `␊`, `␉`, `␋` |
| RightPeriod | `.` on the right of a string | `.` |
| RightSpace | SPACE on the right of a string | `␠` |
| Semicolon | `;` | `;` |
| SingleQuote | `'` | `'` |
| Slash | `/` | `/` |
| SquareBracket | `[`, `]` | `[`, `]` |
##### Encoding example: FTP
@@ -426,6 +428,7 @@ upon backend-specific capabilities.
| HDFS | Yes | No | Yes | Yes | No | No | Yes | No | Yes | Yes |
| HTTP | No | No | No | No | No | No | No | No | No | Yes |
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No | Yes | No |
| Internet Archive | No | Yes | No | No | Yes | Yes | No | Yes | Yes | No |
| Jottacloud | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
| Mail.ru Cloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
| Mega | Yes | No | Yes | Yes | Yes | No | No | Yes | Yes | Yes |
+1 -2
View File
@@ -318,6 +318,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+11
View File
@@ -150,6 +150,16 @@ use these methods. The alternative is to use `--rc-user` and
Default Off.
### --rc-baseurl
Prefix for URLs.
Default is root
### --rc-template
User-specified template.
## Accessing the remote control via the rclone rc command {#api-rc}
Rclone itself implements the remote control protocol in its `rclone
@@ -1285,6 +1295,7 @@ Returns:
- count - number of files
- bytes - number of bytes in those files
- sizeless - number of files with unknown size, included in count but not accounted for in bytes
See the [size command](/commands/rclone_size/) command for more information on the above.
+21
View File
@@ -92,3 +92,24 @@ Configuration file is stored at:
Now transfer it to the remote box (scp, cut paste, ftp, sftp, etc.) and
place it in the correct place (use `rclone config file` on the remote
box to find out where).
## Configuring using SSH Tunnel ##
Linux and MacOS users can utilize SSH Tunnel to redirect the headless box port 53682 to local machine by using the following command:
```
ssh -L localhost:53682:localhost:53682 username@remote_server
```
Then on the headless box run `rclone` config and answer `Y` to the `Use
auto config?` question.
```
...
Remote config
Use auto config?
* Say Y if not sure
* Say N if you are working on a remote or headless machine
y) Yes (default)
n) No
y/n> y
```
Then copy and paste the auth url `http://127.0.0.1:53682/auth?state=xxxxxxxxxxxx` to the browser on your local machine, complete the auth and it is done.
+278 -26
View File
@@ -12,6 +12,8 @@ The S3 backend can be used with a number of different providers:
{{< provider name="Alibaba Cloud (Aliyun) Object Storage System (OSS)" home="https://www.alibabacloud.com/product/oss/" config="/s3/#alibaba-oss" >}}
{{< provider name="Ceph" home="http://ceph.com/" config="/s3/#ceph" >}}
{{< provider name="China Mobile Ecloud Elastic Object Storage (EOS)" home="https://ecloud.10086.cn/home/product-introduction/eos/" config="/s3/#china-mobile-ecloud-eos" >}}
{{< provider name="Cloudflare R2" home="https://blog.cloudflare.com/r2-open-beta/" config="/s3/#cloudflare-r2" >}}
{{< provider name="Arvan Cloud Object Storage (AOS)" home="https://www.arvancloud.com/en/products/cloud-storage" config="/s3/#arvan-cloud-object-storage-aos" >}}
{{< provider name="DigitalOcean Spaces" home="https://www.digitalocean.com/products/object-storage/" config="/s3/#digitalocean-spaces" >}}
{{< provider name="Dreamhost" home="https://www.dreamhost.com/cloud/storage/" config="/s3/#dreamhost" >}}
{{< provider name="IBM COS S3" home="http://www.ibm.com/cloud/object-storage" config="/s3/#ibm-cos-s3" >}}
@@ -70,7 +72,7 @@ name> remote
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, Dreamhost, IBM COS, Minio, and Tencent COS
XX / Amazon S3 Compliant Storage Providers including AWS, Ceph, ChinaMobile, ArvanCloud, Dreamhost, IBM COS, Minio, and Tencent COS
\ "s3"
[snip]
Storage> s3
@@ -567,7 +569,7 @@ A simple solution is to set the `--s3-upload-cutoff 0` and force all the files t
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/s3/s3.go then run make backenddocs" >}}
### Standard options
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
Here are the standard options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
#### --s3-provider
@@ -588,6 +590,8 @@ Properties:
- Ceph Object Storage
- "ChinaMobile"
- China Mobile Ecloud Elastic Object Storage (EOS)
- "ArvanCloud"
- Arvan Cloud Object Storage (AOS)
- "DigitalOcean"
- Digital Ocean Spaces
- "Dreamhost"
@@ -829,7 +833,7 @@ Properties:
- Config: region
- Env Var: RCLONE_S3_REGION
- Provider: !AWS,Alibaba,ChinaMobile,RackCorp,Scaleway,Storj,TencentCOS
- Provider: !AWS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,Storj,TencentCOS
- Type: string
- Required: false
- Examples:
@@ -926,6 +930,24 @@ Endpoint for China Mobile Ecloud Elastic Object Storage (EOS) API.
- "eos-anhui-1.cmecloud.cn"
- Anhui China (Huainan)
#### --s3-endpoint
Endpoint for Arvan Cloud Object Storage (AOS) API.
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Provider: ArvanCloud
- Type: string
- Required: false
- Examples:
- "s3.ir-thr-at1.arvanstorage.com"
- The default endpoint - a good choice if you are unsure.
- Tehran Iran (Asiatech)
- "s3.ir-tbz-sh1.arvanstorage.com"
- Tabriz Iran (Shahriar)
#### --s3-endpoint
Endpoint for IBM COS S3 API.
@@ -1295,7 +1317,7 @@ Properties:
- Config: endpoint
- Env Var: RCLONE_S3_ENDPOINT
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,Scaleway,StackPath,Storj,RackCorp
- Provider: !AWS,IBMCOS,TencentCOS,Alibaba,ChinaMobile,ArvanCloud,Scaleway,StackPath,Storj,RackCorp
- Type: string
- Required: false
- Examples:
@@ -1325,6 +1347,8 @@ Properties:
- Wasabi AP Northeast 1 (Tokyo) endpoint
- "s3.ap-northeast-2.wasabisys.com"
- Wasabi AP Northeast 2 (Osaka) endpoint
- "s3.ir-thr-at1.arvanstorage.com"
- ArvanCloud Tehran Iran (Asiatech) endpoint
#### --s3-location-constraint
@@ -1532,7 +1556,7 @@ Properties:
- Config: location_constraint
- Env Var: RCLONE_S3_LOCATION_CONSTRAINT
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,RackCorp,Scaleway,StackPath,Storj,TencentCOS
- Provider: !AWS,IBMCOS,Alibaba,ChinaMobile,ArvanCloud,RackCorp,Scaleway,StackPath,Storj,TencentCOS
- Type: string
- Required: false
@@ -1604,7 +1628,7 @@ Properties:
- Config: server_side_encryption
- Env Var: RCLONE_S3_SERVER_SIDE_ENCRYPTION
- Provider: AWS,Ceph,ChinaMobile,Minio
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Type: string
- Required: false
- Examples:
@@ -1705,6 +1729,19 @@ The storage class to use when storing new objects in ChinaMobile.
#### --s3-storage-class
The storage class to use when storing new objects in ArvanCloud.
- Config: storage_class
- Env Var: RCLONE_S3_STORAGE_CLASS
- Provider: ArvanCloud
- Type: string
- Required: false
- Examples:
- "STANDARD"
- Standard storage class
#### --s3-storage-class
The storage class to use when storing new objects in Tencent COS.
Properties:
@@ -1747,7 +1784,7 @@ Properties:
### Advanced options
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
Here are the advanced options specific to s3 (Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS).
#### --s3-bucket-acl
@@ -1799,7 +1836,7 @@ Properties:
- Config: sse_customer_algorithm
- Env Var: RCLONE_S3_SSE_CUSTOMER_ALGORITHM
- Provider: AWS,Ceph,ChinaMobile,Minio
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Type: string
- Required: false
- Examples:
@@ -1816,7 +1853,7 @@ Properties:
- Config: sse_customer_key
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY
- Provider: AWS,Ceph,ChinaMobile,Minio
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Type: string
- Required: false
- Examples:
@@ -1834,7 +1871,7 @@ Properties:
- Config: sse_customer_key_md5
- Env Var: RCLONE_S3_SSE_CUSTOMER_KEY_MD5
- Provider: AWS,Ceph,ChinaMobile,Minio
- Provider: AWS,Ceph,ChinaMobile,ArvanCloud,Minio
- Type: string
- Required: false
- Examples:
@@ -2443,12 +2480,19 @@ You will be able to list and copy data but not upload it.
This is the provider used as main example and described in the [configuration](#configuration) section above.
### AWS Snowball Edge
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware appliance used for transferring
bulk data back to AWS. Its main software interface is S3 object storage.
To use rclone with AWS Snowball Edge devices, configure as standard for an 'S3 Compatible Service'
be sure to set `upload_cutoff = 0` otherwise you will run into authentication header issues as
the snowball device does not support query parameter based authentication.
[AWS Snowball](https://aws.amazon.com/snowball/) is a hardware
appliance used for transferring bulk data back to AWS. Its main
software interface is S3 object storage.
To use rclone with AWS Snowball Edge devices, configure as standard
for an 'S3 Compatible Service'.
If using rclone pre v1.59 be sure to set `upload_cutoff = 0` otherwise
you will run into authentication header issues as the snowball device
does not support query parameter based authentication.
With rclone v1.59 or later setting `upload_cutoff` should not be necessary.
eg.
```
@@ -2487,10 +2531,11 @@ server_side_encryption =
storage_class =
```
If you are using an older version of CEPH, e.g. 10.2.x Jewel, then you
may need to supply the parameter `--s3-upload-cutoff 0` or put this in
the config file as `upload_cutoff 0` to work around a bug which causes
uploading of small files to fail.
If you are using an older version of CEPH (e.g. 10.2.x Jewel) and a
version of rclone before v1.59 then you may need to supply the
parameter `--s3-upload-cutoff 0` or put this in the config file as
`upload_cutoff 0` to work around a bug which causes uploading of small
files to fail.
Note also that Ceph sometimes puts `/` in the passwords it gives
users. If you read the secret access key using the command line tools
@@ -2517,6 +2562,106 @@ removed).
Because this is a json dump, it is encoding the `/` as `\/`, so if you
use the secret key as `xxxxxx/xxxx` it will work fine.
### Cloudflare R2 {#cloudflare-r2}
[Cloudflare R2](https://blog.cloudflare.com/r2-open-beta/) Storage
allows developers to store large amounts of unstructured data without
the costly egress bandwidth fees associated with typical cloud storage
services.
Here is an example of making a Cloudflare R2 configuration. First run:
rclone config
This will guide you through an interactive setup process.
Note that all buckets are private, and all are stored in the same
"auto" region. It is necessary to use Cloudflare workers to share the
content of a bucket publicly.
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> r2
Option Storage.
Type of storage to configure.
Choose a number from below, or type in your own value.
...
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, China Mobile, Cloudflare, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, Netease, RackCorp, Scaleway, SeaweedFS, StackPath, Storj, Tencent COS and Wasabi
\ (s3)
...
Storage> s3
Option provider.
Choose your S3 provider.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
...
XX / Cloudflare R2 Storage
\ (Cloudflare)
...
provider> Cloudflare
Option env_auth.
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars).
Only applies if access_key_id and secret_access_key is blank.
Choose a number from below, or type in your own boolean value (true or false).
Press Enter for the default (false).
1 / Enter AWS credentials in the next step.
\ (false)
2 / Get AWS credentials from the environment (env vars or IAM).
\ (true)
env_auth> 1
Option access_key_id.
AWS Access Key ID.
Leave blank for anonymous access or runtime credentials.
Enter a value. Press Enter to leave empty.
access_key_id> ACCESS_KEY
Option secret_access_key.
AWS Secret Access Key (password).
Leave blank for anonymous access or runtime credentials.
Enter a value. Press Enter to leave empty.
secret_access_key> SECRET_ACCESS_KEY
Option region.
Region to connect to.
Choose a number from below, or type in your own value.
Press Enter to leave empty.
1 / R2 buckets are automatically distributed across Cloudflare's data centers for low latency.
\ (auto)
region> 1
Option endpoint.
Endpoint for S3 API.
Required when using an S3 clone.
Enter a value. Press Enter to leave empty.
endpoint> https://ACCOUNT_ID.r2.cloudflarestorage.com
Edit advanced config?
y) Yes
n) No (default)
y/n> n
--------------------
y) Yes this is OK (default)
e) Edit this remote
d) Delete this remote
y/e/d> y
```
This will leave your config looking something like:
```
[r2]
type = s3
provider = Cloudflare
access_key_id = ACCESS_KEY
secret_access_key = SECRET_ACCESS_KEY
region = auto
endpoint = https://ACCOUNT_ID.r2.cloudflarestorage.com
acl = private
```
Now run `rclone lsf r2:` to see your buckets and `rclone lsf
r2:bucket` to look within a bucket.
### Dreamhost
Dreamhost [DreamObjects](https://www.dreamhost.com/cloud/storage/) is
@@ -2615,7 +2760,7 @@ Choose a number from below, or type in your own value
\ "alias"
2 / Amazon Drive
\ "amazon cloud drive"
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, Minio, IBM COS)
3 / Amazon S3 Complaint Storage Providers (Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio, IBM COS)
\ "s3"
4 / Backblaze B2
\ "b2"
@@ -2900,7 +3045,7 @@ Choose `s3` backend
Type of storage to configure.
Choose a number from below, or type in your own value.
[snip]
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
XX / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Lyve Cloud, Minio, RackCorp, SeaweedFS, and Tencent COS
\ (s3)
[snip]
Storage> s3
@@ -3087,7 +3232,7 @@ name> wasabi
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, Minio)
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
\ "s3"
[snip]
Storage> s3
@@ -3201,7 +3346,7 @@ Type of storage to configure.
Enter a string value. Press Enter for the default ("").
Choose a number from below, or type in your own value
[snip]
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
\ "s3"
[snip]
Storage> s3
@@ -3541,6 +3686,114 @@ d) Delete this remote
y/e/d> y
```
### ArvanCloud
[ArvanCloud](https://www.arvancloud.com/en/products/cloud-storage) ArvanCloud Object Storage goes beyond the limited traditional file storage.
It gives you access to backup and archived files and allows sharing.
Files like profile image in the app, images sent by users or scanned documents can be stored securely and easily in our Object Storage service.
ArvanCloud provides an S3 interface which can be configured for use with
rclone like this.
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
n/s> n
name> ArvanCloud
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Amazon S3 (also Dreamhost, Ceph, ChinaMobile, ArvanCloud, Minio)
\ "s3"
[snip]
Storage> s3
Get AWS credentials from runtime (environment variables or EC2/ECS meta data if no env vars). Only applies if access_key_id and secret_access_key is blank.
Choose a number from below, or type in your own value
1 / Enter AWS credentials in the next step
\ "false"
2 / Get AWS credentials from the environment (env vars or IAM)
\ "true"
env_auth> 1
AWS Access Key ID - leave blank for anonymous access or runtime credentials.
access_key_id> YOURACCESSKEY
AWS Secret Access Key (password) - leave blank for anonymous access or runtime credentials.
secret_access_key> YOURSECRETACCESSKEY
Region to connect to.
Choose a number from below, or type in your own value
/ The default endpoint - a good choice if you are unsure.
1 | US Region, Northern Virginia, or Pacific Northwest.
| Leave location constraint empty.
\ "us-east-1"
[snip]
region>
Endpoint for S3 API.
Leave blank if using ArvanCloud to use the default endpoint for the region.
Specify if using an S3 clone such as Ceph.
endpoint> s3.arvanstorage.com
Location constraint - must be set to match the Region. Used when creating buckets only.
Choose a number from below, or type in your own value
1 / Empty for Iran-Tehran Region.
\ ""
[snip]
location_constraint>
Canned ACL used when creating buckets and/or storing objects in S3.
For more info visit https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
Choose a number from below, or type in your own value
1 / Owner gets FULL_CONTROL. No one else has access rights (default).
\ "private"
[snip]
acl>
The server-side encryption algorithm used when storing this object in S3.
Choose a number from below, or type in your own value
1 / None
\ ""
2 / AES256
\ "AES256"
server_side_encryption>
The storage class to use when storing objects in S3.
Choose a number from below, or type in your own value
1 / Default
\ ""
2 / Standard storage class
\ "STANDARD"
storage_class>
Remote config
--------------------
[ArvanCloud]
env_auth = false
access_key_id = YOURACCESSKEY
secret_access_key = YOURSECRETACCESSKEY
region = ir-thr-at1
endpoint = s3.arvanstorage.com
location_constraint =
acl =
server_side_encryption =
storage_class =
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
This will leave the config file looking like this.
```
[ArvanCloud]
type = s3
provider = ArvanCloud
env_auth = false
access_key_id = YOURACCESSKEY
secret_access_key = YOURSECRETACCESSKEY
region =
endpoint = s3.arvanstorage.com
location_constraint =
acl =
server_side_encryption =
storage_class =
```
### Tencent COS {#tencent-cos}
[Tencent Cloud Object Storage (COS)](https://intl.cloud.tencent.com/product/cos) is a distributed storage service offered by Tencent Cloud for unstructured data. It is secure, stable, massive, convenient, low-delay and low-cost.
@@ -3574,7 +3827,7 @@ Choose a number from below, or type in your own value
\ "alias"
3 / Amazon Drive
\ "amazon cloud drive"
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
4 / Amazon S3 Compliant Storage Providers including AWS, Alibaba, Ceph, ChinaMobile, ArvanCloud, Digital Ocean, Dreamhost, IBM COS, Minio, and Tencent COS
\ "s3"
[snip]
Storage> s3
@@ -3787,5 +4040,4 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+1 -2
View File
@@ -251,6 +251,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+1 -2
View File
@@ -415,8 +415,7 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
## Known issues
+1 -2
View File
@@ -280,6 +280,5 @@ this capability cannot determine free space for an rclone mount or
use policy `mfs` (most free space) as a member of an rclone union
remote.
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features)
See [rclone about](https://rclone.org/commands/rclone_about/)
See [List of backends that do not support rclone about](https://rclone.org/overview/#optional-features) and [rclone about](https://rclone.org/commands/rclone_about/)
+1
View File
@@ -73,6 +73,7 @@
<a class="dropdown-item" href="/hdfs/"><i class="fa fa-globe"></i> HDFS (Hadoop Distributed Filesystem)</a>
<a class="dropdown-item" href="/http/"><i class="fa fa-globe"></i> HTTP</a>
<a class="dropdown-item" href="/hubic/"><i class="fa fa-space-shuttle"></i> Hubic</a>
<a class="dropdown-item" href="/internetarchive/"><i class="fa fa-archive"></i> Internet Archive</a>
<a class="dropdown-item" href="/jottacloud/"><i class="fa fa-cloud"></i> Jottacloud</a>
<a class="dropdown-item" href="/koofr/"><i class="fa fa-suitcase"></i> Koofr</a>
<a class="dropdown-item" href="/mailru/"><i class="fa fa-at"></i> Mail.ru Cloud</a>
+2 -2
View File
@@ -230,7 +230,7 @@ func ConfigChoose(state string, name string, help string, n int, getItem func(i
// StatePush pushes a new values onto the front of the config string
func StatePush(state string, values ...string) string {
for i := range values {
values[i] = strings.Replace(values[i], ",", ",", -1) // replace comma with unicode wide version
values[i] = strings.ReplaceAll(values[i], ",", ",") // replace comma with unicode wide version
}
if state != "" {
values = append(values[:len(values):len(values)], state)
@@ -262,7 +262,7 @@ func StatePop(state string) (newState string, value string) {
return "", state
}
value, newState = state[:comma], state[comma+1:]
value = strings.Replace(value, ",", ",", -1) // replace unicode wide comma with comma
value = strings.ReplaceAll(value, ",", ",") // replace unicode wide comma with comma
return newState, value
}
+36
View File
@@ -0,0 +1,36 @@
// Package chunksize calculates a suitable chunk size for large uploads
package chunksize
import (
"github.com/rclone/rclone/fs"
)
/*
Calculator calculates the minimum chunk size needed to fit within the maximum number of parts, rounded up to the nearest fs.Mebi
For most backends, (chunk_size) * (concurrent_upload_routines) memory will be required so we want to use the smallest
possible chunk size that's going to allow the upload to proceed. Rounding up to the nearest fs.Mebi on the assumption
that some backends may only allow integer type parameters when specifying the chunk size.
Returns the default chunk size if it is sufficiently large enough to support the given file size otherwise returns the
smallest chunk size necessary to allow the upload to proceed.
*/
func Calculator(objInfo fs.ObjectInfo, maxParts int, defaultChunkSize fs.SizeSuffix) fs.SizeSuffix {
fileSize := fs.SizeSuffix(objInfo.Size())
requiredChunks := fileSize / defaultChunkSize
if requiredChunks < fs.SizeSuffix(maxParts) || (requiredChunks == fs.SizeSuffix(maxParts) && fileSize%defaultChunkSize == 0) {
return defaultChunkSize
}
minChunk := fileSize / fs.SizeSuffix(maxParts)
remainder := minChunk % fs.Mebi
if remainder != 0 {
minChunk += fs.Mebi - remainder
}
if fileSize/minChunk == fs.SizeSuffix(maxParts) && fileSize%fs.SizeSuffix(maxParts) != 0 { // when right on the boundary, we need to add a MiB
minChunk += fs.Mebi
}
fs.Debugf(objInfo, "size: %v, parts: %v, default: %v, new: %v; default chunk size insufficient, returned new chunk size", fileSize, maxParts, defaultChunkSize, minChunk)
return minChunk
}
+40
View File
@@ -0,0 +1,40 @@
package chunksize
import (
"testing"
"time"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/object"
)
func TestComputeChunkSize(t *testing.T) {
tests := map[string]struct {
fileSize fs.SizeSuffix
maxParts int
defaultChunkSize fs.SizeSuffix
expected fs.SizeSuffix
}{
"default size returned when file size is small enough": {fileSize: 1000, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
"default size returned when file size is just 1 byte small enough": {fileSize: toSizeSuffixMiB(100000) - 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(10), expected: toSizeSuffixMiB(10)},
"no rounding up when everything divides evenly": {fileSize: toSizeSuffixMiB(1000000), maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(100)},
"rounding up to nearest MiB when not quite enough parts": {fileSize: toSizeSuffixMiB(1000000), maxParts: 9999, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
"rounding up to nearest MiB when one extra byte": {fileSize: toSizeSuffixMiB(1000000) + 1, maxParts: 10000, defaultChunkSize: toSizeSuffixMiB(100), expected: toSizeSuffixMiB(101)},
"expected MiB value when rounding sets to absolute minimum": {fileSize: toSizeSuffixMiB(1) - 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(1)},
"expected MiB value when rounding to absolute min with extra": {fileSize: toSizeSuffixMiB(1) + 1, maxParts: 1, defaultChunkSize: toSizeSuffixMiB(1), expected: toSizeSuffixMiB(2)},
}
for name, tc := range tests {
t.Run(name, func(t *testing.T) {
src := object.NewStaticObjectInfo("mock", time.Now(), int64(tc.fileSize), true, nil, nil)
result := Calculator(src, tc.maxParts, tc.defaultChunkSize)
if result != tc.expected {
t.Fatalf("expected: %v, got: %v", tc.expected, result)
}
})
}
}
func toSizeSuffixMiB(size int64) fs.SizeSuffix {
return fs.SizeSuffix(size * int64(fs.Mebi))
}
+2 -2
View File
@@ -248,11 +248,11 @@ func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) {
// "ignore-size") into an environment name
// "RCLONE_CONFIG_MY-REMOTE_IGNORE_SIZE"
func ConfigToEnv(section, name string) string {
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.Replace(name, "-", "_", -1))
return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.ReplaceAll(name, "-", "_"))
}
// OptionToEnv converts an option name, e.g. "ignore-size" into an
// environment name "RCLONE_IGNORE_SIZE"
func OptionToEnv(name string) string {
return "RCLONE_" + strings.ToUpper(strings.Replace(name, "-", "_", -1))
return "RCLONE_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_"))
}
+1 -1
View File
@@ -50,7 +50,7 @@ func setConfigFile(t *testing.T, data string) func() {
// toUnix converts \r\n to \n in buf
func toUnix(buf string) string {
if runtime.GOOS == "windows" {
return strings.Replace(buf, "\r\n", "\n", -1)
return strings.ReplaceAll(buf, "\r\n", "\n")
}
return buf
}
+1 -1
View File
@@ -417,7 +417,7 @@ func ChooseOption(o *fs.Option, name string) string {
fmt.Printf("Option %s.\n", o.Name)
if o.Help != "" {
// Show help string without empty lines.
help := strings.Replace(strings.TrimSpace(o.Help), "\n\n", "\n", -1)
help := strings.ReplaceAll(strings.TrimSpace(o.Help), "\n\n", "\n")
fmt.Println(help)
}
+1 -1
View File
@@ -208,7 +208,7 @@ loop:
value := path[prev : i-1]
// replace any doubled quotes if there were any
if doubled {
value = strings.Replace(value, string(quote)+string(quote), string(quote), -1)
value = strings.ReplaceAll(value, string(quote)+string(quote), string(quote))
}
prev = i + 1
parsed.Config[param] = value
+1 -1
View File
@@ -208,7 +208,7 @@ func parseHelperOptionString(optString string) (opts []string, err error) {
if len(param) == 0 {
return nil, errHelperEmptyOption
}
if param[0] == '-' || param[0] == '_' {
if param[0] == '-' {
return nil, errHelperOptionName
}
prev = i + 1
+1 -1
View File
@@ -20,7 +20,7 @@ func TestMountHelperArgs(t *testing.T) {
src: []string{},
dst: []string{"mount", "--daemon"},
}, {
src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env`},
src: []string{"-o", `x-systemd.automount,vvv,env.HTTPS_PROXY="a b;c,d?EF",ro,rw,args2env,_netdev`},
dst: []string{"mount", "--read-only", "--verbose=3", "--daemon"},
env: "HTTPS_PROXY=a b;c,d?EF",
}}
+14 -5
View File
@@ -138,8 +138,14 @@ func (o *RangeOption) Decode(size int64) (offset, limit int64) {
// absolute fetch using the size passed in and makes sure the range does
// not exceed filesize. Some remotes (e.g. Onedrive, Box) don't support
// range requests which index from the end.
//
// It also adjusts any SeekOption~s, turning them into absolute
// RangeOption~s instead.
func FixRangeOption(options []OpenOption, size int64) {
if size == 0 {
if size < 0 {
// Can't do anything for unknown length objects
return
} else if size == 0 {
// if size 0 then remove RangeOption~s
// replacing with a NullOptions~s which won't be rendered
for i := range options {
@@ -150,18 +156,21 @@ func FixRangeOption(options []OpenOption, size int64) {
}
return
}
for i := range options {
option := options[i]
if x, ok := option.(*RangeOption); ok {
for i, option := range options {
switch x := option.(type) {
case *RangeOption:
// If start is < 0 then fetch from the end
if x.Start < 0 {
x = &RangeOption{Start: size - x.End, End: -1}
options[i] = x
}
if x.End > size {
// If end is too big or undefined, fetch to the end
if x.End > size || x.End < 0 {
x = &RangeOption{Start: x.Start, End: size - 1}
options[i] = x
}
case *SeekOption:
options[i] = &RangeOption{Start: x.Offset, End: size - 1}
}
}
}
+26 -2
View File
@@ -149,6 +149,16 @@ func TestFixRangeOptions(t *testing.T) {
in: []OpenOption{},
want: []OpenOption{},
},
{
name: "Unknown size -1",
in: []OpenOption{
&RangeOption{Start: 1, End: -1},
},
want: []OpenOption{
&RangeOption{Start: 1, End: -1},
},
size: -1,
},
{
name: "Fetch a range with size=0",
in: []OpenOption{
@@ -183,7 +193,7 @@ func TestFixRangeOptions(t *testing.T) {
&RangeOption{Start: 1, End: -1},
},
want: []OpenOption{
&RangeOption{Start: 1, End: -1},
&RangeOption{Start: 1, End: 99},
},
size: 100,
},
@@ -193,7 +203,7 @@ func TestFixRangeOptions(t *testing.T) {
&RangeOption{Start: -1, End: 10},
},
want: []OpenOption{
&RangeOption{Start: 90, End: -1},
&RangeOption{Start: 90, End: 99},
},
size: 100,
},
@@ -207,6 +217,20 @@ func TestFixRangeOptions(t *testing.T) {
},
size: 100,
},
{
name: "SeekOption",
in: []OpenOption{
&HTTPOption{Key: "a", Value: "1"},
&SeekOption{Offset: 10},
&HTTPOption{Key: "b", Value: "2"},
},
want: []OpenOption{
&HTTPOption{Key: "a", Value: "1"},
&RangeOption{Start: 10, End: 99},
&HTTPOption{Key: "b", Value: "2"},
},
size: 100,
},
} {
FixRangeOption(test.in, test.size)
assert.Equal(t, test.want, test.in, test.name)
+1 -1
View File
@@ -121,7 +121,7 @@ func TestDeduplicateFirst(t *testing.T) {
// list until we get one object
var objects, size int64
for try := 1; try <= *fstest.ListRetries; try++ {
objects, size, err = operations.Count(context.Background(), r.Fremote)
objects, size, _, err = operations.Count(context.Background(), r.Fremote)
require.NoError(t, err)
if objects == 1 {
break
+45 -2
View File
@@ -796,6 +796,47 @@ func Overlapping(fdst, fsrc fs.Info) bool {
return strings.HasPrefix(fdstRoot, fsrcRoot) || strings.HasPrefix(fsrcRoot, fdstRoot)
}
// OverlappingFilterCheck returns true if fdst and fsrc point to the same
// underlying Fs and they overlap without fdst being excluded by any filter rule.
func OverlappingFilterCheck(ctx context.Context, fdst fs.Fs, fsrc fs.Fs) bool {
if !SameConfig(fdst, fsrc) {
return false
}
fdstRoot := fixRoot(fdst)
fsrcRoot := fixRoot(fsrc)
if strings.HasPrefix(fdstRoot, fsrcRoot) {
fdstRelative := fdstRoot[len(fsrcRoot):]
return filterCheckR(ctx, fdstRelative, 0, fsrc)
}
return strings.HasPrefix(fsrcRoot, fdstRoot)
}
// filterCheckR checks if fdst would be included in the sync
func filterCheckR(ctx context.Context, fdstRelative string, pos int, fsrc fs.Fs) bool {
include := true
fi := filter.GetConfig(ctx)
includeDirectory := fi.IncludeDirectory(ctx, fsrc)
dirs := strings.SplitAfterN(fdstRelative, "/", pos+2)
newPath := ""
for i := 0; i <= pos; i++ {
newPath += dirs[i]
}
if !strings.HasSuffix(newPath, "/") {
newPath += "/"
}
if strings.HasPrefix(fdstRelative, newPath) {
include, _ = includeDirectory(newPath)
if include {
if newPath == fdstRelative {
return true
}
pos++
include = filterCheckR(ctx, fdstRelative, pos, fsrc)
}
}
return include
}
// SameDir returns true if fdst and fsrc point to the same
// underlying Fs and they are the same directory.
func SameDir(fdst, fsrc fs.Info) bool {
@@ -1070,11 +1111,13 @@ func HashSumStream(ht hash.Type, outputBase64 bool, in io.ReadCloser, w io.Write
// Count counts the objects and their sizes in the Fs
//
// Obeys includes and excludes
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, err error) {
func Count(ctx context.Context, f fs.Fs) (objects int64, size int64, sizelessObjects int64, err error) {
err = ListFn(ctx, f, func(o fs.Object) {
atomic.AddInt64(&objects, 1)
objectSize := o.Size()
if objectSize > 0 {
if objectSize < 0 {
atomic.AddInt64(&sizelessObjects, 1)
} else if objectSize > 0 {
atomic.AddInt64(&size, objectSize)
}
})
+91 -3
View File
@@ -401,10 +401,11 @@ func TestCount(t *testing.T) {
// Check the MaxDepth too
ci.MaxDepth = 1
objects, size, err := operations.Count(ctx, r.Fremote)
objects, size, sizeless, err := operations.Count(ctx, r.Fremote)
require.NoError(t, err)
assert.Equal(t, int64(2), objects)
assert.Equal(t, int64(61), size)
assert.Equal(t, int64(0), sizeless)
}
func TestDelete(t *testing.T) {
@@ -1252,6 +1253,93 @@ func TestOverlapping(t *testing.T) {
}
}
// testFs is for unit testing fs.Fs
type testFs struct {
testFsInfo
}
func (i *testFs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
return nil, nil
}
func (i *testFs) NewObject(ctx context.Context, remote string) (fs.Object, error) { return nil, nil }
func (i *testFs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) {
return nil, nil
}
func (i *testFs) Mkdir(ctx context.Context, dir string) error { return nil }
func (i *testFs) Rmdir(ctx context.Context, dir string) error { return nil }
// copied from TestOverlapping because the behavior of OverlappingFilterCheck should be identical to Overlapping
// when no filters are set
func TestOverlappingFilterCheckWithoutFilter(t *testing.T) {
ctx := context.Background()
src := &testFs{testFsInfo{name: "name", root: "root"}}
slash := string(os.PathSeparator) // native path separator
for _, test := range []struct {
name string
root string
expected bool
}{
{"name", "root", true},
{"namey", "root", false},
{"name", "rooty", false},
{"namey", "rooty", false},
{"name", "roo", false},
{"name", "root/toot", true},
{"name", "root/toot/", true},
{"name", "root" + slash + "toot", true},
{"name", "root" + slash + "toot" + slash, true},
{"name", "", true},
{"name", "/", true},
} {
dst := &testFs{testFsInfo{name: test.name, root: test.root}}
what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
actual := operations.OverlappingFilterCheck(ctx, src, dst)
assert.Equal(t, test.expected, actual, what)
actual = operations.OverlappingFilterCheck(ctx, dst, src)
assert.Equal(t, test.expected, actual, what)
}
}
func TestOverlappingFilterCheckWithFilter(t *testing.T) {
ctx := context.Background()
fi, err := filter.NewFilter(nil)
require.NoError(t, err)
require.NoError(t, fi.Add(false, "*/exclude/"))
fi.Opt.ExcludeFile = ".ignore"
ctx = filter.ReplaceConfig(ctx, fi)
src := &testFs{testFsInfo{name: "name", root: "root"}}
slash := string(os.PathSeparator) // native path separator
for _, test := range []struct {
name string
root string
expected bool
}{
{"name", "root", true},
{"name", "root/", true},
{"name", "root" + slash, true},
{"name", "root/exclude", false},
{"name", "root/exclude/", false},
{"name", "root" + slash + "exclude", false},
{"name", "root" + slash + "exclude" + slash, false},
{"name", "root/.ignore", false},
{"name", "root" + slash + ".ignore", false},
{"namey", "root/include", false},
{"namey", "root/include/", false},
{"namey", "root" + slash + "include", false},
{"namey", "root" + slash + "include" + slash, false},
} {
dst := &testFs{testFsInfo{name: test.name, root: test.root}}
what := fmt.Sprintf("(%q,%q) vs (%q,%q)", src.name, src.root, dst.name, dst.root)
actual := operations.OverlappingFilterCheck(ctx, dst, src)
assert.Equal(t, test.expected, actual, what)
}
}
func TestListFormat(t *testing.T) {
item0 := &operations.ListJSONItem{
Path: "a",
@@ -1406,7 +1494,7 @@ func TestDirMove(t *testing.T) {
require.NoError(t, operations.DirMove(ctx, r.Fremote, "A1", "A2"))
for i := range files {
files[i].Path = strings.Replace(files[i].Path, "A1/", "A2/", -1)
files[i].Path = strings.ReplaceAll(files[i].Path, "A1/", "A2/")
}
fstest.CheckListingWithPrecision(
@@ -1431,7 +1519,7 @@ func TestDirMove(t *testing.T) {
require.NoError(t, operations.DirMove(ctx, r.Fremote, "A2", "A3"))
for i := range files {
files[i].Path = strings.Replace(files[i].Path, "A2/", "A3/", -1)
files[i].Path = strings.ReplaceAll(files[i].Path, "A2/", "A3/")
}
fstest.CheckListingWithPrecision(
+2 -1
View File
@@ -344,13 +344,14 @@ func rcSize(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if err != nil {
return nil, err
}
count, bytes, err := Count(ctx, f)
count, bytes, sizeless, err := Count(ctx, f)
if err != nil {
return nil, err
}
out = make(rc.Params)
out["count"] = count
out["bytes"] = bytes
out["sizeless"] = sizeless
return out, nil
}
+3 -2
View File
@@ -451,8 +451,9 @@ func TestRcSize(t *testing.T) {
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params{
"count": int64(3),
"bytes": int64(120),
"count": int64(3),
"bytes": int64(120),
"sizeless": int64(0),
}, out)
}
Loaded 100 of 128 files, more files were not shown because too many files have changed in this diff. Show more